observability: add k8s API/kubelet tracing, Alloy, Mimir and Loki

Wire kube-apiserver and kubelet tracing to a Jaeger collector on docker-29, deploy Grafana Alloy in-cluster to ship logs/metrics, and stand up Mimir + Loki on docker-30 as their backing stores. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-09 00:05:21 +02:00
parent 95355ef7a5
commit d43ffd488e
29 changed files with 1586 additions and 1 deletions
--- a/vms-home/docker-29/readme.md
+++ b/vms-home/docker-29/readme.md
--- a/vms-home/docker-29/tracing/docker-compose.yaml
+++ b/vms-home/docker-29/tracing/docker-compose.yaml
@@ -0,0 +1,14 @@
+services:
+  jaeger:
+    image: jaegertracing/jaeger:2.18.0
+    container_name: jaeger
+    restart: unless-stopped
+    user: root
+    command: ["--config", "/jaeger/config.yaml"]
+    volumes:
+      - ./jaeger-config.yaml:/jaeger/config.yaml:ro
+      - ./badger:/badger
+    ports:
+      - "4317:4317"    # OTLP gRPC
+      - "4318:4318"    # OTLP HTTP
+      - "16686:16686"  # Jaeger UI
--- a/vms-home/docker-29/tracing/jaeger-config.yaml
+++ b/vms-home/docker-29/tracing/jaeger-config.yaml
@@ -0,0 +1,35 @@
+service:
+  extensions: [jaeger_storage, jaeger_query]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: []
+      exporters: [jaeger_storage_exporter]
+
+extensions:
+  jaeger_storage:
+    backends:
+      badger_store:
+        badger:
+          directories:
+            keys: /badger/key
+            values: /badger/data
+          ephemeral: false
+          ttl:
+            spans: 4h
+          metrics_update_interval: 5m
+  jaeger_query:
+    storage:
+      traces: badger_store
+
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+
+exporters:
+  jaeger_storage_exporter:
+    trace_storage: badger_store
--- a/vms-home/docker-30/loki/.env
+++ b/vms-home/docker-30/loki/.env
@@ -0,0 +1,2 @@
+GARAGE_ACCESS_KEY=GK...
+GARAGE_SECRET_KEY=...
--- a/vms-home/docker-30/loki/docker-compose.yaml
+++ b/vms-home/docker-30/loki/docker-compose.yaml
@@ -0,0 +1,41 @@
+services:
+  loki:
+    image: grafana/loki:3.7.3
+    container_name: loki
+    restart: unless-stopped
+    user: "10001:10001"
+    command:
+      - -config.file=/etc/loki/config.yaml
+      - -config.expand-env=true                # REQUIRED — expands ${GARAGE_*}
+    ports:
+      - "3100:3100"                            # LAN IP, not 0.0.0.0 — push+query
+    volumes:
+      - ./loki-config.yaml:/etc/loki/config.yaml:ro
+      - ./loki-data:/loki                       # WAL / tsdb-index / compactor scratch
+    env_file: [.env]                            # GARAGE_ACCESS_KEY / GARAGE_SECRET_KEY
+    networks: [obs]
+    healthcheck:
+      test: ["CMD-SHELL", "wget -qO- http://localhost:3100/ready || exit 1"]
+      interval: 15s
+      timeout: 5s
+      retries: 10
+      start_period: 30s
+    logging:                                    # don't let Loki's own logs balloon
+      driver: json-file
+      options: { max-size: "10m", max-file: "3" }
+
+  grafana:                                      # optional — drop if you query elsewhere
+    image: grafana/grafana:13.1.0
+    container_name: grafana
+    restart: unless-stopped
+    ports:
+      - "3001:3000" # LAN IP, not               # 3000 is taken by gitea
+    volumes:
+      - grafana-data:/var/lib/grafana
+    networks: [obs]
+
+volumes:
+  grafana-data:
+
+networks:
+  obs: { driver: bridge }
--- a/vms-home/docker-30/loki/loki-config.yaml
+++ b/vms-home/docker-30/loki/loki-config.yaml
@@ -0,0 +1,47 @@
+auth_enabled: false                        # single-tenant; sources split by label
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+  log_level: info
+
+common:
+  path_prefix: /loki
+  replication_factor: 1
+  ring:
+    kvstore: { store: inmemory }
+  storage:
+    s3:
+      endpoint: 192.168.0.30:3900          # no scheme; insecure toggles http
+      region: garage                       # MUST byte-match s3_region in garage.toml
+      bucketnames: loki-chunks
+      access_key_id: ${GARAGE_ACCESS_KEY}
+      secret_access_key: ${GARAGE_SECRET_KEY}
+      s3forcepathstyle: true               # mandatory for Garage
+      insecure: true                       # http on LAN; drop if you TLS-front Garage
+
+schema_config:
+  configs:
+    - from: 2026-07-01                     # <= first-ingest date; never mutate historical
+      store: tsdb
+      object_store: s3
+      schema: v13
+      index: { prefix: index_, period: 24h }
+
+storage_config:
+  tsdb_shipper:
+    active_index_directory: /loki/tsdb-index
+    cache_location: /loki/tsdb-cache
+
+limits_config:
+  allow_structured_metadata: true
+  volume_enabled: true
+  retention_period: 744h                   # 31d; compactor enforces
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+  max_line_size: 256KB
+  ingestion_rate_mb: 8                      # per-tenant; bump if you hit 429s
+  ingestion_burst_size_mb: 16
+
+compactor:
+  working_directory: /loki/compactor
--- a/vms-home/docker-30/mimir/.env
+++ b/vms-home/docker-30/mimir/.env
@@ -0,0 +1,2 @@
+GARAGE_ACCESS_KEY=GK...
+GARAGE_SECRET_KEY=...
--- a/vms-home/docker-30/mimir/docker-compose.yaml
+++ b/vms-home/docker-30/mimir/docker-compose.yaml
@@ -0,0 +1,28 @@
+services:
+  mimir:
+    image: grafana/mimir:3.1.2
+    container_name: mimir
+    restart: unless-stopped
+    command:
+      - -config.file=/etc/mimir/config.yaml
+      - -config.expand-env=true                # REQUIRED — expands ${GARAGE_*}
+      - -target=all                             # monolithic single-binary mode
+    ports:
+      - "9009:9009"                            # LAN IP, not 0.0.0.0 — push+query
+    volumes:
+      - ./mimir-config.yaml:/etc/mimir/config.yaml:ro
+      - ./mimir-data:/data                      # WAL / tsdb / compactor scratch
+    env_file: [.env]                            # GARAGE_ACCESS_KEY / GARAGE_SECRET_KEY
+    networks: [obs]
+    healthcheck:
+      test: ["CMD-SHELL", "wget -qO- http://localhost:9009/ready || exit 1"]
+      interval: 15s
+      timeout: 5s
+      retries: 10
+      start_period: 30s
+    logging:                                    # don't let Mimir's own logs balloon
+      driver: json-file
+      options: { max-size: "10m", max-file: "3" }
+
+networks:
+  obs: { driver: bridge }
--- a/vms-home/docker-30/mimir/mimir-config.yaml
+++ b/vms-home/docker-30/mimir/mimir-config.yaml
@@ -0,0 +1,42 @@
+multitenancy_enabled: false            # single-tenant; mirrors Loki's auth_enabled:false
+
+server:
+  http_listen_port: 9009
+  grpc_listen_port: 9095
+  log_level: info
+
+common:
+  storage:
+    backend: s3
+    s3:
+      endpoint: 192.168.0.30:3900      # no scheme; insecure toggles http
+      region: garage                   # MUST byte-match s3_region in garage.toml
+      access_key_id: ${GARAGE_ACCESS_KEY}
+      secret_access_key: ${GARAGE_SECRET_KEY}
+      insecure: true                   # http on LAN; drop if you TLS-front Garage
+      bucket_lookup_type: path         # path-style — Garage requirement (== s3forcepathstyle)
+
+blocks_storage:
+  s3: { bucket_name: mimir-blocks }
+  tsdb: { dir: /data/tsdb }
+  bucket_store: { sync_dir: /data/tsdb-sync }
+
+ruler_storage:
+  s3: { bucket_name: mimir-ruler }
+
+alertmanager_storage:
+  s3: { bucket_name: mimir-alertmanager }
+
+compactor:
+  data_dir: /data/compactor
+
+ruler:
+  rule_path: /data/ruler
+
+alertmanager:
+  data_dir: /data/alertmanager
+
+limits:
+  compactor_blocks_retention_period: 744h   # 31d; matches Loki's retention_period
+  ingestion_rate: 50000                      # samples/s per tenant; bump if you hit 429s
+  ingestion_burst_size: 100000
--- a/vms-home/docker-30/mimir/readme.md
+++ b/vms-home/docker-30/mimir/readme.md
@@ -0,0 +1,98 @@
+# Mimir
+
+Grafana Mimir, monolithic mode (`-target=all`), single-tenant, backed by Garage S3
+(same bucket-per-Garage-instance pattern as `../loki/`). Metrics ingest via
+Prometheus remote_write — Mimir does not scrape anything itself.
+
+- HTTP (query + push): `9009`
+- gRPC (internal): `9095`
+
+## 1. Garage setup (do this first — before `docker compose up`)
+
+Mimir needs three buckets. Create them and grant the existing Loki Garage key
+read+write access (reuse the key rather than minting a new one, unless you
+want tighter separation):
+
+```bash
+ssh novakj@192.168.0.30
+
+for b in mimir-blocks mimir-ruler mimir-alertmanager; do
+  docker exec garage /garage bucket create "$b"
+done
+
+docker exec garage /garage key list           # find the key name Loki already uses
+
+for b in mimir-blocks mimir-ruler mimir-alertmanager; do
+  docker exec garage /garage bucket allow --read --write "$b" --key <keyname>
+done
+```
+
+## 2. `.env`
+
+Reuse the same Garage credentials Loki uses (see `../loki/.env`):
+
+```
+GARAGE_ACCESS_KEY=GK...
+GARAGE_SECRET_KEY=...
+```
+
+## 3. Start
+
+```bash
+cd /path/to/mimir
+docker compose up -d
+```
+
+## 4. Add Mimir as a Grafana datasource
+
+Reuses the Grafana already running from `../loki/docker-compose.yaml` (`:3001`)
+— no second Grafana instance.
+
+- Type: **Prometheus**
+- URL: `http://192.168.0.30:9009/prometheus`
+- No `X-Scope-OrgID` header needed (`multitenancy_enabled: false`)
+
+## 5. Verify
+
+```bash
+curl http://192.168.0.30:9009/ready                # -> ready
+docker compose ps                                   # mimir healthy
+docker compose logs -f mimir                         # no S3 auth/bucket errors
+
+# push endpoint reachable (expect a 4xx proto-decode error, not connection refused)
+curl http://192.168.0.30:9009/api/v1/push
+```
+
+In Grafana → Explore → Mimir datasource, run `up` or `count({__name__!=""})`.
+Empty result is expected until a producer remote_writes into Mimir.
+
+After ~2h (first block flush interval), confirm blocks landed in Garage:
+
+```bash
+docker exec garage /garage bucket info mimir-blocks
+```
+
+## 6. Wiring a producer (later, not done here)
+
+Point any Prometheus/Alloy remote_write at Mimir:
+
+```yaml
+remote_write:
+  - url: http://192.168.0.30:9009/api/v1/push
+```
+
+`vms/utility-101-shadow/docker/monitoring/prometheus.yml` is the natural first
+candidate to wire up.
+
+## Notes / gotchas
+
+- `bucket_lookup_type: path` in `mimir-config.yaml` is mandatory for Garage
+  (equivalent to Loki's `s3forcepathstyle: true`).
+- `-config.expand-env=true` is required on the command line, or `${GARAGE_*}`
+  in `mimir-config.yaml` won't expand.
+- First metrics won't appear in Garage until the first block flush (~2h) —
+  don't panic if `mimir-blocks` looks empty right after startup.
+- Retention is 31d, set via `limits.compactor_blocks_retention_period` (matches
+  Loki's `retention_period`).
+- `region: garage` in the config must byte-match `s3_region` in Garage's
+  `garage.toml` — same requirement as Loki.