observability: add k8s API/kubelet tracing, Alloy, Mimir and Loki

Wire kube-apiserver and kubelet tracing to a Jaeger collector on
docker-29, deploy Grafana Alloy in-cluster to ship logs/metrics, and
stand up Mimir + Loki on docker-30 as their backing stores.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

vms-home/docker-30/loki/.env

@@ -0,0 +1,2 @@
+GARAGE_ACCESS_KEY=GK...
+GARAGE_SECRET_KEY=...

vms-home/docker-30/loki/docker-compose.yaml

@@ -0,0 +1,41 @@
+services:
+  loki:
+    image: grafana/loki:3.7.3
+    container_name: loki
+    restart: unless-stopped
+    user: "10001:10001"
+    command:
+      - -config.file=/etc/loki/config.yaml
+      - -config.expand-env=true                # REQUIRED — expands ${GARAGE_*}
+    ports:
+      - "3100:3100"                            # LAN IP, not 0.0.0.0 — push+query
+    volumes:
+      - ./loki-config.yaml:/etc/loki/config.yaml:ro
+      - ./loki-data:/loki                       # WAL / tsdb-index / compactor scratch
+    env_file: [.env]                            # GARAGE_ACCESS_KEY / GARAGE_SECRET_KEY
+    networks: [obs]
+    healthcheck:
+      test: ["CMD-SHELL", "wget -qO- http://localhost:3100/ready || exit 1"]
+      interval: 15s
+      timeout: 5s
+      retries: 10
+      start_period: 30s
+    logging:                                    # don't let Loki's own logs balloon
+      driver: json-file
+      options: { max-size: "10m", max-file: "3" }
+
+  grafana:                                      # optional — drop if you query elsewhere
+    image: grafana/grafana:13.1.0
+    container_name: grafana
+    restart: unless-stopped
+    ports:
+      - "3001:3000" # LAN IP, not               # 3000 is taken by gitea
+    volumes:
+      - grafana-data:/var/lib/grafana
+    networks: [obs]
+
+volumes:
+  grafana-data:
+
+networks:
+  obs: { driver: bridge }

vms-home/docker-30/loki/loki-config.yaml

@@ -0,0 +1,47 @@
+auth_enabled: false                        # single-tenant; sources split by label
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+  log_level: info
+
+common:
+  path_prefix: /loki
+  replication_factor: 1
+  ring:
+    kvstore: { store: inmemory }
+  storage:
+    s3:
+      endpoint: 192.168.0.30:3900          # no scheme; insecure toggles http
+      region: garage                       # MUST byte-match s3_region in garage.toml
+      bucketnames: loki-chunks
+      access_key_id: ${GARAGE_ACCESS_KEY}
+      secret_access_key: ${GARAGE_SECRET_KEY}
+      s3forcepathstyle: true               # mandatory for Garage
+      insecure: true                       # http on LAN; drop if you TLS-front Garage
+
+schema_config:
+  configs:
+    - from: 2026-07-01                     # <= first-ingest date; never mutate historical
+      store: tsdb
+      object_store: s3
+      schema: v13
+      index: { prefix: index_, period: 24h }
+
+storage_config:
+  tsdb_shipper:
+    active_index_directory: /loki/tsdb-index
+    cache_location: /loki/tsdb-cache
+
+limits_config:
+  allow_structured_metadata: true
+  volume_enabled: true
+  retention_period: 744h                   # 31d; compactor enforces
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+  max_line_size: 256KB
+  ingestion_rate_mb: 8                      # per-tenant; bump if you hit 429s
+  ingestion_burst_size_mb: 16
+
+compactor:
+  working_directory: /loki/compactor