From d43ffd488ecf7bd4b0d931ae545620836e190d91 Mon Sep 17 00:00:00 2001 From: Jan Novak Date: Thu, 9 Jul 2026 00:05:21 +0200 Subject: [PATCH] observability: add k8s API/kubelet tracing, Alloy, Mimir and Loki Wire kube-apiserver and kubelet tracing to a Jaeger collector on docker-29, deploy Grafana Alloy in-cluster to ship logs/metrics, and stand up Mimir + Loki on docker-30 as their backing stores. Co-Authored-By: Claude Sonnet 5 --- docs/kubernetes-tracing.md | 180 ++++++++++++++++++ docs/kubernetes/alloy.md | 142 ++++++++++++++ .../2026-05-21-2035-tracing-toggle-runbook.md | 140 ++++++++++++++ ...-21-2310-tracing-host-move-to-docker-29.md | 91 +++++++++ docs/plans/2026-07-08-0034-deploy-mimir.md | 165 ++++++++++++++++ .../2026-07-08-2346-alloy-metrics-to-mimir.md | 116 +++++++++++ .../alloy/helmrelease_allow-events.yaml | 42 ++++ .../alloy/helmrelease_alloy-logs.yaml | 106 +++++++++++ .../alloy/helmrelease_alloy-metrics.yaml | 49 +++++ .../alloy/helmrepository_grafana.yaml | 8 + .../alloy/rbac_log-collector.yaml | 15 ++ .../alloy/rbac_metrics-collector.yaml | 25 +++ gitops/home-kubernetes/alloy/readme.md | 10 + .../flux-system/extra-kustomizations.yaml | 15 ++ .../files/manifests/kube-apiserver.yaml | 1 + .../files/tracing-config.yaml | 4 + kubernetes-kvm-terraform/master.tf | 10 + ...-05-21 20:15 - k8s-pod-creation-tracing.md | 157 +++++++++++++++ vms-home/docker-29/readme.md | 0 .../docker-29/tracing/docker-compose.yaml | 14 ++ vms-home/docker-29/tracing/jaeger-config.yaml | 35 ++++ vms-home/docker-30/loki/.env | 2 + vms-home/docker-30/loki/docker-compose.yaml | 41 ++++ vms-home/docker-30/loki/loki-config.yaml | 47 +++++ vms-home/docker-30/mimir/.env | 2 + vms-home/docker-30/mimir/docker-compose.yaml | 28 +++ vms-home/docker-30/mimir/mimir-config.yaml | 42 ++++ vms-home/docker-30/mimir/readme.md | 98 ++++++++++ .../docker/monitoring/prometheus.yml | 2 +- 29 files changed, 1586 insertions(+), 1 deletion(-) create mode 100644 docs/kubernetes-tracing.md create mode 100644 docs/kubernetes/alloy.md create mode 100644 docs/plans/2026-05-21-2035-tracing-toggle-runbook.md create mode 100644 docs/plans/2026-05-21-2310-tracing-host-move-to-docker-29.md create mode 100644 docs/plans/2026-07-08-0034-deploy-mimir.md create mode 100644 docs/plans/2026-07-08-2346-alloy-metrics-to-mimir.md create mode 100644 gitops/home-kubernetes/alloy/helmrelease_allow-events.yaml create mode 100644 gitops/home-kubernetes/alloy/helmrelease_alloy-logs.yaml create mode 100644 gitops/home-kubernetes/alloy/helmrelease_alloy-metrics.yaml create mode 100644 gitops/home-kubernetes/alloy/helmrepository_grafana.yaml create mode 100644 gitops/home-kubernetes/alloy/rbac_log-collector.yaml create mode 100644 gitops/home-kubernetes/alloy/rbac_metrics-collector.yaml create mode 100644 gitops/home-kubernetes/alloy/readme.md create mode 100644 kubernetes-kvm-terraform/files/tracing-config.yaml create mode 100644 plans/2026-05-21 20:15 - k8s-pod-creation-tracing.md create mode 100644 vms-home/docker-29/readme.md create mode 100644 vms-home/docker-29/tracing/docker-compose.yaml create mode 100644 vms-home/docker-29/tracing/jaeger-config.yaml create mode 100644 vms-home/docker-30/loki/.env create mode 100644 vms-home/docker-30/loki/docker-compose.yaml create mode 100644 vms-home/docker-30/loki/loki-config.yaml create mode 100644 vms-home/docker-30/mimir/.env create mode 100644 vms-home/docker-30/mimir/docker-compose.yaml create mode 100644 vms-home/docker-30/mimir/mimir-config.yaml create mode 100644 vms-home/docker-30/mimir/readme.md diff --git a/docs/kubernetes-tracing.md b/docs/kubernetes-tracing.md new file mode 100644 index 0000000..783ce6b --- /dev/null +++ b/docs/kubernetes-tracing.md @@ -0,0 +1,180 @@ +# Kubernetes component tracing + +OpenTelemetry tracing for kube-apiserver and kubelet, shipping OTLP spans to Jaeger on docker-29. Useful for measuring pod-creation latency broken down by component. + +Related: [plans/k8s-pod-creation-tracing.md](../plans/2026-05-21%2020%3A15%20-%20k8s-pod-creation-tracing.md) — setup rationale, storage sizing, etcd tracing notes. + +--- + +## Where the toggles live + +| Component | Config on node | Restart trigger | +|---|---|---| +| kube-apiserver | `/etc/kubernetes/manifests/kube-apiserver.yaml` (flag) + `/etc/kubernetes/tracing-config.yaml` | kubelet detects manifest change, auto-restarts static pod (~5–10 s) | +| kubelet | `/var/lib/kubelet/config.yaml` (`tracing:` block) + `kube-system/kubelet-config` CM | `systemctl restart kubelet` | + +Nodes: kube-master-31 @ 192.168.0.31, kube-node-32 @ 192.168.0.32, kube-node-33 @ 192.168.0.33. + +--- + +## Quick suppress (no component restart) + +Stop the Jaeger backend — components keep building spans but have nowhere to send them. Lowest-risk; use this between measurement sessions. + +```bash +ssh novakj@192.168.0.29 'cd ~/docker-29/tracing && docker compose stop jaeger' +``` + +Resume: + +```bash +ssh novakj@192.168.0.29 'cd ~/docker-29/tracing && docker compose start jaeger' +``` + +Caveat: components still pay the (small) cost of building spans. For true zero-overhead, use **Full disable** below. + +--- + +## Full enable (live cluster) + +### 1. Start Jaeger on docker-30 + +```bash +ssh novakj@192.168.0.29 'cd ~/docker-29/tracing && docker compose up -d' +``` + +### 2. kube-apiserver (master only — 192.168.0.31) + +```bash +ssh novakj@192.168.0.31 bash -s <<'EOF' +# TracingConfiguration file +sudo tee /etc/kubernetes/tracing-config.yaml > /dev/null <<'YAML' +apiVersion: apiserver.config.k8s.io/v1beta1 +kind: TracingConfiguration +endpoint: 192.168.0.29:4317 +samplingRatePerMillion: 1000000 +YAML + +# Back up the manifest before touching it +sudo cp /etc/kubernetes/manifests/kube-apiserver.yaml /root/kube-apiserver.yaml.pre-tracing + +# Add the flag (after --tls-private-key-file line) +sudo sed -i '/--tls-private-key-file=/a\ - --tracing-config-file=/etc/kubernetes/tracing-config.yaml' \ + /etc/kubernetes/manifests/kube-apiserver.yaml + +# Wait for static pod to restart +sleep 10 +sudo crictl ps | grep apiserver +kubectl get --raw /livez +EOF +``` + +### 3. kubelet (all three nodes) + +Run on each node in turn — wait for `Ready` before moving to the next to avoid simultaneous NotReady: + +```bash +for NODE in 192.168.0.31 192.168.0.32 192.168.0.33; do + echo "==> $NODE" + ssh ubuntu@$NODE bash -s <<'EOF' +grep -q '^tracing:' /var/lib/kubelet/config.yaml && echo "already enabled, skipping" && exit 0 +sudo tee -a /var/lib/kubelet/config.yaml > /dev/null <<'YAML' +tracing: + endpoint: 192.168.0.29:4317 + samplingRatePerMillion: 1000000 +YAML +sudo systemctl restart kubelet +EOF + # wait for node Ready before proceeding + kubectl wait node --for=condition=Ready --timeout=60s $(kubectl get node -o wide | awk "/$NODE/{print \$1}") +done +``` + +### 4. Update the cluster ConfigMap (so future joins/reboots pick it up) + +```bash +kubectl -n kube-system edit cm kubelet-config +# under the `kubelet:` key, add: +# tracing: +# endpoint: 192.168.0.29:4317 +# samplingRatePerMillion: 1000000 +``` + +--- + +## Full disable (live cluster) + +### kube-apiserver (master only) + +```bash +ssh novakj@192.168.0.31 bash -s <<'EOF' +# Restore the pre-tracing manifest (exact revert) +sudo cp /root/kube-apiserver.yaml.pre-tracing /etc/kubernetes/manifests/kube-apiserver.yaml +# Optional: remove the config file +sudo rm -f /etc/kubernetes/tracing-config.yaml +EOF +``` + +Static pod restarts automatically. If the backup wasn't taken, remove the flag in-place instead: + +```bash +sudo sed -i '/--tracing-config-file=/d' /etc/kubernetes/manifests/kube-apiserver.yaml +``` + +### kubelet (all three nodes) + +```bash +for NODE in 192.168.0.31 192.168.0.32 192.168.0.33; do + echo "==> $NODE" + ssh novakj@$NODE bash -s <<'EOF' +sudo sed -i '/^tracing:/,/samplingRatePerMillion:/d' /var/lib/kubelet/config.yaml +sudo systemctl restart kubelet +EOF + kubectl wait node --for=condition=Ready --timeout=60s $(kubectl get node -o wide | awk "/$NODE/{print \$1}") +done +``` + +Remove the tracing block from the cluster ConfigMap too: + +```bash +kubectl -n kube-system edit cm kubelet-config +# remove the `tracing:` block under `kubelet:` +``` + +--- + +## Verification + +### After enable + +1. Open Jaeger UI: `http://192.168.0.29:16686` — Services dropdown should show `apiserver` and `kubelet` (appear after first traced request). + +2. Generate a full pod-lifecycle trace: + + ```bash + kubectl run trace-probe --image=registry.k8s.io/pause:3.10 --restart=Never + kubectl wait --for=condition=Ready pod/trace-probe --timeout=60s + kubectl delete pod trace-probe + ``` + +3. In Jaeger UI, search: + - Service `apiserver` — look for spans covering the create/watch flow for `trace-probe`. + - Service `kubelet` — look for `syncPod` spans tied to the same pod. + +4. The gap between apiserver spans and kubelet spans is scheduler time (kube-scheduler has no OTLP tracing in 1.32). Cross-check with `kubectl get events` or pod condition timestamps. + +### After disable + +Repeat step 2 above, then confirm no new spans appear in Jaeger for `apiserver` or `kubelet` within ~1 minute. + +--- + +## Sampling rate cheatsheet + +Change `samplingRatePerMillion` in `/etc/kubernetes/tracing-config.yaml` (apiserver) and `/var/lib/kubelet/config.yaml` (kubelet) then restart each component. + +| Value | Effective rate | Use case | +|---|---|---| +| `1000000` | 100% | Active measurement | +| `10000` | 1% | Steady-state background | +| `0` | off | Disable without removing config | diff --git a/docs/kubernetes/alloy.md b/docs/kubernetes/alloy.md new file mode 100644 index 0000000..6e399ba --- /dev/null +++ b/docs/kubernetes/alloy.md @@ -0,0 +1,142 @@ +# Alloy: deployment & operations + +Three Grafana Alloy instances run in the `monitoring` namespace, all deployed via +the Grafana `alloy` Helm chart (`>=1.8.0 <2.0.0`, image `v1.17.1`) and managed by +Flux with `driftDetection: enabled` (manual `kubectl edit`s get reverted — always +change config via git). Logs and events ship to Loki; metrics ship to Mimir. Every +instance stamps `cluster="homelab"` as an external label. + +--- + +## Instances + +| Instance | Controller | Collects | Sink | +|---|---|---|---| +| `alloy-logs` | DaemonSet (`runAsUser: 0`, one pod per node) | pod logs + node journald | Loki `http://192.168.0.30:3100/loki/api/v1/push` | +| `alloy-events` | Deployment (1 replica) | Kubernetes events | Loki (same endpoint) | +| `alloy-metrics` | Deployment (1 replica) | all ServiceMonitors + PodMonitors cluster-wide | Mimir `http://192.168.0.30:9009/api/v1/push` | + +`alloy-logs` and `alloy-events` are one-per-node / cluster-scoped respectively — +`alloy-metrics` is also a single Deployment, never a DaemonSet, since its +Prometheus-Operator discovery is already cluster-wide (a DaemonSet would scrape +every target once per node). + +--- + +## Files & Flux wiring + +All manifests live in `gitops/home-kubernetes/alloy/`: + +- `helmrepository_grafana.yaml` — the shared `grafana` HelmRepository. +- `helmrelease_alloy-logs.yaml` — logs DaemonSet. +- `helmrelease_allow-events.yaml` — events Deployment (note the `allow` filename typo, kept as-is). +- `helmrelease_alloy-metrics.yaml` — metrics Deployment. +- `rbac_log-collector.yaml` — ClusterRole `alloy-log-reader`, bound to the `alloy-logs`/`alloy-events` ServiceAccounts. +- `rbac_metrics-collector.yaml` — ClusterRole `alloy-metrics-reader`, bound to the `alloy-metrics` ServiceAccount. + +Each HelmRelease sets `rbac.create: false` — RBAC is supplied manually by the two +files above rather than by the chart, since the chart's default ClusterRole +doesn't cover everything each pipeline needs (journald/events reads, or +ServiceMonitor/PodMonitor/kubelet-metrics reads). + +Reconciliation: a Flux `Kustomization` named `alloy` in +`gitops/home-kubernetes/flux-system/extra-kustomizations.yaml`, pointing at +`./gitops/home-kubernetes/alloy`, with `dependsOn: kube-prometheus` — it needs +the `monitoring` namespace and the Prometheus Operator CRDs (ServiceMonitor/ +PodMonitor) to exist first. + +--- + +## How metrics discovery works + +`alloy-metrics` runs two Alloy components — `prometheus.operator.servicemonitors` +and `prometheus.operator.podmonitors` — which discover and scrape **every** +ServiceMonitor/PodMonitor in the cluster, then forward to +`prometheus.remote_write` targeting Mimir. There's no bespoke scrape config for +"standard Kubernetes metrics": kube-prometheus-stack already ships +ServiceMonitors for kubelet, cAdvisor, kube-state-metrics, node-exporter, +apiserver, and coredns, so those are picked up automatically. Any future +ServiceMonitor/PodMonitor added anywhere in the cluster is picked up the same way +with no Alloy config change. + +This is **additive** — kube-prometheus-stack's bundled Prometheus keeps scraping +the same targets into its own local 60d TSDB. Alloy scraping the same targets a +second time and shipping to Mimir is expected double-collection, not a bug. + +--- + +## Operating + +Status: + +```bash +flux get helmreleases -n monitoring +kubectl get pods -n monitoring -l app.kubernetes.io/name=alloy -o wide +``` + +Force reconcile: + +```bash +flux reconcile kustomization alloy --with-source +flux reconcile helmrelease alloy-metrics -n monitoring +``` + +Logs: + +```bash +kubectl -n monitoring logs deploy/alloy-metrics +kubectl -n monitoring logs ds/alloy-logs +kubectl -n monitoring logs deploy/alloy-events +``` + +Alloy UI (component graph, target health, remote_write queue status): + +```bash +kubectl -n monitoring port-forward deploy/alloy-metrics 12345:12345 +# open http://localhost:12345 +``` + +Editing config: change the inline `alloy.configMap.content` block in the +relevant `helmrelease_*.yaml`, commit, push, then force-reconcile (above). +Do not `kubectl edit` the generated ConfigMap directly — `driftDetection` will +revert it on the next reconcile. + +--- + +## Verification / smoke tests + +**Metrics reaching Mimir:** + +```bash +curl -s 'http://192.168.0.30:9009/prometheus/api/v1/query?query=up' | jq '.data.result | length' +# expect > 0 +``` + +Spot-check a few standard series exist and carry the cluster label: + +```bash +curl -s 'http://192.168.0.30:9009/prometheus/api/v1/query?query=kube_pod_info' | jq '.data.result[0].metric' +curl -s 'http://192.168.0.30:9009/prometheus/api/v1/query?query=node_cpu_seconds_total' | jq '.data.result | length' +curl -s 'http://192.168.0.30:9009/prometheus/api/v1/query?query=container_cpu_usage_seconds_total' | jq '.data.result | length' +# confirm "cluster": "homelab" is present in the returned label sets +``` + +**Logs reaching Loki:** + +```bash +curl -s http://192.168.0.30:3100/ready +# then in Grafana / logcli, query {cluster="homelab"} +``` + +--- + +## Troubleshooting + +| Symptom | Likely cause | +|---|---| +| `forbidden` errors in `alloy-metrics` logs | `alloy-metrics-reader` ClusterRole missing a resource/verb — check `rbac_metrics-collector.yaml` against what the component is trying to read | +| No targets shown in the Alloy UI | Prometheus Operator CRDs not installed, or `kube-prometheus` Flux Kustomization not Ready yet (metrics depends on it) | +| `prometheus.remote_write` shows failed sends / 4xx-5xx | Mimir down, or wrong endpoint — must be `http://192.168.0.30:9009/api/v1/push` exactly | +| Loki push failing | Loki down at `192.168.0.30:3100`, check with `curl http://192.168.0.30:3100/ready` | +| Same series appearing to be scraped twice | Expected — kube-prometheus-stack's bundled Prometheus and `alloy-metrics` both scrape the same ServiceMonitors by design (additive, not deduplicated) | +| Manual `kubectl edit` on the Alloy ConfigMap reverts itself | `driftDetection: enabled` — edit the HelmRelease in git instead | diff --git a/docs/plans/2026-05-21-2035-tracing-toggle-runbook.md b/docs/plans/2026-05-21-2035-tracing-toggle-runbook.md new file mode 100644 index 0000000..160d031 --- /dev/null +++ b/docs/plans/2026-05-21-2035-tracing-toggle-runbook.md @@ -0,0 +1,140 @@ +# Tracing enable/disable runbook for K8s components + +## Context + +The pod-creation tracing plan ([plans/2026-05-21 20:15 - k8s-pod-creation-tracing.md](../../plans/2026-05-21%2020%3A15%20-%20k8s-pod-creation-tracing.md)) covers the *one-time setup* of OTLP tracing across kube-apiserver and kubelet, plus the Jaeger backend on docker-29. What's missing is a short operational doc for **toggling tracing on/off as quickly as possible** on the running cluster — useful for short measurement experiments where we want overhead only when we're actively recording. + +The Terraform/cloud-init changes already in flight (`master.tf`, `files/manifests/kube-apiserver.yaml`, `files/tracing-config.yaml`) cover *rebuilds*. This runbook covers the *live* cluster: kube-master-31 + kube-node-32 + kube-node-33, K8s 1.32. + +Output of this plan: a single new file `docs/kubernetes-tracing.md` containing three runbooks (quick suppress, full enable, full disable) and a verification section. No code or manifest changes — pure documentation. + +## What "as quickly as possible" means + +There are two speeds of toggling: + +1. **Quick suppress** — leave the tracing flags/config in place on the components; just stop Jaeger so span exports go nowhere. No K8s component restart, takes seconds, but components still pay the (small) cost of building spans and attempting to export. +2. **Full enable / full disable** — edit configs and restart components. apiserver static-pod restart is automatic via kubelet manifest watch (~5–10s); kubelet itself needs `systemctl restart kubelet` on each node (brief NotReady blip per node, no pod evictions). This is what we'd use to truly remove overhead between experiments. + +The runbook will present both, with quick-suppress as the default for "I'm done measuring for now" and full-disable for "we don't need this on the cluster anymore". + +## Deliverable: `docs/kubernetes-tracing.md` + +### Structure + +``` +# Kubernetes component tracing + +## Components and where the toggles live +- table: component | config file on node | restart trigger + - kube-apiserver | /etc/kubernetes/manifests/kube-apiserver.yaml (flag) + + /etc/kubernetes/tracing-config.yaml (config) | kubelet auto-restarts static pod + - kubelet | /var/lib/kubelet/config.yaml (`tracing:` block) + + kube-system/kubelet-config CM (cluster-wide source of truth) | systemctl restart kubelet + +## Quick suppress (no component restart) +1. `ssh novakj@192.168.0.29 'cd /path/to/tracing && docker compose stop jaeger'` +2. Components continue running with tracing wired up but spans go nowhere. +3. To resume: `docker compose start jaeger`. +Caveat: components still build spans and attempt OTLP export — small CPU/network overhead. For a real "off", use full disable. + +## Full enable (live cluster) + +Prereqs: Jaeger is up on docker-29 (`docker compose up -d` in `docker-29/tracing/`). + +### kube-apiserver (master only) +On 192.168.0.31: +```bash +# 1. Drop the TracingConfiguration file (same content as files/tracing-config.yaml) +sudo tee /etc/kubernetes/tracing-config.yaml > /dev/null <<'EOF' +apiVersion: apiserver.config.k8s.io/v1beta1 +kind: TracingConfiguration +endpoint: 192.168.0.29:4317 +samplingRatePerMillion: 1000000 +EOF + +# 2. Back up the apiserver manifest +sudo cp /etc/kubernetes/manifests/kube-apiserver.yaml /root/kube-apiserver.yaml.pre-tracing + +# 3. Add the flag (insert after --tls-private-key-file line) +sudo sed -i '/--tls-private-key-file=/a\ - --tracing-config-file=/etc/kubernetes/tracing-config.yaml' /etc/kubernetes/manifests/kube-apiserver.yaml + +# 4. Wait for kubelet to restart the static pod (~5-10s) +sudo crictl ps | grep apiserver # new container id == restarted +kubectl get --raw /livez # should return "ok" +``` + +### kubelet (every node: master + workers) + +Per-node edit: +```bash +# Append the tracing block under kubelet config (idempotent: check first) +grep -q '^tracing:' /var/lib/kubelet/config.yaml || sudo tee -a /var/lib/kubelet/config.yaml > /dev/null <<'EOF' +tracing: + endpoint: 192.168.0.29:4317 + samplingRatePerMillion: 1000000 +EOF + +sudo systemctl restart kubelet +``` + +Stagger across nodes (one at a time) so we don't NotReady the whole cluster at once. + +Then make it survive node rebuilds/joins by updating the cluster ConfigMap: +```bash +kubectl -n kube-system edit cm kubelet-config +# add the same `tracing:` block under the `kubelet:` key +``` + +## Full disable (live cluster) + +### kube-apiserver +```bash +# Restore the pre-tracing manifest (preferred — exact revert) +sudo cp /root/kube-apiserver.yaml.pre-tracing /etc/kubernetes/manifests/kube-apiserver.yaml +# OR remove just the flag in place: +sudo sed -i '/--tracing-config-file=/d' /etc/kubernetes/manifests/kube-apiserver.yaml + +# Optional: remove the now-unused config +sudo rm /etc/kubernetes/tracing-config.yaml +``` +kubelet restarts the static pod automatically. + +### kubelet (every node) +```bash +sudo sed -i '/^tracing:/,/samplingRatePerMillion:/d' /var/lib/kubelet/config.yaml +sudo systemctl restart kubelet +``` +Then strip the `tracing:` block from `kube-system/kubelet-config` CM so node rejoins don't reintroduce it. + +## Verification (after enable) +1. Open `http://192.168.0.29:16686` — Services dropdown shows `apiserver` (appears after the first traced request) and `kubelet`. +2. Generate end-to-end traffic: + ```bash + kubectl run trace-probe --image=registry.k8s.io/pause:3.10 --restart=Never + kubectl wait --for=condition=Ready pod/trace-probe --timeout=60s + kubectl delete pod trace-probe + ``` +3. In Jaeger: filter service `apiserver`, look for spans tagged with the probe pod name; filter service `kubelet`, look for `syncPod` spans. +4. To verify *disable*: after running disable steps, repeat (2). Within a minute, no new spans for `apiserver`/`kubelet` should appear in Jaeger. + +## Notes +- 100% sampling is fine for short experiments; for steady-state leave tracing off (quick suppress is enough between sessions). +- The flag/config are GA in K8s 1.32 — no `--feature-gates` needed. +- This runbook only covers apiserver + kubelet. etcd and kube-scheduler tracing are documented as future steps in the parent plan. +``` + +## Files to change + +| File | Change | +|---|---| +| `docs/kubernetes-tracing.md` | **new** — the runbook described above | + +No other files modified. The Terraform/cloud-init side stays as already drafted in the parent plan. + +## Verification of this plan's output + +After the runbook is written: + +1. Skim `docs/kubernetes-tracing.md` end-to-end and check that someone unfamiliar with the setup can follow it without opening the parent plan. +2. Cross-check the sed/tee commands against the actual files on the live master via SSH — confirm line patterns match (e.g., the `--tls-private-key-file=` line is present so the sed insertion lands in the right spot). +3. Dry-run the quick-suppress flow first (it's the lowest-risk path) and confirm Jaeger UI stops receiving new spans within ~30 s. diff --git a/docs/plans/2026-05-21-2310-tracing-host-move-to-docker-29.md b/docs/plans/2026-05-21-2310-tracing-host-move-to-docker-29.md new file mode 100644 index 0000000..d731c7b --- /dev/null +++ b/docs/plans/2026-05-21-2310-tracing-host-move-to-docker-29.md @@ -0,0 +1,91 @@ +# Move Jaeger/OTLP host from 192.168.0.30 (docker-30) → 192.168.0.29 (docker-29) + +## Context + +The Jaeger backend that receives OTLP spans from kube-apiserver and kubelet has been relocated from the `docker-30` host (192.168.0.30) to `docker-29` (192.168.0.29). The new compose stack is already present at [vms-home/docker-29/tracing/](../../vms-home/docker-29/tracing/) and binds `0.0.0.0:4317`/`0.0.0.0:4318`, so the backend side requires no changes. + +The OTLP exporter endpoints on the Kubernetes side, plus the runbooks/plans that document them, still point at `192.168.0.30:4317`. They need to be repointed to `192.168.0.29:4317`. Without this change, apiserver and kubelet spans go nowhere once docker-30's Jaeger is shut down. + +The other docker-30 references in the repo (Vault, Gitea, Zot registry mirror, Kanidm, nginx) are unrelated to tracing and must be left alone. + +## Scope + +In-repo file edits only. Live cluster updates are documented as a follow-up checklist; no live execution is part of this plan. + +## File-level changes + +### 1. Live OTEL config (load-bearing) + +**[kubernetes-kvm-terraform/files/tracing-config.yaml:3](../../kubernetes-kvm-terraform/files/tracing-config.yaml#L3)** +- `endpoint: 192.168.0.30:4317` → `endpoint: 192.168.0.29:4317` + +**[kubernetes-kvm-terraform/master.tf](../../kubernetes-kvm-terraform/master.tf)** — two occurrences inside cloud-init heredocs: +- Line 122 (apiserver `TracingConfiguration` written to `/etc/kubernetes/tracing-config.yaml`): `192.168.0.30:4317` → `192.168.0.29:4317` +- Line 169 (`KubeletConfiguration.tracing.endpoint` in the kubeadm config): `192.168.0.30:4317` → `192.168.0.29:4317` +- **Do NOT touch line 8** (`zot_registry_ip = "192.168.0.30"`) — that's the Zot container-registry mirror, not tracing. + +### 2. Runbook docs + +**[docs/kubernetes-tracing.md](../kubernetes-tracing.md)** — replace tracing-related references throughout (≈10 lines): +- All occurrences of `192.168.0.30:4317` → `192.168.0.29:4317` (lines 54, 83, 99) +- `http://192.168.0.30:16686` → `http://192.168.0.29:16686` (line 150) +- SSH targets `novakj@192.168.0.30` → `novakj@192.168.0.29` (lines 25, 31, 43) +- Remote directory `~/docker-30/tracing` → `~/docker-29/tracing` (lines 25, 31, 43) +- Prose "Jaeger on docker-30" → "Jaeger on docker-29" (lines 3, 40) + +### 3. Plan files (historical record — also requested) + +**[plans/2026-05-21 20:15 - k8s-pod-creation-tracing.md](../../plans/2026-05-21%2020%3A15%20-%20k8s-pod-creation-tracing.md)** — ~12 references: +- `192.168.0.30:4317` (lines 38, 76, 101, 111) and `http://192.168.0.30:16686` (line 141) → `.29` +- `docker-30/tracing/` directory path (lines 13, 16, 117, 128, 155) → `docker-29/tracing/` +- Prose mentions of "docker-30" in tracing context (lines 5, 11) → "docker-29" + +**[docs/plans/2026-05-21-2035-tracing-toggle-runbook.md](2026-05-21-2035-tracing-toggle-runbook.md)** — ~6 references: +- `192.168.0.30:4317` (lines 51, 73) and `http://192.168.0.30:16686` (line 110) → `.29` +- SSH target `novakj@192.168.0.30` (line 35) → `novakj@192.168.0.29` +- Prose mentions of "docker-30" in tracing context (lines 5, 42) → "docker-29" + +### Search safety + +Use a per-file targeted replace with surrounding context (e.g. `endpoint: 192.168.0.30:4317`, `novakj@192.168.0.30`, `docker-30/tracing`), not a blanket repo-wide `sed`. The leave-alone list is large (Zot, Vault, Gitea, Kanidm, nginx, gitignore, claude settings, generated tfstate, commented-out blackbox/prometheus examples) — see Phase 1 exploration for the full inventory. + +## Verification + +### In-repo +```bash +# Confirm no tracing-related .30 references remain +grep -nE '192\.168\.0\.30:(4317|16686)' -r . +grep -nE 'docker-30/tracing' -r . +# Confirm new IP is in place +grep -nE '192\.168\.0\.29:4317' kubernetes-kvm-terraform/ +``` +Both `grep` calls in step 1 should return empty. The leave-alone Zot/Vault/Gitea/nginx hits at port 3000/8200/9443/etc. should still be present (sanity check that the surgical replace didn't over-match). + +Re-run `tofu plan` in [kubernetes-kvm-terraform/](../../kubernetes-kvm-terraform/) to confirm the diff only shows the two endpoint changes and no unrelated drift. + +### Live cluster follow-up (out of scope for this plan's execution, documented for the operator) + +The in-repo changes only affect newly-bootstrapped nodes. The existing master + workers still have the old endpoint baked into their live files. After merging the repo changes, run: + +1. **kube-apiserver** (master, 192.168.0.31): + ```bash + ssh novakj@192.168.0.31 'sudo sed -i s/192.168.0.30:4317/192.168.0.29:4317/ /etc/kubernetes/tracing-config.yaml' + # kubelet auto-restarts the static pod within ~10s + ``` +2. **kubelet** on each node (192.168.0.31, .32, .33), serially: + ```bash + ssh novakj@$NODE 'sudo sed -i s/192.168.0.30:4317/192.168.0.29:4317/ /var/lib/kubelet/config.yaml && sudo systemctl restart kubelet' + kubectl wait node --for=condition=Ready --timeout=60s + ``` +3. **Cluster ConfigMap** so future joins/reboots pick it up: + ```bash + kubectl -n kube-system edit cm kubelet-config + # update the tracing.endpoint value under kubelet: + ``` +4. **End-to-end smoke test** (from [docs/kubernetes-tracing.md](../kubernetes-tracing.md) "Verification" section): + ```bash + kubectl run trace-probe --image=registry.k8s.io/pause:3.10 --restart=Never + kubectl wait --for=condition=Ready pod/trace-probe --timeout=60s + kubectl delete pod trace-probe + ``` + Then open `http://192.168.0.29:16686` and confirm `apiserver` and `kubelet` services show recent spans for `trace-probe`. diff --git a/docs/plans/2026-07-08-0034-deploy-mimir.md b/docs/plans/2026-07-08-0034-deploy-mimir.md new file mode 100644 index 0000000..9e7748d --- /dev/null +++ b/docs/plans/2026-07-08-0034-deploy-mimir.md @@ -0,0 +1,165 @@ +# Deploy Mimir on docker-30 (mirror the Loki setup) + +## Context + +Loki is already deployed on the docker host (`192.168.0.30`) under +`vms-home/docker-30/loki/`, storing chunks in Garage S3 and queried through a +shared Grafana. The user wants Mimir deployed for **metrics** in the same style: +self-contained compose dir, Garage S3 backend, `.env`-supplied credentials, +env-expanded config. + +Decisions made: +- **Mimir only for now.** No producer is wired up in this plan; Mimir simply + exposes its remote_write endpoint (`http://192.168.0.30:9009/api/v1/push`) for + a Prometheus/Alloy to point at later. +- **Reuse the existing Grafana** (the one in the Loki compose) by adding a + Prometheus-type datasource — no second Grafana. + +Mimir runs in **monolithic mode** (`-target=all`, single binary), which is the +right shape for a single-node home lab and matches Loki's single-process model. + +## Reference: how Loki does it (the pattern to copy) + +`vms-home/docker-30/loki/loki-config.yaml` + `docker-compose.yaml`: +- Garage S3 at `192.168.0.30:3900`, `region: garage`, `s3forcepathstyle: true`, + `insecure: true`, bucket `loki-chunks`. +- Creds via `${GARAGE_ACCESS_KEY}` / `${GARAGE_SECRET_KEY}` from `.env`, + expanded with `-config.expand-env=true`. +- `obs` bridge network; Grafana published on `3001`. + +## Files to create (new dir `vms-home/docker-30/mimir/`) + +### 1. `mimir-config.yaml` — monolithic + Garage S3 + +```yaml +multitenancy_enabled: false # single-tenant; Loki mirrors this (auth_enabled:false) + +server: + http_listen_port: 9009 + grpc_listen_port: 9095 + log_level: info + +common: + storage: + backend: s3 + s3: + endpoint: 192.168.0.30:3900 # no scheme; insecure toggles http + region: garage # MUST byte-match s3_region in garage.toml + access_key_id: ${GARAGE_ACCESS_KEY} + secret_access_key: ${GARAGE_SECRET_KEY} + insecure: true # http on LAN + bucket_lookup_type: path # path-style — Garage requirement (== s3forcepathstyle) + +blocks_storage: + s3: { bucket_name: mimir-blocks } + tsdb: { dir: /data/tsdb } + bucket_store:{ sync_dir: /data/tsdb-sync } + +ruler_storage: + s3: { bucket_name: mimir-ruler } + +alertmanager_storage: + s3: { bucket_name: mimir-alertmanager } + +compactor: + data_dir: /data/compactor + +ruler: + rule_path: /data/ruler + +alertmanager: + data_dir: /data/alertmanager + +limits: + compactor_blocks_retention_period: 744h # 31d, matches Loki's retention + ingestion_rate: 50000 # samples/s per tenant; bump on 429s + ingestion_burst_size: 100000 +``` + +Note: the three storage components inherit endpoint/creds from `common.storage`; +only `bucket_name` is overridden per component. Buckets can be collapsed to one +if preferred, but three is cleaner and Garage bucket creation is cheap. + +### 2. `docker-compose.yaml` — copy Loki's, swap image/ports/paths + +```yaml +services: + mimir: + image: grafana/mimir:2.16.1 # pin latest stable — verify tag before apply + container_name: mimir + restart: unless-stopped + command: + - -config.file=/etc/mimir/config.yaml + - -config.expand-env=true # REQUIRED — expands ${GARAGE_*} + - -target=all # monolithic single-binary mode + ports: + - "9009:9009" # push (/api/v1/push) + query (/prometheus) + volumes: + - ./mimir-config.yaml:/etc/mimir/config.yaml:ro + - ./mimir-data:/data + env_file: [.env] + networks: [obs] + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://localhost:9009/ready || exit 1"] + interval: 15s + timeout: 5s + retries: 10 + start_period: 30s + logging: + driver: json-file + options: { max-size: "10m", max-file: "3" } + +networks: + obs: { driver: bridge } +``` + +### 3. `.env` — reuse the same Garage credentials as Loki + +``` +GARAGE_ACCESS_KEY= +GARAGE_SECRET_KEY= +``` + +## Manual steps on docker-30 (execution) + +1. **Create Garage buckets** and grant the existing Loki key access: + ```bash + ssh novakj@192.168.0.30 + for b in mimir-blocks mimir-ruler mimir-alertmanager; do + docker exec garage /garage bucket create "$b" + done + docker exec garage /garage key list # get the key name Loki already uses + for b in mimir-blocks mimir-ruler mimir-alertmanager; do + docker exec garage /garage bucket allow --read --write "$b" --key + done + ``` +2. Copy the `mimir/` dir to the host (or `git pull` if this repo is checked out + there), fill `.env`, then `cd mimir && docker compose up -d`. +3. **Add Mimir datasource to the existing Grafana** (`:3001`): + - Type: **Prometheus** + - URL: `http://192.168.0.30:9009/prometheus` + - No `X-Scope-OrgID` header needed (multitenancy disabled). + +## Verification + +- `curl http://192.168.0.30:9009/ready` → `ready` (also gates the healthcheck). +- `docker compose ps` shows `mimir` healthy; `docker compose logs -f mimir` + clean (no S3 auth / bucket errors — confirms Garage wiring). +- Smoke-test the write path without a producer: + ```bash + curl http://192.168.0.30:9009/api/v1/push # expect 4xx (proto body), NOT conn-refused + ``` +- In Grafana → Explore → Mimir datasource, run `up` or + `count({__name__!=""})`. Empty until a producer remote_writes — expected. +- Confirm blocks land in Garage after ~2h (first block flush): + `docker exec garage /garage bucket info mimir-blocks`. + +## Later (out of scope here): wiring a producer + +Point any Prometheus/Alloy at: +```yaml +remote_write: + - url: http://192.168.0.30:9009/api/v1/push +``` +The existing `vms/utility-101-shadow/docker/monitoring/prometheus.yml` is the +natural first candidate. diff --git a/docs/plans/2026-07-08-2346-alloy-metrics-to-mimir.md b/docs/plans/2026-07-08-2346-alloy-metrics-to-mimir.md new file mode 100644 index 0000000..4a6fcd8 --- /dev/null +++ b/docs/plans/2026-07-08-2346-alloy-metrics-to-mimir.md @@ -0,0 +1,116 @@ +# Alloy → Mimir: scrape Kubernetes metrics + Prometheus Operator monitors + +> On execution, also copy this file into the repo per CLAUDE.md convention: +> `docs/plans/2026-07-08-HHMM-alloy-metrics-to-mimir.md` (get `HHMM` via `date "+%H%M"`). + +## Context + +Mimir is now up on docker-30 (`http://192.168.0.30:9009`, single-tenant, plain +HTTP over LAN) but nothing in the cluster pushes metrics to it. Grafana Alloy +already exists in the repo but is **logs-only** (ships pod/journal logs + k8s +events to Loki) and — importantly — the entire `gitops/home-kubernetes/alloy/` +directory is **not wired into Flux** yet. + +kube-prometheus-stack is deployed and active: it provides the Prometheus +Operator CRDs (ServiceMonitor / PodMonitor), kube-state-metrics, node-exporter, +and ServiceMonitors for all standard targets (kubelet, cAdvisor, apiserver, +coredns, etc.). Its bundled Prometheus scrapes everything into a local 60d TSDB +but does **not** remote_write anywhere. + +**Goal:** add a dedicated `alloy-metrics` instance that scrapes standard +Kubernetes metrics and reads ServiceMonitors/PodMonitors from the Prometheus +Operator, remote-writing to Mimir — additive alongside the existing Prometheus +(no changes to kube-prometheus-stack). Because kube-prometheus-stack already +ships ServiceMonitors for every standard target, a single +`prometheus.operator.servicemonitors` + `prometheus.operator.podmonitors` pair +satisfies **both** requirements at once. As part of wiring the `alloy/` dir into +Flux, the existing `alloy-logs` and `alloy-events` releases also come under +GitOps management (confirmed desired). + +**Decisions confirmed:** additive/coexist with bundled Prometheus; activate all +three Alloy instances (logs, events, metrics). + +## Design + +- Metrics scraping runs as a **single Deployment** (`controller.type: deployment`, + 1 replica) — never a DaemonSet, or every cluster-wide target would be scraped + once per node. This is why it must be a separate release from `alloy-logs` + (which is a DaemonSet). +- Mirror the conventions already in `helmrelease_alloy-logs.yaml`: chart `alloy` + `>=1.8.0 <2.0.0`, image `tag: v1.17.1`, `rbac.create: false`, + `alloy.enableReporting: false`, `driftDetection: enabled`, install/upgrade + `retries: 3`, `interval: 30m`, namespace `monitoring`. +- River config = three components: + 1. `prometheus.operator.servicemonitors "sm"` — discovers/scrapes **all** + ServiceMonitors cluster-wide (covers kubelet, cAdvisor, kube-state-metrics, + node-exporter, apiserver, coredns, controller-manager, scheduler — i.e. the + "standard kubernetes metrics"). + 2. `prometheus.operator.podmonitors "pm"` — discovers/scrapes all PodMonitors. + 3. `prometheus.remote_write "mimir"` → `http://192.168.0.30:9009/api/v1/push`, + `external_labels = { cluster = "homelab" }` (matches the logs config). + Both operator components `forward_to = [prometheus.remote_write.mimir.receiver]`. +- Reuse `cluster = "homelab"` external label for consistency with the Loki side. + +## Files + +### New — `gitops/home-kubernetes/alloy/helmrelease_alloy-metrics.yaml` +HelmRelease `alloy-metrics` (ns `monitoring`), Deployment, serviceAccount +`alloy-metrics`, inline `alloy.configMap.content` with the three components +above. Modest resources (e.g. requests cpu 100m / mem 256Mi, limit mem 768Mi — +operator discovery + scrape buffers use more than the log shipper). + +### New — `gitops/home-kubernetes/alloy/rbac_metrics-collector.yaml` +The `prometheus.operator.*` components need read access beyond the existing +`alloy-log-reader` role. ClusterRole `alloy-metrics-reader` + ClusterRoleBinding +to SA `alloy-metrics` (monitoring), granting: +- core: `namespaces, nodes, nodes/metrics, nodes/proxy, services, endpoints, pods` — get/list/watch +- `discovery.k8s.io`: `endpointslices` — get/list/watch +- `networking.k8s.io`: `ingresses` — get/list/watch +- `monitoring.coreos.com`: `servicemonitors, podmonitors, probes, scrapeconfigs` — get/list/watch +- nonResourceURLs `/metrics`, `/metrics/cadvisor` — get (kubelet scraping) + +(Kept as a separate file so the metrics RBAC is self-contained; the existing +`rbac_log-collector.yaml` stays untouched.) + +### New — `gitops/home-kubernetes/alloy/kustomization.yaml` +`kind: Kustomization` listing all manifests in the dir: `helmrepository_grafana.yaml`, +`helmrelease_alloy-logs.yaml`, `helmrelease_allow-events.yaml`, +`helmrelease_alloy-metrics.yaml`, `rbac_log-collector.yaml`, +`rbac_metrics-collector.yaml`. Makes the Flux build explicit/deterministic. + +### Modified — `gitops/home-kubernetes/flux-system/extra-kustomizations.yaml` +Append an `alloy` Flux Kustomization: +```yaml +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: alloy + namespace: flux-system +spec: + interval: 10m0s + path: ./gitops/home-kubernetes/alloy + prune: true + sourceRef: + kind: GitRepository + name: flux-system + dependsOn: + - name: kube-prometheus # needs the monitoring ns + operator CRDs/ServiceMonitors +``` + +## Verification + +1. Commit + push; force reconcile: `flux reconcile kustomization flux-system --with-source`. +2. `kubectl get kustomizations -A` — `alloy` Ready=True. +3. `kubectl get helmreleases -n monitoring` — `alloy-metrics` (+ logs/events) Ready. +4. `kubectl -n monitoring get pods -l app.kubernetes.io/instance=alloy-metrics` + Running; `kubectl -n monitoring logs deploy/alloy-metrics` shows no RBAC + `forbidden` errors and targets discovered. +5. Alloy UI: `kubectl -n monitoring port-forward deploy/alloy-metrics 12345:12345` + → open `http://localhost:12345`; confirm the two operator components list + healthy targets and `prometheus.remote_write.mimir` shows successful sends. +6. Confirm Mimir is receiving: query it directly, e.g. + `curl -s 'http://192.168.0.30:9009/prometheus/api/v1/query?query=up' | jq '.data.result | length'` + returns > 0 (or check the Grafana Mimir datasource for `up`, + `kube_pod_info`, `node_cpu_seconds_total`, `container_cpu_usage_seconds_total`). +7. Sanity: a `cluster="homelab"` label is present on the ingested series. diff --git a/gitops/home-kubernetes/alloy/helmrelease_allow-events.yaml b/gitops/home-kubernetes/alloy/helmrelease_allow-events.yaml new file mode 100644 index 0000000..efb3e38 --- /dev/null +++ b/gitops/home-kubernetes/alloy/helmrelease_allow-events.yaml @@ -0,0 +1,42 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: alloy-events + namespace: monitoring +spec: + interval: 30m + chart: + spec: + chart: alloy + version: ">=1.8.0 <2.0.0" + sourceRef: { kind: HelmRepository, name: grafana, namespace: monitoring } + interval: 12h + install: { remediation: { retries: 3 } } + upgrade: { remediation: { retries: 3 } } + driftDetection: { mode: enabled } + values: + controller: + type: deployment + replicas: 1 # cluster-scoped → exactly one + image: + tag: v1.17.1 + rbac: { create: false } + serviceAccount: { create: true, name: alloy-events } + alloy: + enableReporting: false + resources: + requests: { cpu: 50m, memory: 96Mi } + limits: { memory: 256Mi } + configMap: + content: |- + loki.source.kubernetes_events "events" { + forward_to = [loki.process.events.receiver] + } + loki.process "events" { + stage.static_labels { values = { source = "kubernetes-events" } } + forward_to = [loki.write.loki.receiver] + } + loki.write "loki" { + endpoint { url = "http://192.168.0.30:3100/loki/api/v1/push" } + external_labels = { cluster = "homelab" } + } \ No newline at end of file diff --git a/gitops/home-kubernetes/alloy/helmrelease_alloy-logs.yaml b/gitops/home-kubernetes/alloy/helmrelease_alloy-logs.yaml new file mode 100644 index 0000000..44bb89d --- /dev/null +++ b/gitops/home-kubernetes/alloy/helmrelease_alloy-logs.yaml @@ -0,0 +1,106 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: alloy-logs + namespace: monitoring +spec: + interval: 30m + chart: + spec: + chart: alloy + version: ">=1.8.0 <2.0.0" # newest 1.x each reconcile; pin exact if you want determinism + sourceRef: { kind: HelmRepository, name: grafana, namespace: monitoring } + interval: 12h # poll cadence for new chart versions + install: { remediation: { retries: 3 } } + upgrade: { remediation: { retries: 3 } } + driftDetection: { mode: enabled } # revert manual kubectl edits + values: + controller: + type: daemonset + image: + tag: v1.17.1 + rbac: { create: false } + serviceAccount: { create: true, name: alloy-logs } + alloy: + enableReporting: false + securityContext: { runAsUser: 0, runAsGroup: 0 } # journald read + mounts: + varlog: true + resources: + requests: { cpu: 100m, memory: 128Mi } + limits: { memory: 512Mi } + extraEnv: + - name: NODE_NAME + valueFrom: { fieldRef: { fieldPath: spec.nodeName } } + configMap: + content: |- + discovery.kubernetes "pods" { + role = "pod" + selectors { + role = "pod" + field = "spec.nodeName=" + sys.env("NODE_NAME") + } + } + discovery.relabel "pods" { + targets = discovery.kubernetes.pods.targets + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_label_app"] + target_label = "app" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } + rule { + source_labels = ["__meta_kubernetes_pod_node_name"] + target_label = "node" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + } + loki.source.kubernetes "pods" { + targets = discovery.relabel.pods.output + forward_to = [loki.process.pods.receiver] + } + loki.process "pods" { + stage.static_labels { + values = { source = "kubernetes" } + } + forward_to = [loki.write.loki.receiver] + } + + discovery.relabel "journal" { + targets = [] + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + rule { + source_labels = ["__journal__hostname"] + target_label = "node" + } + rule { + source_labels = ["__journal_priority_keyword"] + target_label = "level" + } + } + loki.source.journal "node" { + path = "/var/log/journal" + max_age = "12h" + labels = { source = "node-journal" } + relabel_rules = discovery.relabel.journal.rules + forward_to = [loki.write.loki.receiver] + } + + loki.write "loki" { + endpoint { + url = "http://192.168.0.30:3100/loki/api/v1/push" + } + external_labels = { cluster = "homelab" } + } \ No newline at end of file diff --git a/gitops/home-kubernetes/alloy/helmrelease_alloy-metrics.yaml b/gitops/home-kubernetes/alloy/helmrelease_alloy-metrics.yaml new file mode 100644 index 0000000..69c084a --- /dev/null +++ b/gitops/home-kubernetes/alloy/helmrelease_alloy-metrics.yaml @@ -0,0 +1,49 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: alloy-metrics + namespace: monitoring +spec: + interval: 30m + chart: + spec: + chart: alloy + version: ">=1.8.0 <2.0.0" + sourceRef: { kind: HelmRepository, name: grafana, namespace: monitoring } + interval: 12h + install: { remediation: { retries: 3 } } + upgrade: { remediation: { retries: 3 } } + driftDetection: { mode: enabled } + values: + controller: + type: deployment + replicas: 1 # cluster-scoped scrape target discovery → exactly one + image: + tag: v1.17.1 + rbac: { create: false } + serviceAccount: { create: true, name: alloy-metrics } + alloy: + enableReporting: false + resources: + requests: { cpu: 100m, memory: 256Mi } + limits: { memory: 768Mi } + configMap: + content: |- + // Discovers every ServiceMonitor cluster-wide. kube-prometheus-stack already ships + // ServiceMonitors for kubelet/cAdvisor, kube-state-metrics, node-exporter, apiserver, + // coredns, controller-manager and scheduler — so this alone covers "standard k8s metrics". + prometheus.operator.servicemonitors "sm" { + forward_to = [prometheus.remote_write.mimir.receiver] + } + + // Discovers every PodMonitor cluster-wide, same selector scope as above. + prometheus.operator.podmonitors "pm" { + forward_to = [prometheus.remote_write.mimir.receiver] + } + + prometheus.remote_write "mimir" { + endpoint { + url = "http://192.168.0.30:9009/api/v1/push" + } + external_labels = { cluster = "homelab" } + } diff --git a/gitops/home-kubernetes/alloy/helmrepository_grafana.yaml b/gitops/home-kubernetes/alloy/helmrepository_grafana.yaml new file mode 100644 index 0000000..3bca3ee --- /dev/null +++ b/gitops/home-kubernetes/alloy/helmrepository_grafana.yaml @@ -0,0 +1,8 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: grafana + namespace: monitoring +spec: + interval: 1h + url: https://grafana.github.io/helm-charts \ No newline at end of file diff --git a/gitops/home-kubernetes/alloy/rbac_log-collector.yaml b/gitops/home-kubernetes/alloy/rbac_log-collector.yaml new file mode 100644 index 0000000..af59fa4 --- /dev/null +++ b/gitops/home-kubernetes/alloy/rbac_log-collector.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: { name: alloy-log-reader } +rules: + - apiGroups: [""] + resources: ["pods", "pods/log", "namespaces", "nodes", "nodes/proxy", "events"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: { name: alloy-log-reader } +roleRef: { apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: alloy-log-reader } +subjects: + - { kind: ServiceAccount, name: alloy-logs, namespace: monitoring } + - { kind: ServiceAccount, name: alloy-events, namespace: monitoring } \ No newline at end of file diff --git a/gitops/home-kubernetes/alloy/rbac_metrics-collector.yaml b/gitops/home-kubernetes/alloy/rbac_metrics-collector.yaml new file mode 100644 index 0000000..428af44 --- /dev/null +++ b/gitops/home-kubernetes/alloy/rbac_metrics-collector.yaml @@ -0,0 +1,25 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: { name: alloy-metrics-reader } +rules: + - apiGroups: [""] + resources: ["namespaces", "nodes", "nodes/metrics", "nodes/proxy", "services", "endpoints", "pods"] + verbs: ["get", "list", "watch"] + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list", "watch"] + - apiGroups: ["networking.k8s.io"] + resources: ["ingresses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["monitoring.coreos.com"] + resources: ["servicemonitors", "podmonitors", "probes", "scrapeconfigs"] + verbs: ["get", "list", "watch"] + - nonResourceURLs: ["/metrics", "/metrics/cadvisor"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: { name: alloy-metrics-reader } +roleRef: { apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: alloy-metrics-reader } +subjects: + - { kind: ServiceAccount, name: alloy-metrics, namespace: monitoring } diff --git a/gitops/home-kubernetes/alloy/readme.md b/gitops/home-kubernetes/alloy/readme.md new file mode 100644 index 0000000..e642d4d --- /dev/null +++ b/gitops/home-kubernetes/alloy/readme.md @@ -0,0 +1,10 @@ +## debugging alloy + +```bash +POD=$(kubectl -n monitoring get pod -l app.kubernetes.io/name=alloy -o name | head -1) +kubectl -n monitoring debug -it $POD --image=nicolaka/netshoot --target=alloy -- bash + + +curl -v http://192.168.0.30:3100/ready + +``` \ No newline at end of file diff --git a/gitops/home-kubernetes/flux-system/extra-kustomizations.yaml b/gitops/home-kubernetes/flux-system/extra-kustomizations.yaml index db47b31..d3f4933 100644 --- a/gitops/home-kubernetes/flux-system/extra-kustomizations.yaml +++ b/gitops/home-kubernetes/flux-system/extra-kustomizations.yaml @@ -81,6 +81,21 @@ spec: --- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization +metadata: + name: alloy + namespace: flux-system +spec: + interval: 10m0s + path: ./gitops/home-kubernetes/alloy + prune: true + sourceRef: + kind: GitRepository + name: flux-system + dependsOn: + - name: kube-prometheus # needs the monitoring ns + operator CRDs/ServiceMonitors +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization metadata: name: ingress-nginx namespace: flux-system diff --git a/kubernetes-kvm-terraform/files/manifests/kube-apiserver.yaml b/kubernetes-kvm-terraform/files/manifests/kube-apiserver.yaml index 446cb03..2c0f6f7 100644 --- a/kubernetes-kvm-terraform/files/manifests/kube-apiserver.yaml +++ b/kubernetes-kvm-terraform/files/manifests/kube-apiserver.yaml @@ -44,6 +44,7 @@ spec: - --service-cluster-ip-range=10.96.0.0/12 - --tls-cert-file=/etc/kubernetes/pki/apiserver.crt - --tls-private-key-file=/etc/kubernetes/pki/apiserver.key + - --tracing-config-file=/etc/kubernetes/tracing-config.yaml image: registry.k8s.io/kube-apiserver:v1.32.11 imagePullPolicy: IfNotPresent livenessProbe: diff --git a/kubernetes-kvm-terraform/files/tracing-config.yaml b/kubernetes-kvm-terraform/files/tracing-config.yaml new file mode 100644 index 0000000..92e5d1d --- /dev/null +++ b/kubernetes-kvm-terraform/files/tracing-config.yaml @@ -0,0 +1,4 @@ +apiVersion: apiserver.config.k8s.io/v1beta1 +kind: TracingConfiguration +endpoint: 192.168.0.29:4317 +samplingRatePerMillion: 1000000 diff --git a/kubernetes-kvm-terraform/master.tf b/kubernetes-kvm-terraform/master.tf index 5bf6174..6dc23f0 100644 --- a/kubernetes-kvm-terraform/master.tf +++ b/kubernetes-kvm-terraform/master.tf @@ -115,6 +115,13 @@ locals { skip_verify = true override_path = true + - path: /etc/kubernetes/tracing-config.yaml + content: | + apiVersion: apiserver.config.k8s.io/v1beta1 + kind: TracingConfiguration + endpoint: 192.168.0.29:4317 + samplingRatePerMillion: 1000000 + - path: /etc/kubernetes/auth-config.yaml content: | apiVersion: apiserver.config.k8s.io/v1beta1 @@ -158,6 +165,9 @@ locals { apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration cgroupDriver: systemd + tracing: + endpoint: 192.168.0.29:4317 + samplingRatePerMillion: 1000000 - path: /etc/profile.d/kubectl.sh content: | diff --git a/plans/2026-05-21 20:15 - k8s-pod-creation-tracing.md b/plans/2026-05-21 20:15 - k8s-pod-creation-tracing.md new file mode 100644 index 0000000..b53064c --- /dev/null +++ b/plans/2026-05-21 20:15 - k8s-pod-creation-tracing.md @@ -0,0 +1,157 @@ +# Pod-creation latency tracing with OpenTelemetry + +## Context + +Goal: measure end-to-end latency of `kubectl apply` → pod running, broken down by component. The approach is to enable native OpenTelemetry tracing in Kubernetes 1.32 components (kube-apiserver, kubelet, etcd) and ship spans to a Jaeger backend running outside the cluster on `docker-29`. With all three reporting under the same trace id (object-resource correlation), Jaeger will let us see how much time is spent in admission/etcd, on the wire, and inside the kubelet's pod sync loop. + +Constraints / known gaps: +- **kube-scheduler has no native OTLP tracing yet**; there will be a gap between "object stored" and "kubelet sees assignment". We can infer it from event timestamps (`kubectl get events`) or `.status.conditions` PodScheduled/Initialized/ContainersReady/Ready timestamps. +- Cluster is K8s 1.32, kubeadm-bootstrapped via Terraform/cloud-init. Per Phase-3 decision, the canonical source-of-truth is the Terraform cloud-init; changes there cover future rebuilds. To activate tracing on the *currently running* nodes we also need a one-time manual application step (documented at the bottom). + +## Backend: Jaeger on docker-29 + +New directory `docker-29/tracing/` containing `docker-compose.yaml`. Uses Jaeger all-in-one v1 with Badger on-disk storage and the built-in OTLP receiver enabled. + +```yaml +# docker-29/tracing/docker-compose.yaml +services: + jaeger: + image: jaegertracing/all-in-one:1.62 + container_name: jaeger + restart: unless-stopped + environment: + SPAN_STORAGE_TYPE: badger + BADGER_EPHEMERAL: "false" + BADGER_DIRECTORY_VALUE: /badger/data + BADGER_DIRECTORY_KEY: /badger/key + BADGER_SPAN_STORE_TTL: 4h # auto-expire traces older than 4h + BADGER_MAINTENANCE_INTERVAL: 5m # GC cycle that reclaims disk + COLLECTOR_OTLP_ENABLED: "true" + volumes: + - ./badger:/badger + ports: + - "4317:4317" # OTLP gRPC (apiserver + kubelet exporters) + - "4318:4318" # OTLP HTTP (optional) + - "16686:16686" # Jaeger UI +``` + +Endpoint that the cluster will send to: `192.168.0.29:4317`. Jaeger UI: `http://192.168.0.29:16686`. + +The OTLP receiver in Jaeger all-in-one accepts **plaintext** OTLP/gRPC by default — matches `endpoint: ...:4317` on the K8s side with no TLS. Acceptable for a home lab on a private LAN. + +### Storage sizing and retention + +Two complementary levers control how much disk Badger uses: + +1. **TTL** — `BADGER_SPAN_STORE_TTL=4h` deletes spans older than 4 hours; Badger's value-log GC reclaims disk on `BADGER_MAINTENANCE_INTERVAL` (default 5m). This is the hard upper bound on retention. +2. **Sampling rate** — controls *how many* spans get written in the first place. Set in the apiserver/kubelet TracingConfiguration (`samplingRatePerMillion`). + +Rough estimate for this cluster (1 master + 2 workers, mostly idle, Flux reconciling continuously): + +| Sampling | Span ingest rate | Disk footprint at 4h TTL | +|---|---|---| +| 100% (`1000000`) | ~500–3000 spans/s | ~6–40 GB steady-state | +| 1% (`10000`) | ~5–30 spans/s | ~60–400 MB steady-state | +| 0.1% (`1000`) | ~0.5–3 spans/s | ~6–40 MB steady-state | + +Note: 100% sampling on this cluster is sized for short experiments, not long-term operation. The 4h TTL prevents runaway growth if I forget to lower it, but Badger has no built-in *size* cap — only TTL + GC. If a hard size cap is needed, mount `./badger/` from a fixed-size LVM volume or zfs dataset with a quota. + +**Recommended progression**: +- Start with `samplingRatePerMillion: 1000000` (100%) and `TTL=4h` so the first measurements are easy to find. +- After the first round of experiments, drop sampling to `10000` (1%) for steady-state and adjust `TTL` upward if a longer history is useful. + +## kube-apiserver tracing + +Two changes, both in the Terraform cloud-init / referenced manifest: + +### 1. Write the TracingConfiguration file (master.tf cloud-init) + +Add to `write_files` in [kubernetes-kvm-terraform/master.tf](kubernetes-kvm-terraform/master.tf): + +```yaml +- path: /etc/kubernetes/tracing-config.yaml + content: | + apiVersion: apiserver.config.k8s.io/v1beta1 + kind: TracingConfiguration + endpoint: 192.168.0.29:4317 + samplingRatePerMillion: 1000000 # 100% sampling for measurement work +``` + +The apiserver static pod already mounts `/etc/kubernetes` read-only via the `k8s-config` volume, so the file becomes visible inside the container automatically — no new volume mount needed. + +### 2. Add the apiserver flag + +Edit [kubernetes-kvm-terraform/files/manifests/kube-apiserver.yaml](kubernetes-kvm-terraform/files/manifests/kube-apiserver.yaml) — add one new flag in the `command:` list (alphabetical placement after `--tls-private-key-file` works; ordering isn't enforced): + +```yaml +- --tracing-config-file=/etc/kubernetes/tracing-config.yaml +``` + +`APIServerTracing` is GA in 1.32 — no feature-gate flag required. + +## kubelet tracing + +Add a `tracing:` block to the existing `KubeletConfiguration` document inside the kubeadm-config heredoc in [kubernetes-kvm-terraform/master.tf](kubernetes-kvm-terraform/master.tf) (around line 158): + +```yaml +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +cgroupDriver: systemd +tracing: + endpoint: 192.168.0.29:4317 + samplingRatePerMillion: 1000000 +``` + +`KubeletTracing` is beta and on by default in 1.32 — no feature gate needed. + +Kubeadm stores this `KubeletConfiguration` in the `kubelet-config` ConfigMap in `kube-system`, so workers joining via `kubeadm join` pick the tracing block up automatically. The two existing workers (kube-node-32, kube-node-33) need a one-time manual nudge (see below) because they were already joined before the change. + +## etcd tracing (optional, second-step) + +etcd supports OTLP via flags `--experimental-enable-distributed-tracing`, `--experimental-distributed-tracing-address=192.168.0.29:4317`, and `--experimental-distributed-tracing-sampling-rate=1000000`. Enabling these requires editing the etcd static-pod manifest on the master — propose deferring until after the apiserver+kubelet path is verified, so we keep the first iteration narrow. + +## Files to change + +| File | Change | +|---|---| +| `docker-29/tracing/docker-compose.yaml` | **new** — Jaeger all-in-one + Badger | +| `kubernetes-kvm-terraform/master.tf` | add `tracing-config.yaml` to `write_files`; add `tracing:` block to `KubeletConfiguration` | +| `kubernetes-kvm-terraform/files/manifests/kube-apiserver.yaml` | add `--tracing-config-file=/etc/kubernetes/tracing-config.yaml` | +| `kubernetes-kvm-terraform/files/tracing-config.yaml` | **new** — reference copy of the TracingConfiguration (mirrors the inline content in master.tf, matches the existing `auth-config.yaml` convention) | + +No changes to `nodes-on-homer.tf` / `nodes-on-beelink.tf` are needed — workers receive kubelet config from the cluster ConfigMap on join. + +## Applying to the currently running cluster + +The Terraform changes cover future rebuilds. To activate tracing **now** without rebuilding: + +1. **Start Jaeger** on docker-29: `cd docker-29/tracing && docker compose up -d` (creates `./badger/` for storage). +2. **On the master (kube-master-31, 192.168.0.31)**: + - `scp` or write `/etc/kubernetes/tracing-config.yaml` with the TracingConfiguration shown above. + - Edit `/etc/kubernetes/manifests/kube-apiserver.yaml` to add `--tracing-config-file=/etc/kubernetes/tracing-config.yaml`. Kubelet will detect the manifest change and restart the apiserver static pod within seconds. + - Watch: `crictl ps | grep apiserver`, then `kubectl get --raw /livez`. +3. **Update the kubelet-config ConfigMap** in `kube-system`: + - `kubectl -n kube-system edit cm kubelet-config` and add the same `tracing:` block under `kubelet:`. +4. **On every node** (master + both workers): edit `/var/lib/kubelet/config.yaml` to append the `tracing:` block, then `systemctl restart kubelet`. Pods will *not* be evicted by a kubelet restart, but expect a brief NotReady blip per node — stagger the restarts. + +## Verification + +End-to-end test once everything is live: + +1. UI reachable: open `http://192.168.0.29:16686` and check "Services" dropdown populates with `apiserver` (it will appear only after the first traced request). +2. Generate traffic that exercises the full path: + ``` + kubectl run trace-probe --image=registry.k8s.io/pause:3.10 --restart=Never + kubectl wait --for=condition=Ready pod/trace-probe --timeout=60s + kubectl delete pod trace-probe + ``` +3. In Jaeger UI, search service `apiserver`, operation `KubernetesAPI` (or similar) — you should see a trace covering admission/etcd write. Search service `kubelet`, operation `syncPod` — kubelet-side pod sync spans should be visible. +4. Cross-check the gap (scheduler) by comparing pod `metadata.creationTimestamp`, the `PodScheduled` condition `lastTransitionTime`, and the kubelet span start. The diff between those values quantifies the scheduler portion that we cannot trace natively. +5. Confirm Badger persistence: `docker compose restart jaeger`, reload UI — earlier traces should still be present. + +## Notes / risks + +- 100% sampling rate is fine for measurement; once we're done we should drop to e.g. `samplingRatePerMillion: 10000` (1%) so storage doesn't explode. +- Badger volume `./badger/` on docker-29 will grow with span volume; size budget worth watching but not urgent. +- `tracing.endpoint` is plaintext OTLP/gRPC and traverses the LAN — fine for a home lab but explicitly *not* encrypted. +- If apiserver fails to come back after the manifest edit, the most common cause is a typo in the new flag; revert is straightforward — keep a copy of the original `/etc/kubernetes/manifests/kube-apiserver.yaml` in `/root/` before editing. diff --git a/vms-home/docker-29/readme.md b/vms-home/docker-29/readme.md new file mode 100644 index 0000000..e69de29 diff --git a/vms-home/docker-29/tracing/docker-compose.yaml b/vms-home/docker-29/tracing/docker-compose.yaml new file mode 100644 index 0000000..376b14d --- /dev/null +++ b/vms-home/docker-29/tracing/docker-compose.yaml @@ -0,0 +1,14 @@ +services: + jaeger: + image: jaegertracing/jaeger:2.18.0 + container_name: jaeger + restart: unless-stopped + user: root + command: ["--config", "/jaeger/config.yaml"] + volumes: + - ./jaeger-config.yaml:/jaeger/config.yaml:ro + - ./badger:/badger + ports: + - "4317:4317" # OTLP gRPC + - "4318:4318" # OTLP HTTP + - "16686:16686" # Jaeger UI diff --git a/vms-home/docker-29/tracing/jaeger-config.yaml b/vms-home/docker-29/tracing/jaeger-config.yaml new file mode 100644 index 0000000..2329a36 --- /dev/null +++ b/vms-home/docker-29/tracing/jaeger-config.yaml @@ -0,0 +1,35 @@ +service: + extensions: [jaeger_storage, jaeger_query] + pipelines: + traces: + receivers: [otlp] + processors: [] + exporters: [jaeger_storage_exporter] + +extensions: + jaeger_storage: + backends: + badger_store: + badger: + directories: + keys: /badger/key + values: /badger/data + ephemeral: false + ttl: + spans: 4h + metrics_update_interval: 5m + jaeger_query: + storage: + traces: badger_store + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +exporters: + jaeger_storage_exporter: + trace_storage: badger_store diff --git a/vms-home/docker-30/loki/.env b/vms-home/docker-30/loki/.env new file mode 100644 index 0000000..b815428 --- /dev/null +++ b/vms-home/docker-30/loki/.env @@ -0,0 +1,2 @@ +GARAGE_ACCESS_KEY=GK... +GARAGE_SECRET_KEY=... \ No newline at end of file diff --git a/vms-home/docker-30/loki/docker-compose.yaml b/vms-home/docker-30/loki/docker-compose.yaml new file mode 100644 index 0000000..7ced72e --- /dev/null +++ b/vms-home/docker-30/loki/docker-compose.yaml @@ -0,0 +1,41 @@ +services: + loki: + image: grafana/loki:3.7.3 + container_name: loki + restart: unless-stopped + user: "10001:10001" + command: + - -config.file=/etc/loki/config.yaml + - -config.expand-env=true # REQUIRED — expands ${GARAGE_*} + ports: + - "3100:3100" # LAN IP, not 0.0.0.0 — push+query + volumes: + - ./loki-config.yaml:/etc/loki/config.yaml:ro + - ./loki-data:/loki # WAL / tsdb-index / compactor scratch + env_file: [.env] # GARAGE_ACCESS_KEY / GARAGE_SECRET_KEY + networks: [obs] + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://localhost:3100/ready || exit 1"] + interval: 15s + timeout: 5s + retries: 10 + start_period: 30s + logging: # don't let Loki's own logs balloon + driver: json-file + options: { max-size: "10m", max-file: "3" } + + grafana: # optional — drop if you query elsewhere + image: grafana/grafana:13.1.0 + container_name: grafana + restart: unless-stopped + ports: + - "3001:3000" # LAN IP, not # 3000 is taken by gitea + volumes: + - grafana-data:/var/lib/grafana + networks: [obs] + +volumes: + grafana-data: + +networks: + obs: { driver: bridge } \ No newline at end of file diff --git a/vms-home/docker-30/loki/loki-config.yaml b/vms-home/docker-30/loki/loki-config.yaml new file mode 100644 index 0000000..e389098 --- /dev/null +++ b/vms-home/docker-30/loki/loki-config.yaml @@ -0,0 +1,47 @@ +auth_enabled: false # single-tenant; sources split by label + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: info + +common: + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: { store: inmemory } + storage: + s3: + endpoint: 192.168.0.30:3900 # no scheme; insecure toggles http + region: garage # MUST byte-match s3_region in garage.toml + bucketnames: loki-chunks + access_key_id: ${GARAGE_ACCESS_KEY} + secret_access_key: ${GARAGE_SECRET_KEY} + s3forcepathstyle: true # mandatory for Garage + insecure: true # http on LAN; drop if you TLS-front Garage + +schema_config: + configs: + - from: 2026-07-01 # <= first-ingest date; never mutate historical + store: tsdb + object_store: s3 + schema: v13 + index: { prefix: index_, period: 24h } + +storage_config: + tsdb_shipper: + active_index_directory: /loki/tsdb-index + cache_location: /loki/tsdb-cache + +limits_config: + allow_structured_metadata: true + volume_enabled: true + retention_period: 744h # 31d; compactor enforces + reject_old_samples: true + reject_old_samples_max_age: 168h + max_line_size: 256KB + ingestion_rate_mb: 8 # per-tenant; bump if you hit 429s + ingestion_burst_size_mb: 16 + +compactor: + working_directory: /loki/compactor \ No newline at end of file diff --git a/vms-home/docker-30/mimir/.env b/vms-home/docker-30/mimir/.env new file mode 100644 index 0000000..32c7131 --- /dev/null +++ b/vms-home/docker-30/mimir/.env @@ -0,0 +1,2 @@ +GARAGE_ACCESS_KEY=GK... +GARAGE_SECRET_KEY=... diff --git a/vms-home/docker-30/mimir/docker-compose.yaml b/vms-home/docker-30/mimir/docker-compose.yaml new file mode 100644 index 0000000..5aa1428 --- /dev/null +++ b/vms-home/docker-30/mimir/docker-compose.yaml @@ -0,0 +1,28 @@ +services: + mimir: + image: grafana/mimir:3.1.2 + container_name: mimir + restart: unless-stopped + command: + - -config.file=/etc/mimir/config.yaml + - -config.expand-env=true # REQUIRED — expands ${GARAGE_*} + - -target=all # monolithic single-binary mode + ports: + - "9009:9009" # LAN IP, not 0.0.0.0 — push+query + volumes: + - ./mimir-config.yaml:/etc/mimir/config.yaml:ro + - ./mimir-data:/data # WAL / tsdb / compactor scratch + env_file: [.env] # GARAGE_ACCESS_KEY / GARAGE_SECRET_KEY + networks: [obs] + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://localhost:9009/ready || exit 1"] + interval: 15s + timeout: 5s + retries: 10 + start_period: 30s + logging: # don't let Mimir's own logs balloon + driver: json-file + options: { max-size: "10m", max-file: "3" } + +networks: + obs: { driver: bridge } diff --git a/vms-home/docker-30/mimir/mimir-config.yaml b/vms-home/docker-30/mimir/mimir-config.yaml new file mode 100644 index 0000000..0ba4740 --- /dev/null +++ b/vms-home/docker-30/mimir/mimir-config.yaml @@ -0,0 +1,42 @@ +multitenancy_enabled: false # single-tenant; mirrors Loki's auth_enabled:false + +server: + http_listen_port: 9009 + grpc_listen_port: 9095 + log_level: info + +common: + storage: + backend: s3 + s3: + endpoint: 192.168.0.30:3900 # no scheme; insecure toggles http + region: garage # MUST byte-match s3_region in garage.toml + access_key_id: ${GARAGE_ACCESS_KEY} + secret_access_key: ${GARAGE_SECRET_KEY} + insecure: true # http on LAN; drop if you TLS-front Garage + bucket_lookup_type: path # path-style — Garage requirement (== s3forcepathstyle) + +blocks_storage: + s3: { bucket_name: mimir-blocks } + tsdb: { dir: /data/tsdb } + bucket_store: { sync_dir: /data/tsdb-sync } + +ruler_storage: + s3: { bucket_name: mimir-ruler } + +alertmanager_storage: + s3: { bucket_name: mimir-alertmanager } + +compactor: + data_dir: /data/compactor + +ruler: + rule_path: /data/ruler + +alertmanager: + data_dir: /data/alertmanager + +limits: + compactor_blocks_retention_period: 744h # 31d; matches Loki's retention_period + ingestion_rate: 50000 # samples/s per tenant; bump if you hit 429s + ingestion_burst_size: 100000 diff --git a/vms-home/docker-30/mimir/readme.md b/vms-home/docker-30/mimir/readme.md new file mode 100644 index 0000000..80d41ae --- /dev/null +++ b/vms-home/docker-30/mimir/readme.md @@ -0,0 +1,98 @@ +# Mimir + +Grafana Mimir, monolithic mode (`-target=all`), single-tenant, backed by Garage S3 +(same bucket-per-Garage-instance pattern as `../loki/`). Metrics ingest via +Prometheus remote_write — Mimir does not scrape anything itself. + +- HTTP (query + push): `9009` +- gRPC (internal): `9095` + +## 1. Garage setup (do this first — before `docker compose up`) + +Mimir needs three buckets. Create them and grant the existing Loki Garage key +read+write access (reuse the key rather than minting a new one, unless you +want tighter separation): + +```bash +ssh novakj@192.168.0.30 + +for b in mimir-blocks mimir-ruler mimir-alertmanager; do + docker exec garage /garage bucket create "$b" +done + +docker exec garage /garage key list # find the key name Loki already uses + +for b in mimir-blocks mimir-ruler mimir-alertmanager; do + docker exec garage /garage bucket allow --read --write "$b" --key +done +``` + +## 2. `.env` + +Reuse the same Garage credentials Loki uses (see `../loki/.env`): + +``` +GARAGE_ACCESS_KEY=GK... +GARAGE_SECRET_KEY=... +``` + +## 3. Start + +```bash +cd /path/to/mimir +docker compose up -d +``` + +## 4. Add Mimir as a Grafana datasource + +Reuses the Grafana already running from `../loki/docker-compose.yaml` (`:3001`) +— no second Grafana instance. + +- Type: **Prometheus** +- URL: `http://192.168.0.30:9009/prometheus` +- No `X-Scope-OrgID` header needed (`multitenancy_enabled: false`) + +## 5. Verify + +```bash +curl http://192.168.0.30:9009/ready # -> ready +docker compose ps # mimir healthy +docker compose logs -f mimir # no S3 auth/bucket errors + +# push endpoint reachable (expect a 4xx proto-decode error, not connection refused) +curl http://192.168.0.30:9009/api/v1/push +``` + +In Grafana → Explore → Mimir datasource, run `up` or `count({__name__!=""})`. +Empty result is expected until a producer remote_writes into Mimir. + +After ~2h (first block flush interval), confirm blocks landed in Garage: + +```bash +docker exec garage /garage bucket info mimir-blocks +``` + +## 6. Wiring a producer (later, not done here) + +Point any Prometheus/Alloy remote_write at Mimir: + +```yaml +remote_write: + - url: http://192.168.0.30:9009/api/v1/push +``` + +`vms/utility-101-shadow/docker/monitoring/prometheus.yml` is the natural first +candidate to wire up. + +## Notes / gotchas + +- `bucket_lookup_type: path` in `mimir-config.yaml` is mandatory for Garage + (equivalent to Loki's `s3forcepathstyle: true`). +- `-config.expand-env=true` is required on the command line, or `${GARAGE_*}` + in `mimir-config.yaml` won't expand. +- First metrics won't appear in Garage until the first block flush (~2h) — + don't panic if `mimir-blocks` looks empty right after startup. +- Retention is 31d, set via `limits.compactor_blocks_retention_period` (matches + Loki's `retention_period`). +- `region: garage` in the config must byte-match `s3_region` in Garage's + `garage.toml` — same requirement as Loki. diff --git a/vms/utility-101-shadow/docker/monitoring/prometheus.yml b/vms/utility-101-shadow/docker/monitoring/prometheus.yml index 1ec11bc..795e127 100644 --- a/vms/utility-101-shadow/docker/monitoring/prometheus.yml +++ b/vms/utility-101-shadow/docker/monitoring/prometheus.yml @@ -45,7 +45,7 @@ scrape_configs: - https://gitea.home.hrajfrisbee.cz/ - https://vault.hrajfrisbee.cz/ - https://idm.home.hrajfrisbee.cz/ - - https://maru-hleda-byt.home.hrajfrisbee.cz/mapa_bytu.html + # - https://maru-hleda-byt.home.hrajfrisbee.cz/mapa_bytu.html # - https://nonexistent.home.hrajfrisbee.cz/ relabel_configs: - source_labels: [__address__]