observability: add k8s API/kubelet tracing, Alloy, Mimir and Loki

Wire kube-apiserver and kubelet tracing to a Jaeger collector on
docker-29, deploy Grafana Alloy in-cluster to ship logs/metrics, and
stand up Mimir + Loki on docker-30 as their backing stores.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

gitops/home-kubernetes/alloy/helmrelease_allow-events.yaml

@@ -0,0 +1,42 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: alloy-events
+  namespace: monitoring
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: alloy
+      version: ">=1.8.0 <2.0.0"
+      sourceRef: { kind: HelmRepository, name: grafana, namespace: monitoring }
+      interval: 12h
+  install: { remediation: { retries: 3 } }
+  upgrade: { remediation: { retries: 3 } }
+  driftDetection: { mode: enabled }
+  values:
+    controller:
+      type: deployment
+      replicas: 1                          # cluster-scoped → exactly one
+    image:
+      tag: v1.17.1
+    rbac:           { create: false }
+    serviceAccount: { create: true, name: alloy-events }
+    alloy:
+      enableReporting: false
+      resources:
+        requests: { cpu: 50m, memory: 96Mi }
+        limits:   { memory: 256Mi }
+      configMap:
+        content: |-
+          loki.source.kubernetes_events "events" {
+            forward_to = [loki.process.events.receiver]
+          }
+          loki.process "events" {
+            stage.static_labels { values = { source = "kubernetes-events" } }
+            forward_to = [loki.write.loki.receiver]
+          }
+          loki.write "loki" {
+            endpoint { url = "http://192.168.0.30:3100/loki/api/v1/push" }
+            external_labels = { cluster = "homelab" }
+          }

gitops/home-kubernetes/alloy/helmrelease_alloy-logs.yaml

@@ -0,0 +1,106 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: alloy-logs
+  namespace: monitoring
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: alloy
+      version: ">=1.8.0 <2.0.0"          # newest 1.x each reconcile; pin exact if you want determinism
+      sourceRef: { kind: HelmRepository, name: grafana, namespace: monitoring }
+      interval: 12h                        # poll cadence for new chart versions
+  install: { remediation: { retries: 3 } }
+  upgrade: { remediation: { retries: 3 } }
+  driftDetection: { mode: enabled }        # revert manual kubectl edits
+  values:
+    controller:
+      type: daemonset
+    image:
+      tag: v1.17.1
+    rbac:           { create: false }
+    serviceAccount: { create: true, name: alloy-logs }
+    alloy:
+      enableReporting: false
+      securityContext: { runAsUser: 0, runAsGroup: 0 }   # journald read
+      mounts:
+        varlog: true
+      resources:
+        requests: { cpu: 100m, memory: 128Mi }
+        limits:   { memory: 512Mi }
+      extraEnv:
+        - name: NODE_NAME
+          valueFrom: { fieldRef: { fieldPath: spec.nodeName } }
+      configMap:
+        content: |-
+          discovery.kubernetes "pods" {
+            role = "pod"
+            selectors {
+              role  = "pod"
+              field = "spec.nodeName=" + sys.env("NODE_NAME")
+            }
+          }
+          discovery.relabel "pods" {
+            targets = discovery.kubernetes.pods.targets
+            rule {
+              source_labels = ["__meta_kubernetes_namespace"]
+              target_label  = "namespace"
+            }
+            rule {
+              source_labels = ["__meta_kubernetes_pod_label_app"]
+              target_label  = "app"
+            }
+            rule {
+              source_labels = ["__meta_kubernetes_pod_container_name"]
+              target_label  = "container"
+            }
+            rule {
+              source_labels = ["__meta_kubernetes_pod_node_name"]
+              target_label  = "node"
+            }
+            rule {
+              source_labels = ["__meta_kubernetes_pod_name"]
+              target_label  = "pod"
+            }
+          }
+          loki.source.kubernetes "pods" {
+            targets    = discovery.relabel.pods.output
+            forward_to = [loki.process.pods.receiver]
+          }
+          loki.process "pods" {
+            stage.static_labels {
+              values = { source = "kubernetes" }
+            }
+            forward_to = [loki.write.loki.receiver]
+          }
+
+          discovery.relabel "journal" {
+            targets = []
+            rule {
+              source_labels = ["__journal__systemd_unit"]
+              target_label  = "unit"
+            }
+            rule {
+              source_labels = ["__journal__hostname"]
+              target_label  = "node"
+            }
+            rule {
+              source_labels = ["__journal_priority_keyword"]
+              target_label  = "level"
+            }
+          }
+          loki.source.journal "node" {
+            path          = "/var/log/journal"
+            max_age       = "12h"
+            labels        = { source = "node-journal" }
+            relabel_rules = discovery.relabel.journal.rules
+            forward_to    = [loki.write.loki.receiver]
+          }
+
+          loki.write "loki" {
+            endpoint {
+              url = "http://192.168.0.30:3100/loki/api/v1/push"
+            }
+            external_labels = { cluster = "homelab" }
+          }

gitops/home-kubernetes/alloy/helmrelease_alloy-metrics.yaml

@@ -0,0 +1,49 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: alloy-metrics
+  namespace: monitoring
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: alloy
+      version: ">=1.8.0 <2.0.0"
+      sourceRef: { kind: HelmRepository, name: grafana, namespace: monitoring }
+      interval: 12h
+  install: { remediation: { retries: 3 } }
+  upgrade: { remediation: { retries: 3 } }
+  driftDetection: { mode: enabled }
+  values:
+    controller:
+      type: deployment
+      replicas: 1                          # cluster-scoped scrape target discovery → exactly one
+    image:
+      tag: v1.17.1
+    rbac:           { create: false }
+    serviceAccount: { create: true, name: alloy-metrics }
+    alloy:
+      enableReporting: false
+      resources:
+        requests: { cpu: 100m, memory: 256Mi }
+        limits:   { memory: 768Mi }
+      configMap:
+        content: |-
+          // Discovers every ServiceMonitor cluster-wide. kube-prometheus-stack already ships
+          // ServiceMonitors for kubelet/cAdvisor, kube-state-metrics, node-exporter, apiserver,
+          // coredns, controller-manager and scheduler — so this alone covers "standard k8s metrics".
+          prometheus.operator.servicemonitors "sm" {
+            forward_to = [prometheus.remote_write.mimir.receiver]
+          }
+
+          // Discovers every PodMonitor cluster-wide, same selector scope as above.
+          prometheus.operator.podmonitors "pm" {
+            forward_to = [prometheus.remote_write.mimir.receiver]
+          }
+
+          prometheus.remote_write "mimir" {
+            endpoint {
+              url = "http://192.168.0.30:9009/api/v1/push"
+            }
+            external_labels = { cluster = "homelab" }
+          }

gitops/home-kubernetes/alloy/helmrepository_grafana.yaml

@@ -0,0 +1,8 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: grafana
+  namespace: monitoring
+spec:
+  interval: 1h
+  url: https://grafana.github.io/helm-charts

gitops/home-kubernetes/alloy/rbac_log-collector.yaml

@@ -0,0 +1,15 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata: { name: alloy-log-reader }
+rules:
+  - apiGroups: [""]
+    resources: ["pods", "pods/log", "namespaces", "nodes", "nodes/proxy", "events"]
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata: { name: alloy-log-reader }
+roleRef: { apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: alloy-log-reader }
+subjects:
+  - { kind: ServiceAccount, name: alloy-logs,   namespace: monitoring }
+  - { kind: ServiceAccount, name: alloy-events, namespace: monitoring }

gitops/home-kubernetes/alloy/rbac_metrics-collector.yaml

@@ -0,0 +1,25 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata: { name: alloy-metrics-reader }
+rules:
+  - apiGroups: [""]
+    resources: ["namespaces", "nodes", "nodes/metrics", "nodes/proxy", "services", "endpoints", "pods"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["discovery.k8s.io"]
+    resources: ["endpointslices"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["networking.k8s.io"]
+    resources: ["ingresses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["monitoring.coreos.com"]
+    resources: ["servicemonitors", "podmonitors", "probes", "scrapeconfigs"]
+    verbs: ["get", "list", "watch"]
+  - nonResourceURLs: ["/metrics", "/metrics/cadvisor"]
+    verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata: { name: alloy-metrics-reader }
+roleRef: { apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: alloy-metrics-reader }
+subjects:
+  - { kind: ServiceAccount, name: alloy-metrics, namespace: monitoring }

gitops/home-kubernetes/alloy/readme.md

@@ -0,0 +1,10 @@
+## debugging alloy
+
+```bash
+POD=$(kubectl -n monitoring get pod -l app.kubernetes.io/name=alloy -o name | head -1)
+kubectl -n monitoring debug -it $POD --image=nicolaka/netshoot --target=alloy -- bash
+
+
+curl -v http://192.168.0.30:3100/ready
+
+```

gitops/home-kubernetes/flux-system/extra-kustomizations.yaml

@@ -81,6 +81,21 @@ spec:
 ---
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
+metadata:
+  name: alloy
+  namespace: flux-system
+spec:
+  interval: 10m0s
+  path: ./gitops/home-kubernetes/alloy
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  dependsOn:
+    - name: kube-prometheus   # needs the monitoring ns + operator CRDs/ServiceMonitors
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
 metadata:
   name: ingress-nginx
   namespace: flux-system