home-kubernetes/kubernetes-kvm-terraform/kubernetes-deployments/kube-prometheus.md

# 1. Add repo
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update

# 2. Install CRDs separately (production best practice - avoids Helm CRD lifecycle issues)
kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.80.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml
kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.80.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml
kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.80.0/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml
kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.80.0/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml
kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.80.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml
kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.80.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml
kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.80.0/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml
kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.80.0/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml
kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.80.0/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
kubectl apply --server-side -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.80.0/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml

# 3. Create values file (production baseline)
cat <<EOF > kube-prometheus-values.yaml
prometheus:
  prometheusSpec:
    retention: 60d
    storageSpec:
      volumeClaimTemplate:
        spec:
          # storageClassName: <your-storage-class>
          accessModes: ["ReadWriteOnce"]
          resources:
            requests:
              storage: 20Gi
    resources:
      requests:
        memory: 0.5Gi
        cpu: 500m
      limits:
        memory: 4Gi
        cpu: 2
    # Critical for ServiceMonitor discovery across namespaces
    serviceMonitorSelectorNilUsesHelmValues: false
    podMonitorSelectorNilUsesHelmValues: false
    ruleSelectorNilUsesHelmValues: false

alertmanager:
  alertmanagerSpec:
    storage:
      volumeClaimTemplate:
        spec:
          # storageClassName: <your-storage-class>
          accessModes: ["ReadWriteOnce"]
          resources:
            requests:
              storage: 3Gi

grafana:
  persistence:
    enabled: true
    # storageClassName: <your-storage-class>
    size: 10Gi
  adminPassword: admin

prometheusOperator:
  admissionWebhooks:
    certManager:
      enabled: false  # Set true if using cert-manager
EOF

# 4. Install
helm install kube-prometheus-stack prometheus-community/kube-prometheus-stack \
  -n monitoring --create-namespace \
  --set prometheusOperator.createCustomResource=false \
  -f kube-prometheus-values.yaml

# 5. Verify
kubectl -n monitoring get pods
kubectl -n monitoring get prometheuses
kubectl -n monitoring get servicemonitors --all-namespaces