gitops/velero: add manifests and runbook - kustomization is yet to be
created
This commit is contained in:
Jan Novak
141
gitops/home-kubernetes/velero/helmrelease.yaml
Normal file
141
gitops/home-kubernetes/velero/helmrelease.yaml
Normal file
@@ -0,0 +1,141 @@
|
||||
apiVersion: helm.toolkit.fluxcd.io/v2
|
||||
kind: HelmRelease
|
||||
metadata:
|
||||
name: velero
|
||||
namespace: velero
|
||||
spec:
|
||||
interval: 30m
|
||||
chart:
|
||||
spec:
|
||||
chart: velero
|
||||
version: "11.3.2" # Velero 1.16.x - latest stable as of Jan 2025
|
||||
sourceRef:
|
||||
kind: HelmRepository
|
||||
name: vmware-tanzu
|
||||
namespace: flux-system
|
||||
install:
|
||||
crds: CreateReplace
|
||||
remediation:
|
||||
retries: 3
|
||||
upgrade:
|
||||
crds: CreateReplace
|
||||
remediation:
|
||||
retries: 3
|
||||
values:
|
||||
# Node agent for filesystem backups (kopia/restic)
|
||||
deployNodeAgent: true
|
||||
nodeAgent:
|
||||
podVolumePath: /var/lib/kubelet/pods
|
||||
# nodeAgent.privileged removed in chart 8.x+, use containerSecurityContext instead
|
||||
containerSecurityContext:
|
||||
privileged: true
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
memory: 1Gi
|
||||
|
||||
configuration:
|
||||
# backupStorageLocation - note: provider is at same level as bucket, not nested
|
||||
backupStorageLocation:
|
||||
- name: default
|
||||
provider: aws
|
||||
bucket: velero-backups # create this bucket in minio first
|
||||
accessMode: ReadWrite
|
||||
default: true
|
||||
config:
|
||||
region: us-east-1 # minio ignores but required
|
||||
s3ForcePathStyle: "true"
|
||||
s3Url: http://192.168.0.2:9000 # adjust to your minio service
|
||||
|
||||
# Volume snapshot location (for CSI snapshots, optional)
|
||||
volumeSnapshotLocation:
|
||||
- name: default
|
||||
provider: aws
|
||||
config:
|
||||
region: us-east-1
|
||||
|
||||
# Use kopia for fs backups (restic deprecated, kopia is default in 1.14+)
|
||||
uploaderType: kopia
|
||||
|
||||
# Default TTL for backups
|
||||
defaultBackupTTL: 720h # 30 days
|
||||
|
||||
# Features
|
||||
defaultVolumesToFsBackup: false # opt-in via annotation per-pod
|
||||
|
||||
# Credentials
|
||||
credentials:
|
||||
useSecret: true
|
||||
existingSecret: velero-minio-credentials
|
||||
|
||||
# Velero server resources
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
memory: 512Mi
|
||||
|
||||
# Schedules
|
||||
schedules:
|
||||
daily-all-namespaces:
|
||||
disabled: false
|
||||
schedule: "0 3 * * *" # 3 AM daily
|
||||
useOwnerReferencesInBackup: false
|
||||
template:
|
||||
ttl: 168h # 7 days
|
||||
storageLocation: default
|
||||
includedNamespaces:
|
||||
- "*"
|
||||
excludedNamespaces:
|
||||
- kube-system
|
||||
- kube-public
|
||||
- kube-node-lease
|
||||
- flux-system
|
||||
- velero
|
||||
excludedResources:
|
||||
- events
|
||||
- events.events.k8s.io
|
||||
snapshotVolumes: false
|
||||
defaultVolumesToFsBackup: true
|
||||
|
||||
weekly-full:
|
||||
disabled: false
|
||||
schedule: "0 4 * * 0" # Sunday 4 AM
|
||||
template:
|
||||
ttl: 720h # 30 days
|
||||
storageLocation: default
|
||||
includedNamespaces:
|
||||
- "*"
|
||||
excludedNamespaces:
|
||||
- kube-system
|
||||
- kube-public
|
||||
- kube-node-lease
|
||||
snapshotVolumes: false
|
||||
defaultVolumesToFsBackup: true
|
||||
|
||||
# Init containers for plugins - AWS plugin for S3-compatible storage
|
||||
# Note: CSI plugin merged into velero core in v1.14, no separate initContainer needed
|
||||
initContainers:
|
||||
- name: velero-plugin-for-aws
|
||||
image: velero/velero-plugin-for-aws:v1.11.0 # compatible with Velero 1.15/1.16
|
||||
imagePullPolicy: IfNotPresent
|
||||
volumeMounts:
|
||||
- mountPath: /target
|
||||
name: plugins
|
||||
|
||||
# Metrics
|
||||
metrics:
|
||||
enabled: true
|
||||
serviceMonitor:
|
||||
enabled: false # set true if using prometheus-operator
|
||||
additionalLabels: {}
|
||||
|
||||
# Disable volume snapshots if not using CSI snapshotter
|
||||
snapshotsEnabled: false
|
||||
|
||||
# Pod annotations/labels
|
||||
podAnnotations: {}
|
||||
podLabels: {}
|
||||
8
gitops/home-kubernetes/velero/helmrepository.yaml
Normal file
8
gitops/home-kubernetes/velero/helmrepository.yaml
Normal file
@@ -0,0 +1,8 @@
|
||||
apiVersion: source.toolkit.fluxcd.io/v1
|
||||
kind: HelmRepository
|
||||
metadata:
|
||||
name: vmware-tanzu
|
||||
namespace: flux-system
|
||||
spec:
|
||||
interval: 24h
|
||||
url: https://vmware-tanzu.github.io/helm-charts
|
||||
4
gitops/home-kubernetes/velero/namespace.yaml
Normal file
4
gitops/home-kubernetes/velero/namespace.yaml
Normal file
@@ -0,0 +1,4 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: velero
|
||||
10
gitops/home-kubernetes/velero/secret-minio.yaml
Normal file
10
gitops/home-kubernetes/velero/secret-minio.yaml
Normal file
@@ -0,0 +1,10 @@
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: velero-minio-credentials
|
||||
namespace: velero
|
||||
stringData:
|
||||
cloud: |
|
||||
[default]
|
||||
aws_access_key_id=k8s
|
||||
aws_secret_access_key=poh9ieceHohnum5e
|
||||
625
gitops/home-kubernetes/velero/velero-runbook.md
Normal file
625
gitops/home-kubernetes/velero/velero-runbook.md
Normal file
@@ -0,0 +1,625 @@
|
||||
# Velero Backup & Recovery Runbook
|
||||
|
||||
## Quick Reference
|
||||
|
||||
| Operation | Command |
|
||||
|-----------|---------|
|
||||
| List backups | `velero backup get` |
|
||||
| Backup status | `velero backup describe <name> --details` |
|
||||
| Browse backup contents | `velero backup describe <name> --details \| grep -A100 "Resource List"` |
|
||||
| Restore full namespace | `velero restore create --from-backup <name> --include-namespaces <ns>` |
|
||||
| Restore single PVC | `velero restore create --from-backup <name> --include-resources pvc,pv --selector app=<label>` |
|
||||
| Restore specific files | See [Specific File Restore](#specific-file-restore) |
|
||||
|
||||
---
|
||||
|
||||
## 1. Browsing Backup Contents
|
||||
|
||||
### List All Backups
|
||||
|
||||
```bash
|
||||
# All backups with status
|
||||
velero backup get
|
||||
|
||||
# Backups for specific schedule
|
||||
velero backup get -l velero.io/schedule-name=daily-all-namespaces
|
||||
|
||||
# JSON output for scripting
|
||||
velero backup get -o json | jq '.items[] | {name: .metadata.name, phase: .status.phase, started: .status.startTimestamp}'
|
||||
```
|
||||
|
||||
### Inspect Backup Contents
|
||||
|
||||
```bash
|
||||
# Full backup details including all resources
|
||||
velero backup describe <backup-name> --details
|
||||
|
||||
# List backed-up namespaces
|
||||
velero backup describe <backup-name> --details | grep -A 5 "Namespaces:"
|
||||
|
||||
# List all resources in backup
|
||||
velero backup describe <backup-name> --details | grep -A 200 "Resource List:" | head -100
|
||||
|
||||
# Check which PVCs were backed up
|
||||
velero backup describe <backup-name> --details | grep -i persistentvolumeclaim
|
||||
|
||||
# Check pod volume backups (kopia/restic)
|
||||
velero backup describe <backup-name> --details | grep -A 50 "Pod Volume Backups"
|
||||
```
|
||||
|
||||
### View Backup Logs
|
||||
|
||||
```bash
|
||||
# Stream logs
|
||||
velero backup logs <backup-name>
|
||||
|
||||
# Search for errors
|
||||
velero backup logs <backup-name> | grep -i error
|
||||
|
||||
# Check specific namespace backup
|
||||
velero backup logs <backup-name> | grep "namespace=seafile"
|
||||
```
|
||||
|
||||
### Browse Kopia Repository Directly
|
||||
|
||||
For direct file-level inspection of kopia backups in MinIO:
|
||||
|
||||
```bash
|
||||
# Get kopia repository password from velero secret
|
||||
KOPIA_PASSWORD=$(kubectl get secret -n velero velero-repo-credentials -o jsonpath='{.data.repository-password}' | base64 -d)
|
||||
|
||||
# Connect to repository (run from a pod with minio access or port-forward)
|
||||
kopia repository connect s3 \
|
||||
--bucket=velero-backups \
|
||||
--endpoint=minio.minio.svc.cluster.local:9000 \
|
||||
--access-key=<MINIO_ACCESS_KEY> \
|
||||
--secret-access-key=<MINIO_SECRET_KEY> \
|
||||
--password="${KOPIA_PASSWORD}" \
|
||||
--prefix=kopia/<cluster-name>/
|
||||
|
||||
# List snapshots
|
||||
kopia snapshot list --all
|
||||
|
||||
# Browse specific snapshot
|
||||
kopia snapshot list <snapshot-id>
|
||||
kopia ls <snapshot-id>
|
||||
|
||||
# Mount for browsing (requires FUSE)
|
||||
mkdir /tmp/kopia-mount
|
||||
kopia mount <snapshot-id> /tmp/kopia-mount &
|
||||
ls /tmp/kopia-mount/
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 2. Full Namespace Restore
|
||||
|
||||
### Restore to Same Cluster (Disaster Recovery)
|
||||
|
||||
```bash
|
||||
# Restore entire namespace
|
||||
velero restore create seafile-restore \
|
||||
--from-backup daily-all-namespaces-20250115030000 \
|
||||
--include-namespaces seafile \
|
||||
--wait
|
||||
|
||||
# Monitor restore progress
|
||||
velero restore describe seafile-restore --details
|
||||
velero restore logs seafile-restore -f
|
||||
```
|
||||
|
||||
### Restore to Different Namespace
|
||||
|
||||
```bash
|
||||
velero restore create seafile-test-restore \
|
||||
--from-backup daily-all-namespaces-20250115030000 \
|
||||
--include-namespaces seafile \
|
||||
--namespace-mappings seafile:seafile-restored \
|
||||
--wait
|
||||
```
|
||||
|
||||
### Restore with Resource Filtering
|
||||
|
||||
```bash
|
||||
# Restore only specific resource types
|
||||
velero restore create restore-pvcs-only \
|
||||
--from-backup <backup-name> \
|
||||
--include-namespaces seafile \
|
||||
--include-resources persistentvolumeclaims,persistentvolumes \
|
||||
--wait
|
||||
|
||||
# Exclude certain resources
|
||||
velero restore create restore-no-secrets \
|
||||
--from-backup <backup-name> \
|
||||
--include-namespaces seafile \
|
||||
--exclude-resources secrets \
|
||||
--wait
|
||||
|
||||
# Restore by label selector
|
||||
velero restore create restore-app \
|
||||
--from-backup <backup-name> \
|
||||
--selector app.kubernetes.io/name=seafile \
|
||||
--wait
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. Single PVC/Volume Restore
|
||||
|
||||
### Restore Specific PVC
|
||||
|
||||
```bash
|
||||
# First, scale down the workload using the PVC
|
||||
kubectl scale deployment seafile -n seafile --replicas=0
|
||||
|
||||
# Delete the corrupted/problematic PVC (data will be restored)
|
||||
kubectl delete pvc seafile-data -n seafile
|
||||
|
||||
# Restore just that PVC
|
||||
velero restore create restore-seafile-pvc \
|
||||
--from-backup <backup-name> \
|
||||
--include-namespaces seafile \
|
||||
--include-resources persistentvolumeclaims,persistentvolumes \
|
||||
--selector app=seafile \
|
||||
--wait
|
||||
|
||||
# Scale back up
|
||||
kubectl scale deployment seafile -n seafile --replicas=1
|
||||
```
|
||||
|
||||
### Restore PVC to New Name (Side-by-Side)
|
||||
|
||||
```bash
|
||||
# Create restore with transforms
|
||||
cat <<EOF | kubectl apply -f -
|
||||
apiVersion: velero.io/v1
|
||||
kind: Restore
|
||||
metadata:
|
||||
name: restore-pvc-new-name
|
||||
namespace: velero
|
||||
spec:
|
||||
backupName: <backup-name>
|
||||
includedNamespaces:
|
||||
- seafile
|
||||
includedResources:
|
||||
- persistentvolumeclaims
|
||||
- persistentvolumes
|
||||
labelSelector:
|
||||
matchLabels:
|
||||
app: seafile
|
||||
restorePVs: true
|
||||
namespaceMapping:
|
||||
seafile: seafile-recovery
|
||||
EOF
|
||||
|
||||
# Or use restore hooks to rename
|
||||
velero restore create restore-pvc-renamed \
|
||||
--from-backup <backup-name> \
|
||||
--include-namespaces seafile \
|
||||
--namespace-mappings seafile:seafile-temp \
|
||||
--wait
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. Specific File Restore
|
||||
|
||||
Velero doesn't support single-file restore natively. Use kopia directly:
|
||||
|
||||
### Method 1: Kopia Direct Restore
|
||||
|
||||
```bash
|
||||
# Find the backup/snapshot containing your file
|
||||
# First, get velero's kopia repo credentials
|
||||
REPO_PASSWORD=$(kubectl get secret -n velero velero-repo-credentials \
|
||||
-o jsonpath='{.data.repository-password}' | base64 -d)
|
||||
|
||||
# Run a debug pod with kopia
|
||||
kubectl run kopia-restore --rm -it \
|
||||
--image=kopia/kopia:latest \
|
||||
--restart=Never \
|
||||
--namespace=velero \
|
||||
-- /bin/sh
|
||||
|
||||
# Inside the pod:
|
||||
kopia repository connect s3 \
|
||||
--bucket=velero-backups \
|
||||
--endpoint=minio.minio.svc.cluster.local:9000 \
|
||||
--access-key=<ACCESS_KEY> \
|
||||
--secret-access-key=<SECRET_KEY> \
|
||||
--password="<REPO_PASSWORD>" \
|
||||
--prefix=kopia/<cluster>/
|
||||
|
||||
# List snapshots for specific PVC
|
||||
kopia snapshot list --all | grep seafile
|
||||
|
||||
# Restore specific file
|
||||
kopia restore <snapshot-id>/path/to/file.txt /tmp/restored-file.txt
|
||||
|
||||
# Restore specific directory
|
||||
kopia restore <snapshot-id>/data/uploads/ /tmp/restored-uploads/
|
||||
```
|
||||
|
||||
### Method 2: Mount and Copy
|
||||
|
||||
```bash
|
||||
# Create a temporary pod that mounts the backup
|
||||
cat <<EOF | kubectl apply -f -
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: backup-browser
|
||||
namespace: velero
|
||||
spec:
|
||||
containers:
|
||||
- name: browser
|
||||
image: kopia/kopia:latest
|
||||
command: ["sleep", "3600"]
|
||||
env:
|
||||
- name: KOPIA_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: velero-repo-credentials
|
||||
key: repository-password
|
||||
volumeMounts:
|
||||
- name: restore-target
|
||||
mountPath: /restore
|
||||
volumes:
|
||||
- name: restore-target
|
||||
emptyDir: {}
|
||||
EOF
|
||||
|
||||
# Exec in and restore files
|
||||
kubectl exec -it -n velero backup-browser -- /bin/sh
|
||||
# ... run kopia commands inside
|
||||
```
|
||||
|
||||
### Method 3: Full PVC Restore + Copy + Delete
|
||||
|
||||
```bash
|
||||
# 1. Restore PVC to temp namespace
|
||||
velero restore create temp-restore \
|
||||
--from-backup <backup-name> \
|
||||
--include-namespaces seafile \
|
||||
--namespace-mappings seafile:temp-restore \
|
||||
--include-resources pvc,pv \
|
||||
--wait
|
||||
|
||||
# 2. Create a pod to access both PVCs
|
||||
cat <<EOF | kubectl apply -f -
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: file-copier
|
||||
namespace: seafile
|
||||
spec:
|
||||
containers:
|
||||
- name: copier
|
||||
image: alpine
|
||||
command: ["sleep", "3600"]
|
||||
volumeMounts:
|
||||
- name: current
|
||||
mountPath: /current
|
||||
- name: restored
|
||||
mountPath: /restored
|
||||
volumes:
|
||||
- name: current
|
||||
persistentVolumeClaim:
|
||||
claimName: seafile-data
|
||||
- name: restored
|
||||
persistentVolumeClaim:
|
||||
claimName: seafile-data # in temp-restore namespace - need cross-ns mount or copy via node
|
||||
EOF
|
||||
|
||||
# Alternative: use rsync between namespaces
|
||||
kubectl exec -n temp-restore deployment/temp-pod -- tar cf - /data/specific-file.txt | \
|
||||
kubectl exec -i -n seafile deployment/seafile -- tar xf - -C /
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. Database-Specific Recovery
|
||||
|
||||
### MariaDB (via mariadb-operator)
|
||||
|
||||
Velero fs-backup of running DB may be inconsistent. Prefer operator backups:
|
||||
|
||||
```bash
|
||||
# List operator backups
|
||||
kubectl get backups.k8s.mariadb.com -n mariadb
|
||||
|
||||
# Restore from operator backup
|
||||
kubectl apply -f - <<EOF
|
||||
apiVersion: k8s.mariadb.com/v1alpha1
|
||||
kind: Restore
|
||||
metadata:
|
||||
name: mariadb-restore
|
||||
namespace: mariadb
|
||||
spec:
|
||||
mariaDbRef:
|
||||
name: mariadb
|
||||
backupRef:
|
||||
name: mariadb-backup-20250115
|
||||
EOF
|
||||
```
|
||||
|
||||
If you must restore from Velero:
|
||||
|
||||
```bash
|
||||
# 1. Scale down mariadb
|
||||
kubectl scale statefulset mariadb -n mariadb --replicas=0
|
||||
|
||||
# 2. Restore PVC
|
||||
velero restore create mariadb-pvc-restore \
|
||||
--from-backup <backup-name> \
|
||||
--include-namespaces mariadb \
|
||||
--include-resources pvc,pv \
|
||||
--wait
|
||||
|
||||
# 3. Scale back up - DB will recover from WAL
|
||||
kubectl scale statefulset mariadb -n mariadb --replicas=1
|
||||
|
||||
# 4. Verify data integrity
|
||||
kubectl exec -it -n mariadb mariadb-0 -- mariadb -e "CHECK TABLE important_table;"
|
||||
```
|
||||
|
||||
### Redis
|
||||
|
||||
```bash
|
||||
# If Redis is persistent (RDB/AOF)
|
||||
kubectl scale statefulset redis -n redis --replicas=0
|
||||
|
||||
velero restore create redis-restore \
|
||||
--from-backup <backup-name> \
|
||||
--include-namespaces redis \
|
||||
--wait
|
||||
|
||||
kubectl scale statefulset redis -n redis --replicas=1
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. Backup Management
|
||||
|
||||
### Create On-Demand Backup
|
||||
|
||||
```bash
|
||||
# Full backup
|
||||
velero backup create manual-backup-$(date +%Y%m%d-%H%M%S) \
|
||||
--default-volumes-to-fs-backup \
|
||||
--snapshot-volumes=false \
|
||||
--wait
|
||||
|
||||
# Specific namespace pre-maintenance
|
||||
velero backup create pre-upgrade-seafile-$(date +%Y%m%d) \
|
||||
--include-namespaces seafile \
|
||||
--default-volumes-to-fs-backup \
|
||||
--wait
|
||||
```
|
||||
|
||||
### Delete Old Backups
|
||||
|
||||
```bash
|
||||
# Delete specific backup
|
||||
velero backup delete <backup-name> --confirm
|
||||
|
||||
# Delete backups older than 30 days (careful!)
|
||||
velero backup get -o json | jq -r '.items[] | select(.status.startTimestamp < (now - 2592000 | todate)) | .metadata.name' | xargs -I {} velero backup delete {} --confirm
|
||||
```
|
||||
|
||||
### Check Backup Storage Location Health
|
||||
|
||||
```bash
|
||||
velero backup-location get
|
||||
velero backup-location describe default
|
||||
|
||||
# Verify connectivity
|
||||
kubectl logs -n velero deployment/velero | grep -i "backup storage location"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 7. Disaster Recovery Procedures
|
||||
|
||||
### Complete Cluster Rebuild
|
||||
|
||||
```bash
|
||||
# 1. Install Velero on new cluster with same config
|
||||
helm upgrade --install velero vmware-tanzu/velero \
|
||||
-n velero --create-namespace \
|
||||
-f velero-values.yaml
|
||||
|
||||
# 2. Wait for velero to sync backup list from S3
|
||||
sleep 60
|
||||
velero backup get
|
||||
|
||||
# 3. Restore namespaces in order (dependencies first)
|
||||
# Restore storage/infra
|
||||
velero restore create restore-infra \
|
||||
--from-backup <latest-backup> \
|
||||
--include-namespaces minio,cert-manager \
|
||||
--wait
|
||||
|
||||
# Restore databases
|
||||
velero restore create restore-databases \
|
||||
--from-backup <latest-backup> \
|
||||
--include-namespaces mariadb,redis \
|
||||
--wait
|
||||
|
||||
# Restore applications
|
||||
velero restore create restore-apps \
|
||||
--from-backup <latest-backup> \
|
||||
--include-namespaces seafile,plane \
|
||||
--wait
|
||||
```
|
||||
|
||||
### Restore Schedule After Accidental Deletion
|
||||
|
||||
```bash
|
||||
# Schedules are cluster resources, restore from backup
|
||||
velero restore create restore-schedules \
|
||||
--from-backup <backup-name> \
|
||||
--include-resources schedules.velero.io \
|
||||
--wait
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 8. Troubleshooting
|
||||
|
||||
### Backup Stuck/Failed
|
||||
|
||||
```bash
|
||||
# Check velero logs
|
||||
kubectl logs -n velero deployment/velero --tail=100
|
||||
|
||||
# Check node-agent on specific node
|
||||
kubectl logs -n velero -l name=node-agent --tail=100
|
||||
|
||||
# Check backup details for errors
|
||||
velero backup describe <backup-name> --details | grep -i -A5 "error\|warning\|failed"
|
||||
|
||||
# Common issues:
|
||||
# - Node-agent not running on node with PV
|
||||
kubectl get pods -n velero -l name=node-agent -o wide
|
||||
# - PVC not annotated for backup
|
||||
kubectl get pvc -A -o json | jq '.items[] | select(.metadata.annotations["backup.velero.io/backup-volumes"] != null)'
|
||||
```
|
||||
|
||||
### Restore Not Restoring Volumes
|
||||
|
||||
```bash
|
||||
# Check if backup has pod volume backups
|
||||
velero backup describe <backup-name> --details | grep -A20 "Pod Volume Backups"
|
||||
|
||||
# Verify restore is configured to restore PVs
|
||||
velero restore describe <restore-name> --details | grep -i "restorePVs"
|
||||
|
||||
# Force PV restore
|
||||
velero restore create <name> \
|
||||
--from-backup <backup> \
|
||||
--restore-volumes=true \
|
||||
--wait
|
||||
```
|
||||
|
||||
### Kopia Repository Issues
|
||||
|
||||
```bash
|
||||
# Check repository status
|
||||
kubectl exec -n velero deployment/velero -- \
|
||||
velero repo get
|
||||
|
||||
# Unlock stuck repository
|
||||
kubectl exec -n velero deployment/velero -- \
|
||||
velero repo unlock <repo-name>
|
||||
|
||||
# Maintenance (runs automatically, but can trigger manually)
|
||||
kubectl exec -n velero deployment/velero -- \
|
||||
velero repo maintenance run
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 9. Monitoring & Alerting
|
||||
|
||||
### Prometheus Metrics
|
||||
|
||||
Key metrics to monitor:
|
||||
|
||||
```promql
|
||||
# Backup success rate
|
||||
sum(velero_backup_success_total) / sum(velero_backup_attempt_total)
|
||||
|
||||
# Backup duration
|
||||
velero_backup_duration_seconds{schedule="daily-all-namespaces"}
|
||||
|
||||
# Backup size
|
||||
velero_backup_items_total{backup="<name>"}
|
||||
|
||||
# Failed backups in last 24h
|
||||
increase(velero_backup_failure_total[24h])
|
||||
```
|
||||
|
||||
### AlertManager Rules
|
||||
|
||||
```yaml
|
||||
groups:
|
||||
- name: velero
|
||||
rules:
|
||||
- alert: VeleroBackupFailed
|
||||
expr: increase(velero_backup_failure_total[1h]) > 0
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Velero backup failed"
|
||||
|
||||
- alert: VeleroBackupMissing
|
||||
expr: time() - velero_backup_last_successful_timestamp{schedule="daily-all-namespaces"} > 86400
|
||||
for: 1h
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "No successful backup in 24h"
|
||||
|
||||
- alert: VeleroNodeAgentDown
|
||||
expr: kube_daemonset_status_number_unavailable{daemonset="node-agent"} > 0
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 10. Regular Maintenance Tasks
|
||||
|
||||
### Weekly
|
||||
|
||||
```bash
|
||||
# Verify recent backup integrity
|
||||
velero backup describe $(velero backup get -o json | jq -r '.items | sort_by(.status.startTimestamp) | last | .metadata.name') --details
|
||||
|
||||
# Check backup storage usage
|
||||
mc ls minio/velero-backups --summarize
|
||||
```
|
||||
|
||||
### Monthly
|
||||
|
||||
```bash
|
||||
# Test restore to scratch namespace
|
||||
velero restore create monthly-test-$(date +%Y%m) \
|
||||
--from-backup $(velero backup get -o json | jq -r '.items[0].metadata.name') \
|
||||
--include-namespaces seafile \
|
||||
--namespace-mappings seafile:restore-test \
|
||||
--wait
|
||||
|
||||
# Verify restored data
|
||||
kubectl exec -n restore-test deploy/seafile -- ls -la /data
|
||||
|
||||
# Cleanup test
|
||||
kubectl delete namespace restore-test
|
||||
velero restore delete monthly-test-$(date +%Y%m) --confirm
|
||||
```
|
||||
|
||||
### Quarterly
|
||||
|
||||
- Full DR test: restore to separate cluster
|
||||
- Review retention policies
|
||||
- Audit backup coverage (new namespaces/PVCs added?)
|
||||
- Update velero/plugin versions
|
||||
|
||||
---
|
||||
|
||||
## Appendix: Common Label Selectors
|
||||
|
||||
```bash
|
||||
# Backup by app label
|
||||
--selector app.kubernetes.io/name=seafile
|
||||
|
||||
# Backup by component
|
||||
--selector app.kubernetes.io/component=database
|
||||
|
||||
# Exclude specific pods from backup
|
||||
# (add to pod annotation)
|
||||
kubectl annotate pod <pod> backup.velero.io/backup-volumes-excludes=cache,tmp
|
||||
```
|
||||
Reference in New Issue
Block a user