diff --git a/CLAUDE.md b/CLAUDE.md index 1026f73..32263e1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -226,3 +226,6 @@ A lightweight `check` job (`go vet` / `go build` / `go test -short`) gates the b - `+kubebuilder:rbac` markers are package-level. Keep them in their own comment block, never inside a type's or function's doc comment: controller-gen silently ignores them there. After `make manifests`, check `git diff config/rbac/role.yaml`. +- `make test-integration-proxmox` runs the proxmox provider's integration tests against + a real Proxmox VE host. They skip unless `PROXMOX_TEST_URL`, `_TOKEN_FILE`, `_NODE`, + `_TEMPLATE_VMID` and `_ISO_STORAGE` are set; see docs/testing.md for the full env block. diff --git a/Makefile b/Makefile index d5fd541..356027d 100644 --- a/Makefile +++ b/Makefile @@ -200,6 +200,23 @@ kvm-ssh-secret: ## Create or update the kvm-ssh Secret from KVM_SSH_KEY (private --from-file=known_hosts="$(KVM_KNOWN_HOSTS)" \ --dry-run=client -o yaml | "$(KUBECTL)" apply -f - +PROXMOX_TOKEN_FILE ?= $(HOME)/.config/proxmox-token +PROXMOX_CA_FILE ?= $(HOME)/.config/proxmox-ca.pem +PROXMOX_NAMESPACE ?= egress-proxies-operator-system + +.PHONY: proxmox-token-secret +proxmox-token-secret: ## Create or update the proxmox-token Secret from PROXMOX_TOKEN_FILE (user@realm!tokenid=secret) and PROXMOX_CA_FILE. + @test -f "$(PROXMOX_TOKEN_FILE)" || { echo "PROXMOX_TOKEN_FILE=$(PROXMOX_TOKEN_FILE) not found"; exit 1; } + @test -f "$(PROXMOX_CA_FILE)" || { echo "PROXMOX_CA_FILE=$(PROXMOX_CA_FILE) not found"; exit 1; } + @test "$$(grep -cv '^[[:space:]]*$$' "$(PROXMOX_TOKEN_FILE)")" = 1 && \ + grep -qxE '[[:space:]]*[^@[:space:]]+@[^![:space:]]+![^=[:space:]]+=[^[:space:]]+[[:space:]]*' "$(PROXMOX_TOKEN_FILE)" || \ + { echo "PROXMOX_TOKEN_FILE=$(PROXMOX_TOKEN_FILE) is not one line of user@realm!tokenid=secret"; exit 1; } + @grep -q 'BEGIN CERTIFICATE' "$(PROXMOX_CA_FILE)" || { echo "PROXMOX_CA_FILE=$(PROXMOX_CA_FILE) holds no PEM certificate"; exit 1; } + "$(KUBECTL)" -n $(PROXMOX_NAMESPACE) create secret generic proxmox-token \ + --from-file=token="$(PROXMOX_TOKEN_FILE)" \ + --from-file=ca.pem="$(PROXMOX_CA_FILE)" \ + --dry-run=client -o yaml | "$(KUBECTL)" apply -f - + .PHONY: deploy deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. cd config/manager && "$(KUSTOMIZE)" edit set image controller=${IMG} diff --git a/README.md b/README.md index 87d9d1a..80cbc65 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,7 @@ can be two differently-configured instances of the same `gcp` type (see | `gcp` | A Compute Engine VM with an ephemeral external IP | `project` (required), `network`, `networkTag`, `diskSizeGb` | The real egress fleet. Deliberately uses only four API calls (Insert / Get / Delete / AggregatedList), all fire-and-forget: `Create` returns as soon as the operation is submitted and the reconciler discovers progress by polling `Get`. Auth is Application Default Credentials — workload identity in-cluster, `gcloud` ADC locally; no key-file plumbing. | | `kvm` | A libvirt VM on one of your own hypervisors, reached over SSH | `sshKeyFile`, `knownHostsFile` (unless `insecureIgnoreHostKey`), `machineTypes`, `hosts` (required); `images`, `diskSizeGb`, `vendorData`, `installGuestAgent` | Home-lab or on-prem egress. The base image is downloaded into each host's pool on first use; `Create` waits for the VM to start. See [docs/kvm-provider.md](docs/kvm-provider.md). | | `kvm2` | The same: a libvirt VM on one of your own hypervisors, reached over SSH | the same block shape as `kvm`, under `kvm2:` | The second-generation libvirt provider, built from the provider contract. Adds a child span per libvirt RPC, a pool-space precheck reported as quota (the reconciler waits instead of failing), SSH credentials re-read on every dial, and image downloads and health checks on connections of their own. It does not adopt `kvm`'s VMs. See [docs/kvm2-provider.md](docs/kvm2-provider.md). | +| `proxmox` | A VM on a Proxmox VE cluster: a full clone of a template you prepare, reached through the Proxmox REST API with an API token | `url`, `tokenFile`, `nodes`, `isoStorage`, `templates`, `machineTypes` (required); `caFile` or `insecureSkipTLSVerify`, `storage`, `bridge`, `vlanTag`, `vmidRange`, `requestTimeoutSeconds`, `vendorData`, `installGuestAgent` | Home-lab or on-prem egress without SSH. Cloud-init goes on a seed ISO uploaded through the API; the address comes from the QEMU guest agent, so the proxy network needs a route to a package mirror unless the template ships the agent. `Create` waits for the VM to run. The token is read on every request, from the optional `proxmox-token` Secret (`make proxmox-token-secret`). Built from the provider contract; not yet run against a real host end to end. See [docs/proxmox-provider.md](docs/proxmox-provider.md). | ### Adding a provider diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index c334b6d..3abe901 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -166,6 +166,18 @@ spec: - name: kvm-ssh mountPath: /etc/proxy-operator/kvm readOnly: true + # API token and CA for the proxmox provider (tokenFile / caFile in + # the providers config point here). Optional: the mount is empty when + # the Secret does not exist, and only proxmox entries need it. The + # token file is read on every request; the CA once, on first use. + # Create it with: + # kubectl -n egress-proxies-operator-system create secret generic proxmox-token \ + # --from-file=token= \ + # --from-file=ca.pem= + # or `make proxmox-token-secret`. + - name: proxmox-token + mountPath: /etc/proxy-operator/proxmox + readOnly: true # Scratch space: the root filesystem is read-only, and the kvm # provider builds each VM's cloud-init seed ISO through a temp # directory (a few hundred KB, deleted right after upload). @@ -180,6 +192,11 @@ spec: secret: secretName: kvm-ssh optional: true + # The proxmox provider's token and CA; see the mount above. + - name: proxmox-token + secret: + secretName: proxmox-token + optional: true - name: tmp emptyDir: sizeLimit: 64Mi diff --git a/config/manager/providers_config.yaml b/config/manager/providers_config.yaml index b3ae1c5..95beb55 100644 --- a/config/manager/providers_config.yaml +++ b/config/manager/providers_config.yaml @@ -84,4 +84,32 @@ data: bridge: br0 # socket: /var/run/libvirt/libvirt-sock # pool: default + # proxmox clones a prepared template on a Proxmox VE cluster through its + # REST API and seeds it with a NoCloud ISO uploaded to isoStorage; it needs + # no SSH. Both files come from the optional proxmox-token Secret mounted by + # config/manager/manager.yaml. The url's host must be a name or address in + # the node certificate. See docs/proxmox-provider.md. + # - name: proxmox-home + # type: proxmox + # proxmox: + # url: https://192.168.0.8:8006 + # tokenFile: /etc/proxy-operator/proxmox/token + # caFile: /etc/proxy-operator/proxmox/ca.pem + # # insecureSkipTLSVerify: true # instead of caFile; logged as a warning + # nodes: [pv1-1] # placement.zone names a node; optional with one node + # isoStorage: proxy-seeds # iso content; the token needs Datastore.Allocate here + # # storage: local-lvm # clone target; default is the template's disk storage + # # bridge: vmbr0 # replaces the cloned net0's bridge + # vlanTag: 4 # set on the cloned net0, MAC kept + # # vmidRange: {min: 100000, max: 199999} + # # requestTimeoutSeconds: 30 + # # installGuestAgent: false + # # vendorData: | + # # #cloud-config + # # ... + # templates: # placement.image names a key; add keys, never repoint one + # debian-13: {vmid: 9001} + # machineTypes: # placement.machineType names a key here + # small: {cores: 1, memoryMiB: 1024} + # medium: {cores: 2, memoryMiB: 2048, diskGiB: 10} diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index 518981b..37bab9d 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -9,5 +9,6 @@ resources: - proxy_gcp.yaml - proxy_kvm.yaml - proxy_kvm2.yaml +- proxy_proxmox.yaml - proxy_external.yaml # +kubebuilder:scaffold:manifestskustomizesamples diff --git a/config/samples/providers-config.yaml b/config/samples/providers-config.yaml index 4dc272d..0ea9918 100644 --- a/config/samples/providers-config.yaml +++ b/config/samples/providers-config.yaml @@ -88,3 +88,31 @@ providers: # bridge: br0 # # socket: /var/run/libvirt/libvirt-sock # # pool: default + # proxmox clones a prepared template on a Proxmox VE cluster through its + # REST API and seeds it with a NoCloud ISO uploaded to isoStorage; it needs + # no SSH. Both files come from the optional proxmox-token Secret mounted by + # config/manager/manager.yaml. The url's host must be a name or address in + # the node certificate. See docs/proxmox-provider.md. + # - name: proxmox-home + # type: proxmox + # proxmox: + # url: https://192.168.0.8:8006 + # tokenFile: /etc/proxy-operator/proxmox/token + # caFile: /etc/proxy-operator/proxmox/ca.pem + # # insecureSkipTLSVerify: true # instead of caFile; logged as a warning + # nodes: [pv1-1] # placement.zone names a node; optional with one node + # isoStorage: proxy-seeds # iso content; the token needs Datastore.Allocate here + # # storage: local-lvm # clone target; default is the template's disk storage + # # bridge: vmbr0 # replaces the cloned net0's bridge + # vlanTag: 4 # set on the cloned net0, MAC kept + # # vmidRange: {min: 100000, max: 199999} + # # requestTimeoutSeconds: 30 + # # installGuestAgent: false + # # vendorData: | + # # #cloud-config + # # ... + # templates: # placement.image names a key; add keys, never repoint one + # debian-13: {vmid: 9001} + # machineTypes: # placement.machineType names a key here + # small: {cores: 1, memoryMiB: 1024} + # medium: {cores: 2, memoryMiB: 2048, diskGiB: 10} diff --git a/config/samples/proxy_proxmox.yaml b/config/samples/proxy_proxmox.yaml new file mode 100644 index 0000000..7e9b361 --- /dev/null +++ b/config/samples/proxy_proxmox.yaml @@ -0,0 +1,46 @@ +# A Managed proxy backed by a Proxmox VE cluster through the proxmox +# provider: the operator full-clones a prepared template on the named node, +# attaches a NoCloud seed ISO carrying this cloud-init, and starts the VM on +# the configured network (here VLAN 4 on vmbr0, IP from that network's +# DHCP), where cloud-init installs Squid. Requires a providers-config entry +# named "proxmox-home" (see providers-config.yaml) and the proxmox-token +# Secret (docs/proxmox-provider.md). +# +# placement.zone is a node name from proxmox.nodes (optional when only one +# node is configured); machineType and image are keys in the provider's +# machineTypes and templates maps. The address comes from the guest agent, +# which a vendor-data script installs at first boot, so the VM's network +# needs a route to a package mirror. +apiVersion: crawl.example.com/v1alpha1 +kind: Proxy +metadata: + name: proxy-proxmox-sample +spec: + mode: Managed + provider: proxmox-home + placement: + zone: pv1-1 + machineType: small + image: debian-13 + port: 3128 + cloudInit: + inline: | + #cloud-config + packages: + - squid + write_files: + - path: /etc/squid/conf.d/proxy-operator.conf + content: | + http_port 3128 + http_access allow all + via off + forwarded_for off + runcmd: + - systemctl restart squid + # Optional: log in with `ssh debian@` (the Debian image's default + # user) to look around; `qm terminal ` on the node works too. + # ssh_authorized_keys: + # - ssh-ed25519 AAAA... you@laptop + attributes: + geo: home + purpose: crawl diff --git a/docs/architecture.md b/docs/architecture.md index 03d458f..b8f3ece 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -11,6 +11,7 @@ | gcp provider | `internal/provider/gcp` | library | — | Compute Engine VMs, four API calls, fire-and-forget ops | | kvm provider | `internal/provider/kvm` | library | — | VMs on libvirt hosts over SSH-forwarded sockets; image cache, overlay disks, NoCloud seeds | | kvm2 provider | `internal/provider/kvm2` | library | — | the same backend, built clean-room from the provider contract as a second type: per-RPC child spans, pool-space quota, per-call budgets, its own metadata namespace, credentials re-read on every dial | +| proxmox provider | `internal/provider/proxmox` | library | — | VMs on a Proxmox VE cluster through its REST API: full clones of a template, a NoCloud seed uploaded through the API, the VMID derived from the Proxy UID, every task waited on, HTTP child spans | | Shared provider helpers | `internal/provider/nocloud`, `internal/provider/keyedlock` | library | — | the NoCloud seed ISO and vendor-data builder, and one-slot semaphores by key; used by kvm2 and proxmox | | Health engine | `internal/health` | Runnable | yes | through-the-proxy probes, thresholds, transition events | | Lease store | `internal/lease` | Runnable (expiry sweep) | no | in-memory leases + cooldowns, single mutex | @@ -204,6 +205,37 @@ outlives its ctx or budget kills the pooled connection (the only way to unblock go-libvirt), so other calls in flight on that host fail once with Transient and retry; the next call redials. Each RPC is a child span kvm2. under provider.*, plus kvm2.dial and kvm2.download. + +proxmox provider (internal/provider/proxmox/) - the Proxmox VE REST API, one URL per cluster +prov.Create ──► placement checked against the catalogue (Permanent before any request) + GET /cluster/resources: a VM with the name → ownership check ─┐ every step + else: template config checked (not a template, cloud-init drive, │ idempotent, + no net0 or boot disk it needs = Permanent) │ under a + VMID = hash(UID) in vmidRange, probing on; a taken VMID → │ per-name lock + ownership check: wait out a lock, adopt our name + mark + │ shared with + UID, probe past another name, Permanent otherwise │ Delete; each + POST .../clone (full, description = the mark) → wait task │ task's UPID + upload seed ISO to isoStorage (overwrites) → wait task │ polled every + PUT .../config: cores, memory, agent=1, tag, ide2 = seed, │ 1s to its end + net0 bridge/tag when configured │ + resize if diskGiB is larger → wait task │ + start → wait task → poll status/current until running (60s) ─┘ +prov.Get ──► /cluster/resources (absent = NotFound; lock or node unknown = Transient) + + config (name and mark must match, else NotFound: VMID reused) + + status/current (paused and hibernated = Stopped) + + agent network-get-interfaces (first routable IPv4; agent not + up yet = Provisioning) +prov.Delete ──► resources + config: gone, not ours or unmarked = leave it; + hibernated = Permanent; stop → destroy (purge) → delete the seed + on the VM's node (or on every node when no VM of ours was left) +prov.ListByTag ─► one /cluster/resources for the whole cluster, one config read per + candidate (tag or proxy- name); the description's mark decides +Check ──► per node: /version, node status, isoStorage status, templates in + /cluster/resources +providerID = vms//: no node, because VMs migrate; the node is looked +up in /cluster/resources on every call. Token read from its file on every +request. Each request is an HTTP child span from the parent-gated +transport wrapper; a cancelled request ends only itself. ``` The reconciler never watches provider-side resources (Pods or GCP VMs). @@ -388,6 +420,12 @@ incoming trace context): from a span decorator over the seam, applied once in the dialer, so no RPC is unspanned. The `libvirt call` log line of an RPC is written inside its span. The `kvm` provider has no child spans. +- proxmox child spans: one `HTTP ` client span per API request, + from `tracing.RestConfigWrapper()` on the provider's own HTTP transport, + under `provider.*` or `providerhealth.check`. Parent-gated like the + Kubernetes API spans, so a request without a parent starts none. The + provider's logging decorator starts no span; its `proxmox api call` line + carries the operation span's IDs. - Kubernetes API calls become client child spans via a wrapped `rest.Config` transport — gated on an existing parent span, so informer list/watch long-polls and leader-election renewals never create root @@ -413,12 +451,13 @@ the slog handler's ctx. Answers "can this cluster reach each provider's backend right now?" without waiting for a Proxy to fail. A provider opts in by implementing `provider.Checker`: `Check(ctx) []TargetStatus` returns one verdict per -target and never fails as a whole. All four providers do: +target and never fails as a whole. All five providers do: | Provider | Target | What one check does | |---|---|---| | kvm | each `kvm.hosts` entry | over the pooled SSH+libvirt connection: libvirt version and hostname, then the configured pool must exist and be running; reports pool capacity and free space ([details](kvm-provider.md#health-check)) | | kvm2 | each `kvm2.hosts` entry | on a fresh SSH+libvirt connection dialled for the check and closed after it: libvirt version and hostname, then the configured pool must exist and be running; `details` carry `user`, `pool`, `hostname`, `libvirtVersion`, `poolState`, `poolCapacityGiB` and `poolAvailableGiB` ([details](kvm2-provider.md#health-check)) | +| proxmox | each `proxmox.nodes` entry | four requests on the shared HTTP client: API version, node status, the `isoStorage` status (it must be active and hold `iso` content), and the cluster's resources (every configured template must exist and be a template); `details` carry `isoStorage`, `tokenID`, `pveVersion`, `kernel`, `isoStorageAvailGiB` and `templates` ([details](proxmox-provider.md#health-check)) | | kubernetes | `apiserver` | lists at most one managed Pod cluster-wide: the verb and scope `ListByTag` and GC rely on | | gcp | the project | one page of an aggregated instance list with `maxResults=1`: credentials, Compute API enabled, `compute.instances.list` | @@ -704,3 +743,29 @@ they read as choices, not accidents. Chronological by build step. the Secret needs no restart; nothing is cached. The constructor only checks that the files exist, to warn early without failing the provider set. +- **proxmox derives the VMID from the Proxy UID and checks ownership + before using a taken one.** Proxmox addresses VMs by VMID, and names are + not unique, so the name cannot be the key. A hash of the UID placed in + `vmidRange` makes a retry land on the same VMID; a taken one is waited + on while locked, then adopted when it has our name, mark and UID, + probed past when it has another name, and refused otherwise. The wait + is what stops a retry after a crash mid-clone from cloning a duplicate: + a clone in progress has no name yet, so the by-name lookup misses it. +- **proxmox writes its mark into the VM's description**, in the clone + call itself, so no VM of ours exists unmarked. Tags cannot be passed to + the clone, and the clone's own creation time is copied from the + template. The `proxy_operator` tag is set afterwards for the Proxmox UI + and to narrow GC's candidates; the mark decides. +- **proxmox delivers cloud-init as a seed ISO uploaded through the API**, + not as Proxmox's own cloud-init drive or a `cicustom` snippet. The + upload endpoint does not accept snippets, and the operator has no shell + on the nodes; the seed code is kvm2's, extracted into + `internal/provider/nocloud`. Templates must therefore carry no Proxmox + cloud-init drive, and seeds of VMs destroyed outside the operator are a + documented GC blind spot. +- **proxmox resolves a VM's node on every call** from + `/cluster/resources`, and its providerID names none, because VMs migrate + between nodes. The cost is one extra request per operation; the gain is + that a migrated VM stays addressable and the listing, which includes the + guests of an unreachable node as `status: unknown`, never mistakes a + down node for a deleted VM. diff --git a/docs/plans-executions/2026-09-18-1502-proxmox-provider.md b/docs/plans-executions/2026-09-18-1502-proxmox-provider.md index 21e77f4..aca7c8d 100644 --- a/docs/plans-executions/2026-09-18-1502-proxmox-provider.md +++ b/docs/plans-executions/2026-09-18-1502-proxmox-provider.md @@ -13,7 +13,7 @@ Pairs with [docs/plans/2026-09-18-1502-proxmox-provider.md](../plans/2026-09-18- - [x] Phase 6 — Create - [x] Phase 7 — Delete, Checker, CRD comment - [x] Phase 8 — Contract and observability sweep, integration test, testing.md -- [ ] Phase 9 — Deployment and documentation +- [x] Phase 9 — Deployment and documentation - [ ] Phase 10 — Review, real-host validation, changelog, MR ## Phase 0 — Plan file, branch, host recon, fixtures @@ -1121,3 +1121,180 @@ Worth noting: `go test -short ./...` run of this phase. - `.claude/settings.json` stays out of the commit. Nothing in this phase talked to the host. + +## Phase 9 — Deployment and documentation + +Deployment: the optional `proxmox-token` Secret (keys `token` and `ca.pem`) +is mounted read-only at `/etc/proxy-operator/proxmox` in +`config/manager/manager.yaml`, next to `kvm-ssh` and commented the same way. +The Makefile gains `proxmox-token-secret`. Both providers configs gain a +commented `proxmox-home` entry after kvm2's. `config/samples/proxy_proxmox.yaml` +is listed in the samples kustomization. There is no RBAC change and no +`hack/providers-dev.yaml` entry. Docs: `docs/proxmox-provider.md` follows +kvm2's guide heading for heading and has no `—`. The rows and sections +named below went into `providers-existing.md`, `architecture.md`, the +README, `providers.md` and `CLAUDE.md`. `docs/testing.md` and the +integration Makefile target were done in Phase 8 and were not touched; the +guide's Testing section links to them. + +Where the spec and the plan's Phase 9 text disagree, the spec won. The URL +is `https://192.168.0.8:8006`, not 192.168.2.8, which is not in the node +certificate. The Secret has two keys, and the target takes +`PROXMOX_CA_FILE` as well as `PROXMOX_TOKEN_FILE`. + +Judgment calls: + +- **The target also checks both files' content.** The token file must be + exactly one non-blank line matching + `@!=`. That is the shape `readToken` + accepts: `TrimSpace`, then an anchored regexp. The check uses `grep -c` + and `grep -qx`, so the secret is never printed. The CA check mirrors + `kvm-ssh-secret`'s `PRIVATE KEY` check: the file must contain + `BEGIN CERTIFICATE`. With `insecureSkipTLSVerify` the guide says to + create the Secret by hand with the `token` key only. +- **The sample-entry test covers both files.** `commentedEntry` finds the + `# - name: proxmox-home` line and takes its indent. It keeps the lines + that continue at `# ` and strips `# ` at that indent. The same + helper then works for the ConfigMap, whose entry sits six spaces deep + inside a `|` block. So `TestLoadConfig_proxmoxSampleEntry` has a + `sample` subtest and a `configmap` subtest. It checks every field the + spec names, and that the optional fields stay commented out. +- **A second test goes beyond the plan:** + `TestLoadConfig_proxmoxSampleProxyMatchesSampleEntry`. The sample Proxy's + provider, zone, machine type and image must resolve in the sample + entry. kvm has exactly this bug as a known gap (`proxy_kvm.yaml` places + on `nas1`). +- **The sample entry also has `medium: {cores: 2, memoryMiB: 2048, + diskGiB: 10}`**, next to the `small` the spec asks for, so `diskGiB` + appears somewhere. The optional fields are commented out with a short + note each, because no live proxmox entry exists to explain them. The + kvm2 entry can rely on kvm's. +- **In the ConfigMap the proxmox entry is commented out**, although kvm2's + is live there. It needs a host and a token. +- **providers-existing.md needed more than the column and the section.** + The intro now says five providers and links the guide. In the shared + gaps, six sentences would otherwise be false, and each now names + proxmox where it applies: adoption verified in two of the five, Create + blocking in all three VM providers, no Terminated, the list policy, the + collector gap, and the "kvm2 does this" in proposed features. The new + section is §8, so the shared gaps and proposed features move to §9 and + §10. No doc links those sections by anchor (grep). +- **architecture.md §11 now says "All five providers do".** The sentence + counts Checkers, and proxmox has one. The four Decisions bullets are + entry 23's: the VMID from the UID with the ownership check, the mark in + the description, the seed uploaded through the API, and the node + resolved per call. +- **The guide lists the ten design limits first, then four more.** They + are the hibernated-VM latch, the one-request seed upload, the open-proxy + sample (kvm2's guide lists it too), and "not yet proven end to end" with + the three answers never captured. The Phase 6 to 8 guide items are + each in the section where a reader looks for them. The config write + replacing `agent` and `tags` is in *What lands on Proxmox*, and so are + the seed cleanup, `qm unlock` and the 403 that makes Delete Permanent. + The certificate rule is in *Cluster-side setup*. CA caching versus + token rotation is in both. +- **The guide resumes a hibernated VM with `qm start`, not + `qm resume`.** A VM suspended to disk resumes from its state file on + its next start; `qm resume` is for a VM paused in RAM. +- **The guide's `pvesm add dir proxy-seeds --path /var/lib/proxy-seeds + --content iso` is the standard form, not a recorded command.** Phase 0 + says only that the user created `proxy-seeds` as a `dir` storage with + ISO content; the path is illustrative. + +The token-file check, run without touching a cluster: `KUBECTL=echo` +prints only the apply stage, and `KUBECTL=true` runs nothing. Scratch +files covered one good line, two lines, a bare secret, blank lines around +a good line, a non-PEM CA and a missing file. The real files on this +workstation pass: + +```bash +make -n proxmox-token-secret +make -s proxmox-token-secret PROXMOX_TOKEN_FILE=$D/ PROXMOX_CA_FILE=$D/ca KUBECTL=echo +# good, ws: "apply -f -"; two, bad: "... is not one line of user@realm!tokenid=secret"; noca: "... holds no PEM certificate" +make -s proxmox-token-secret KUBECTL=true && echo "real files pass" # real files pass +``` + +The mutation check for the sample tests edited one line per run, +asserting a single match. It ran the two tests, restored the file from a +copy, and at the end `sha256sum -c` confirmed the three files. All five +mutations failed the tests: `vlanTag: 5000` and `isoStorag:` in the +sample, `192.168.2.8` and `type: kvm2` in the ConfigMap, and +`image: debian-12` in the sample Proxy. `type: kvm2` was rerun after lint +removed the test's own type comparison. It still fails, because +`LoadConfig` refuses a proxmox block under another type. + +The link sweep is a short script. It takes the docs as arguments, so it +can be rerun after any later edit. It skips fenced code and inline code +spans. For each relative link it checks that the file exists, and that +the `#anchor` matches a GitHub-style slug of a heading in the target file +(or the same file). Before the real run it was checked on a scratch copy +of the guide with five links added. It reported exactly the three bad +ones: a missing file, a missing anchor in `testing.md`, and a missing +same-file anchor. It checked 78 relative links in the six edited docs, +plus this log's, and found 0 broken: + +```bash +cat > doclinks.py <<'PY' +import os, re, sys +def slug(h, seen): + s = re.sub(r"[^\w\- ]", "", h.strip().lower()).replace(" ", "-") + n = seen.get(s, 0); seen[s] = n + 1 + return s if n == 0 else f"{s}-{n}" +def prose(path): + out, fence = [], False + for line in open(path, encoding="utf-8"): + if line.lstrip().startswith("```"): fence = not fence; continue + if not fence: out.append(line) + return out +def anchors(path): + seen = {} + return {slug(re.sub(r"`", "", l.lstrip("#").strip()), seen) for l in prose(path) if re.match(r"#{1,6} ", l)} +bad = 0 +for doc in sys.argv[1:]: + for n, line in enumerate(prose(doc), 1): + for target in re.findall(r"\]\(([^)\s]+)\)", re.sub(r"`[^`]*`", "", line)): + if re.match(r"[a-z]+:", target): continue + f, _, a = target.partition("#") + p = os.path.normpath(os.path.join(os.path.dirname(doc), f)) if f else doc + if not os.path.exists(p): print(f"{doc}: missing file {target}"); bad += 1 + elif a and p.endswith(".md") and a not in anchors(p): print(f"{doc}: missing anchor {target}"); bad += 1 +print(f"{bad} broken"); sys.exit(1 if bad else 0) +PY +python3 doclinks.py docs/proxmox-provider.md docs/providers-existing.md docs/architecture.md \ + README.md docs/providers.md CLAUDE.md docs/plans-executions/2026-09-18-1502-proxmox-provider.md # 0 broken +``` + +Verification: + +```bash +bin/kustomize build config/default >/dev/null && bin/kustomize build config/samples >/dev/null # ok +go test -count=1 -run 'TestLoadConfig_proxmoxSample' -v ./internal/provider/ # 3 PASS +grep -c '—' docs/proxmox-provider.md # 0 +go build ./... && go vet ./... && go vet -tags=integration ./... && go test -short -count=1 ./... # all ok +bin/golangci-lint run --disable staticcheck --max-same-issues 0 --max-issues-per-linter 0 ./internal/provider/ +git status --short config/ # the four edited files named above, plus proxy_proxmox.yaml +``` + +Worth noting: + +- Lint on `internal/provider` is at its baseline of three goconst hits. + The first version of the new test pushed it to five: its + `p.Type != "proxmox"` made `"proxmox"` a three-time literal, flagged in + both `config.go` and the test. That comparison added nothing, so it + went. `internal/provider/proxmox` was not touched. +- The `tmp` volume comment in `manager.yaml` still says only the kvm + provider stages seeds there. kvm2 and proxmox do too. It was already + stale for kvm2, so it was left out of this phase's diff. +- The architecture §5 block's right-hand bracket was aligned by + character, not byte (`→` is three bytes), so every line puts `│` in + column 83. +- Nothing in this phase talked to the Proxmox host or a cluster. + `.claude/settings.json` stays out of the commit. The parked kvm2 items + (S5.1, and the 200 ms `TestCheck_hostsCheckedConcurrently`) did not + bite: `go test -short ./...` passed. +- Next is Phase 10: the first real run of `TestProxmox_lifecycle` and + `TestProxmoxCheck_realHost` on pv1-1, with the env block in + `docs/testing.md`. The three answers never captured go with it, and the + guide's "Not yet proven end to end" gap and the README's matching + clause come out once they pass. Then the CHANGELOG entry after the user + confirms, and the MR. diff --git a/docs/providers-existing.md b/docs/providers-existing.md index 80cb104..4ea4851 100644 --- a/docs/providers-existing.md +++ b/docs/providers-existing.md @@ -1,17 +1,20 @@ # The existing providers This is the appendix to [provider-contract.md](provider-contract.md): how the -four providers that exist today (kubernetes, gcp, kvm, kvm2) meet the -contract, which profile each one chose on every axis, and what is still +five providers that exist today (kubernetes, gcp, kvm, kvm2, proxmox) meet +the contract, which profile each one chose on every axis, and what is still missing. It is reference material for whoever writes the next provider, not a user guide; the user guides are [kvm-provider.md](kvm-provider.md), [kvm2-provider.md](kvm2-provider.md), +[proxmox-provider.md](proxmox-provider.md), [gcp-vm-validation.md](gcp-vm-validation.md) and [gcp-in-specific-project.md](gcp-in-specific-project.md). The README's -"Implemented providers" table stays the short public summary. kvm2 is the -one provider built from the contract itself (clean-room, from -[kvm2-provider-spec.md](kvm2-provider-spec.md), without reading kvm's code), -so its column is also a record of what the contract produced. +"Implemented providers" table stays the short public summary. kvm2 and +proxmox are the two providers built from the contract itself (kvm2 +clean-room from [kvm2-provider-spec.md](kvm2-provider-spec.md) without +reading kvm's code, proxmox from +[proxmox-provider-spec.md](proxmox-provider-spec.md)), so their columns are +also a record of what the contract produced. The per-provider call flows are drawn in [architecture.md §5](architecture.md#5-what-provider-calls-do-in-the-outside-world), @@ -27,59 +30,59 @@ Code references are a file path plus a function or symbol name. One row per axis of contract §4, in the same order, with the profile name and the symbol that shows it. -| Axis | kubernetes | gcp | kvm | kvm2 | -|---|---|---|---|---| -| `readiness` | immediate: a Pod is scheduled within seconds (`stateFromPod`) | provisioning-window: minutes of boot (`mapState`) | provisioning-window: boot plus, on first use, an image download (`imageCache.ensure`) | provisioning-window: boot plus, on first use, an image download (`ensureImage`) | -| `create-return` | submit: returns after `client.Create` | submit: `Insert` is never awaited, `Operation.Wait` is not called | **blocking**: waits through `DomainCreate` (`ensureRunning`), and the first image fetch runs inside (`imageCache.fetch`) | **blocking**: waits through `DomainCreate` and a 15-second poll for running (`ensureRunning`, `waitRunning`), and the first image fetch runs inside (`fetchImage`) | -| `delete-return` | returns-early: `client.Delete`, Pod terminates on its own | returns-early: `Delete` submits an operation each time | waits: destroy, undefine and volume removal complete before return (`removeDomain`, `removeVolumes`) | waits: destroy, undefine and volume removal complete before return (`removeDomain`, `removeVolumes`) | -| `ip-source` | assigned-at-create: `PodIP` (`instanceFromPod`) | assigned-at-create: the external NAT IP of the first NIC (`toInstance`) | discovered: ARP table first, then the guest agent, first routable IPv4 (`address`, `firstIPv4`) | discovered: the guest agent first, then the ARP table, first routable IPv4 (`addressSources`, `address`, `firstIPv4`) | -| `gc-mark` | native-labels: Pod labels (`buildPod`) | native-labels: instance labels (`buildInsertRequest`) | written-metadata: a namespaced `` element written in `DomainDefineXML` (`metadata.go`) | written-metadata: an element in kvm2's own namespace written in `DomainDefineXML` (`metadataURI`, `newInstanceMetadata`) | -| `created-at` | backend-timestamp: `CreationTimestamp` | backend-timestamp: `CreationTimestamp`, parse error gives zero | written-at-create: `createdAt` in the metadata element (`newInstanceMetadata`) | written-at-create: `createdAt` in the metadata element, parse error gives zero (`newInstanceMetadata`, `createdAt`) | -| `list-partial` | n/a: one call, all namespaces | omit-unreachable: `ReturnPartialSuccess` on `AggregatedList` | fail-whole per host, one unreadable domain skipped (`ListByTag`, `managedInstance`) | fail-whole per host, one unreadable domain skipped unless ctx ended; `ID`, `UID` and `CreatedAt` from the name and the mark only, no state or address RPC (`ListByTag`) | -| `quota` | absent: a quota 403 is Permanent (`classify`) | detected: 429, and 403 with reason `quotaExceeded` or `rateLimitExceeded` (`classify`, `hasReason`) | absent (`classify`) | detected: a pool-space precheck before any volume is created (`checkPoolSpace`, `neededBytes`) | -| `not-found` | status-404: `apierrors.IsNotFound` | status-404: `isNotFound` | error-code: `ErrNoDomain`, `ErrNoStorageVol` (`hasCode`) | error-code: `ErrNoDomain`, `ErrNoStorageVol`, Transient inside Create (`classifyLibvirt`) | -| `primary-key` | name-is-key: `namespace/name` | name-is-key: `zones//instances/` | name-is-key: `hosts//domains/` | name-is-key: `hosts//domains/`, with a UID guard on adoption (`create`) | -| `provisioning-state` | native: `PodPending` | native: `PROVISIONING`, `STAGING`, `REPAIRING` | create-blocks: a domain is defined and started in one `Create`; `nostate` maps to Provisioning | create-blocks: defined, started and polled to running in one `Create`; `nostate` maps to Provisioning (`mapState`) | -| `async-ops` | none | task-handle, but the handle is discarded: an insert that fails after acceptance is never read | none | none | -| `secondary-resources` | none | cascade: `AutoDelete` on the boot disk (`buildInsertRequest`) | explicit: overlay and seed removed by name (`removeVolumes`); base image kept | explicit: overlay and seed removed by name (`removeVolumes`); base image kept | -| `credentials` | workload-identity: in-cluster config or kubeconfig (`NewWithTransportWrapper`) | environment: Application Default Credentials | mounted-files: SSH key and known_hosts from the optional `kvm-ssh` Secret, read lazily (`newLazySSH`) | mounted-files: the same `kvm-ssh` Secret, read on every dial and never cached (`loadCredentials`) | -| `client-kind` | vendor-sdk: controller-runtime `client.Client`, no seam interface | vendor-sdk: `cloud.google.com/go/compute` behind `instancesAPI` | rpc-over-ssh: `go-libvirt` over an SSH-forwarded unix socket, behind `libvirtAPI` (`conn.go`) | rpc-over-ssh: `go-libvirt` over an SSH-forwarded unix socket, behind the ctx-first `backend` seam (`backend.go`) | -| `timeouts` | ctx-only: no `rest.Config.Timeout` is set | ctx-only: SDK defaults, with SDK retries on Get and List | per-call for the dial only (`defaultHandshakeTimeout` in `dialSSH`); RPCs are ctx-only and a cancelled `call` closes the connection | per-call: a budget around the dial, every RPC, the uploads and the download (`defaultBudgets`, `call`); a cancelled or overlong call kills the pooled connection | -| `logging` | none | tiered: V(1) outcomes and `logAPIError`, V(2) detail, V(5) wire log (`wirelog.go`) | tiered: V(1) outcomes and one `libvirt call` line per RPC (`trace.go`), V(2) XML and addresses; no wire tier | tiered: V(1) outcomes and one `libvirt call` line per RPC (`observe.go`), V(2) XML and addresses; no wire tier | -| `child-spans` | transport-wrapper: `tracing.RestConfigWrapper()` on its own `rest.Config`, parent-gated | sdk-instrumentation: the SDK adds `otelhttp` itself; not parent-gated, but every call site has a parent | **none**: `trace.go` is a log decorator, not spans | span-decorator: `kvm2.` per RPC (`observe`), plus `kvm2.dial` (`libvirtDialer`) and `kvm2.download` (`fetchImage`) | -| `backend-test` | cluster-e2e: kind (`test/e2e`) | manual-runbook: [gcp-vm-validation.md](gcp-vm-validation.md) | integration-tag: `kvm_integration_test.go`, `make test-integration-kvm` | integration-tag: `kvm2_integration_test.go`, `make test-integration-kvm2` | +| Axis | kubernetes | gcp | kvm | kvm2 | proxmox | +|---|---|---|---|---|---| +| `readiness` | immediate: a Pod is scheduled within seconds (`stateFromPod`) | provisioning-window: minutes of boot (`mapState`) | provisioning-window: boot plus, on first use, an image download (`imageCache.ensure`) | provisioning-window: boot plus, on first use, an image download (`ensureImage`) | provisioning-window: boot plus the guest agent's install at first boot, about 50 s to an address on the test host (`agentAddress`) | +| `create-return` | submit: returns after `client.Create` | submit: `Insert` is never awaited, `Operation.Wait` is not called | **blocking**: waits through `DomainCreate` (`ensureRunning`), and the first image fetch runs inside (`imageCache.fetch`) | **blocking**: waits through `DomainCreate` and a 15-second poll for running (`ensureRunning`, `waitRunning`), and the first image fetch runs inside (`fetchImage`) | **blocking**: waits through the clone, upload, resize and start tasks and a 60-second poll for running (`provision`, `waitRunning`) | +| `delete-return` | returns-early: `client.Delete`, Pod terminates on its own | returns-early: `Delete` submits an operation each time | waits: destroy, undefine and volume removal complete before return (`removeDomain`, `removeVolumes`) | waits: destroy, undefine and volume removal complete before return (`removeDomain`, `removeVolumes`) | waits: the stop, destroy and seed delete tasks complete before return (`destroy`, `deleteSeed`) | +| `ip-source` | assigned-at-create: `PodIP` (`instanceFromPod`) | assigned-at-create: the external NAT IP of the first NIC (`toInstance`) | discovered: ARP table first, then the guest agent, first routable IPv4 (`address`, `firstIPv4`) | discovered: the guest agent first, then the ARP table, first routable IPv4 (`addressSources`, `address`, `firstIPv4`) | discovered: the guest agent only, first routable IPv4 in the agent's order (`agentAddress`, `firstIPv4`) | +| `gc-mark` | native-labels: Pod labels (`buildPod`) | native-labels: instance labels (`buildInsertRequest`) | written-metadata: a namespaced `` element written in `DomainDefineXML` (`metadata.go`) | written-metadata: an element in kvm2's own namespace written in `DomainDefineXML` (`metadataURI`, `newInstanceMetadata`) | written-metadata: three `key=value` lines in the VM's description, passed in the clone call itself (`formatDescription`, `parseDescription`) | +| `created-at` | backend-timestamp: `CreationTimestamp` | backend-timestamp: `CreationTimestamp`, parse error gives zero | written-at-create: `createdAt` in the metadata element (`newInstanceMetadata`) | written-at-create: `createdAt` in the metadata element, parse error gives zero (`newInstanceMetadata`, `createdAt`) | written-at-create: `proxy-operator-created-at` in the description, parse error gives zero (`parseDescription`) | +| `list-partial` | n/a: one call, all namespaces | omit-unreachable: `ReturnPartialSuccess` on `AggregatedList` | fail-whole per host, one unreadable domain skipped (`ListByTag`, `managedInstance`) | fail-whole per host, one unreadable domain skipped unless ctx ended; `ID`, `UID` and `CreatedAt` from the name and the mark only, no state or address RPC (`ListByTag`) | fail-whole for the one cluster-wide `/cluster/resources` call, one unreadable VM skipped unless ctx ended; `ID`, `UID` and `CreatedAt` from the description only, no status or agent call (`ListByTag`) | +| `quota` | absent: a quota 403 is Permanent (`classify`) | detected: 429, and 403 with reason `quotaExceeded` or `rateLimitExceeded` (`classify`, `hasReason`) | absent (`classify`) | detected: a pool-space precheck before any volume is created (`checkPoolSpace`, `neededBytes`) | absent: thin storage overprovisions, so no out-of-space answer exists to match; free space goes to the Checker's details (`classify`) | +| `not-found` | status-404: `apierrors.IsNotFound` | status-404: `isNotFound` | error-code: `ErrNoDomain`, `ErrNoStorageVol` (`hasCode`) | error-code: `ErrNoDomain`, `ErrNoStorageVol`, Transient inside Create (`classifyLibvirt`) | error-code: the 500 `Configuration file '…' does not exist`, plus list-and-miss on `/cluster/resources`; Transient inside Create (`isNotExist`, `classifyAPI`) | +| `primary-key` | name-is-key: `namespace/name` | name-is-key: `zones//instances/` | name-is-key: `hosts//domains/` | name-is-key: `hosts//domains/`, with a UID guard on adoption (`create`) | **allocated-id**: `vms//`, the VMID from an FNV-1a hash of the UID with forward probing, each taken VMID put through an ownership check with a UID guard (`vmidFor`, `allocate`, `inspect`) | +| `provisioning-state` | native: `PodPending` | native: `PROVISIONING`, `STAGING`, `REPAIRING` | create-blocks: a domain is defined and started in one `Create`; `nostate` maps to Provisioning | create-blocks: defined, started and polled to running in one `Create`; `nostate` maps to Provisioning (`mapState`) | create-blocks: cloned, configured, started and polled to running in one `Create`; `qmpstatus: prelaunch` maps to Provisioning (`mapState`) | +| `async-ops` | none | task-handle, but the handle is discarded: an insert that fails after acceptance is never read | none | none | task-handle and backend-lock: every clone, upload, resize, start, stop, destroy and volume delete returns a UPID polled to its end within the call (`waitTask`); a VM's lock is waited on in Create and Transient in Get and Delete (`waitUnlocked`) | +| `secondary-resources` | none | cascade: `AutoDelete` on the boot disk (`buildInsertRequest`) | explicit: overlay and seed removed by name (`removeVolumes`); base image kept | explicit: overlay and seed removed by name (`removeVolumes`); base image kept | explicit: the seed ISO deleted by name (`deleteSeed`, `deleteSeedEverywhere`); the disks go with the destroy's `purge=1&destroy-unreferenced-disks=1` | +| `credentials` | workload-identity: in-cluster config or kubeconfig (`NewWithTransportWrapper`) | environment: Application Default Credentials | mounted-files: SSH key and known_hosts from the optional `kvm-ssh` Secret, read lazily (`newLazySSH`) | mounted-files: the same `kvm-ssh` Secret, read on every dial and never cached (`loadCredentials`) | mounted-files: the token from the optional `proxmox-token` Secret, read on every request (`readToken`); the CA from the same Secret, loaded once and cached (`httpClient`) | +| `client-kind` | vendor-sdk: controller-runtime `client.Client`, no seam interface | vendor-sdk: `cloud.google.com/go/compute` behind `instancesAPI` | rpc-over-ssh: `go-libvirt` over an SSH-forwarded unix socket, behind `libvirtAPI` (`conn.go`) | rpc-over-ssh: `go-libvirt` over an SSH-forwarded unix socket, behind the ctx-first `backend` seam (`backend.go`) | thin-client: `net/http` and `encoding/json` behind the ctx-first `api` seam, one method per endpoint (`api.go`, `httpAPI`) | +| `timeouts` | ctx-only: no `rest.Config.Timeout` is set | ctx-only: SDK defaults, with SDK retries on Get and List | per-call for the dial only (`defaultHandshakeTimeout` in `dialSSH`); RPCs are ctx-only and a cancelled `call` closes the connection | per-call: a budget around the dial, every RPC, the uploads and the download (`defaultBudgets`, `call`); a cancelled or overlong call kills the pooled connection | per-call: dial and TLS handshake 10 s, each request `requestTimeoutSeconds`, a budget per task wait (`defaultBudgets`); a cancelled request ends only itself | +| `logging` | none | tiered: V(1) outcomes and `logAPIError`, V(2) detail, V(5) wire log (`wirelog.go`) | tiered: V(1) outcomes and one `libvirt call` line per RPC (`trace.go`), V(2) XML and addresses; no wire tier | tiered: V(1) outcomes and one `libvirt call` line per RPC (`observe.go`), V(2) XML and addresses; no wire tier | tiered: V(1) outcomes and one `proxmox api call` line per request (`observe.go`), V(2) parameters, config summaries and addresses; no wire tier | +| `child-spans` | transport-wrapper: `tracing.RestConfigWrapper()` on its own `rest.Config`, parent-gated | sdk-instrumentation: the SDK adds `otelhttp` itself; not parent-gated, but every call site has a parent | **none**: `trace.go` is a log decorator, not spans | span-decorator: `kvm2.` per RPC (`observe`), plus `kvm2.dial` (`libvirtDialer`) and `kvm2.download` (`fetchImage`) | transport-wrapper: `tracing.RestConfigWrapper()` on the client's transport, parent-gated, one `HTTP ` span per request (`newHTTPAPI`) | +| `backend-test` | cluster-e2e: kind (`test/e2e`) | manual-runbook: [gcp-vm-validation.md](gcp-vm-validation.md) | integration-tag: `kvm_integration_test.go`, `make test-integration-kvm` | integration-tag: `kvm2_integration_test.go`, `make test-integration-kvm2` | integration-tag: `proxmox_integration_test.go`, `make test-integration-proxmox`; compiled, not yet run against a host | ## 2. Other differences Rows that are not profile axes but that a new provider author compares against. -| Feature | kubernetes | gcp | kvm | kvm2 | -|---|---|---|---|---| -| Create verifies the adopted instance's UID | no | no | no | **yes**: a domain without the kvm2 mark or with a different non-empty UID is Permanent; a marked one with a matching or empty UID is adopted (`create`) | -| Placement used | none | Zone, MachineType, Image | Zone (host), MachineType, Image | Zone (host), MachineType, Image | -| CloudInit delivered | **no**: ignored | yes: `user-data` metadata | yes: NoCloud seed ISO built in-process (`buildSeedISO`) | yes: NoCloud seed ISO built in-process (`nocloud.BuildSeedISO`) | -| Port used | yes: Squid config and containerPort (`squidConf`) | no: firewall is manual | no | no | -| States produced | Provisioning, Running, Terminated | all four | Provisioning, Running, Stopped | Provisioning, Running, Stopped | -| Deleting instance still reports its state | yes: a Pod being deleted reports its phase | n/a | n/a | n/a: Delete waits, and a concurrent Get sees the domain until it is undefined | -| Error.Provider is the configured name | **no**: hardcoded `"kubernetes"` | yes | yes | yes (`wrapErr`) | -| Checker target | `apiserver` | the project | each host, concurrently | each host, concurrently, on a fresh connection | -| Checker Details | none | none | `user`, `pool`, `hostname`, `libvirtVersion`, `poolState`, `poolCapacity`, `poolAvailable` | `user`, `pool`, `hostname`, `libvirtVersion`, `poolState`, `poolCapacityGiB`, `poolAvailableGiB`, filled in as far as the check got | -| Credential rotation without restart | n/a | via ADC | no: cached after first successful load | yes: both files read on every dial (`loadCredentials`) | -| Connection handling | client-go | SDK client, never closed | one pooled SSH plus libvirt connection per host, 15s keepalive (`hostPool`) | one pooled SSH plus libvirt connection per host, 15s keepalive (`hostTable`); dedicated connections for the image upload and the check (`dialFresh`) | -| Unit-test double | controller-runtime fake client | `fakeAPI` behind `instancesAPI`, plus `httptest` for the real SDK client in `check_test.go` | `fakeAPI` behind `libvirtAPI`, plus an in-process `x/crypto/ssh` server in `conn_test.go` | `fakeBackend` behind `backend` (`fake_test.go`), plus an in-process `x/crypto/ssh` server (`sshserver_test.go`) and go-libvirt's own mock for the adapter | -| Proxy software | generated Squid config (`pod.go`) | from `cloudInit`; the image must run cloud-init | from `cloudInit`, plus a vendor-data script that installs the guest agent (`seed.go`) | from `cloudInit`, plus a vendor-data script that installs the guest agent (`nocloud.GuestAgentScript`) | -| IP reachable from the operator | Pod network, so in-cluster only (not from `make run-dev` on a laptop) | external IP, once a firewall rule allows the cluster's egress IP | LAN address on the host's bridge; the operator needs a route to that LAN | LAN address on the host's bridge; the operator needs a route to that LAN | -| Distinct egress IP per proxy | no: the cluster's egress | yes: ephemeral external IP | whatever the host's LAN routes through; not per VM by itself | whatever the host's LAN routes through; not per VM by itself | +| Feature | kubernetes | gcp | kvm | kvm2 | proxmox | +|---|---|---|---|---|---| +| Create verifies the adopted instance's UID | no | no | no | **yes**: a domain without the kvm2 mark or with a different non-empty UID is Permanent; a marked one with a matching or empty UID is adopted (`create`) | **yes**: a VM with the proxy's name and no mark, or with a different non-empty UID, is Permanent; a marked one with a matching or empty UID is adopted (`inspect`) | +| Placement used | none | Zone, MachineType, Image | Zone (host), MachineType, Image | Zone (host), MachineType, Image | Zone (node), MachineType, Image (a template key) | +| CloudInit delivered | **no**: ignored | yes: `user-data` metadata | yes: NoCloud seed ISO built in-process (`buildSeedISO`) | yes: NoCloud seed ISO built in-process (`nocloud.BuildSeedISO`) | yes: NoCloud seed ISO built in-process (`nocloud.BuildSeedISO`) and uploaded through the API (`uploadSeed`) | +| Port used | yes: Squid config and containerPort (`squidConf`) | no: firewall is manual | no | no | no | +| States produced | Provisioning, Running, Terminated | all four | Provisioning, Running, Stopped | Provisioning, Running, Stopped | Provisioning, Running, Stopped | +| Deleting instance still reports its state | yes: a Pod being deleted reports its phase | n/a | n/a | n/a: Delete waits, and a concurrent Get sees the domain until it is undefined | n/a: Delete waits, and a concurrent Get sees the VM (Stopped once stopped) until the destroy removes its config | +| Error.Provider is the configured name | **no**: hardcoded `"kubernetes"` | yes | yes | yes (`wrapErr`) | yes (`wrapErr`) | +| Checker target | `apiserver` | the project | each host, concurrently | each host, concurrently, on a fresh connection | each node, concurrently, on the shared HTTP client | +| Checker Details | none | none | `user`, `pool`, `hostname`, `libvirtVersion`, `poolState`, `poolCapacity`, `poolAvailable` | `user`, `pool`, `hostname`, `libvirtVersion`, `poolState`, `poolCapacityGiB`, `poolAvailableGiB`, filled in as far as the check got | `isoStorage`, `tokenID`, `pveVersion`, `kernel`, `isoStorageAvailGiB`, `templates`, filled in as far as the check got | +| Credential rotation without restart | n/a | via ADC | no: cached after first successful load | yes: both files read on every dial (`loadCredentials`) | the token yes, read on every request (`readToken`); the CA no, cached after first use | +| Connection handling | client-go | SDK client, never closed | one pooled SSH plus libvirt connection per host, 15s keepalive (`hostPool`) | one pooled SSH plus libvirt connection per host, 15s keepalive (`hostTable`); dedicated connections for the image upload and the check (`dialFresh`) | one `net/http` client, 4 idle connections per host, HTTP/1.1 only; a cancelled request ends only itself | +| Unit-test double | controller-runtime fake client | `fakeAPI` behind `instancesAPI`, plus `httptest` for the real SDK client in `check_test.go` | `fakeAPI` behind `libvirtAPI`, plus an in-process `x/crypto/ssh` server in `conn_test.go` | `fakeBackend` behind `backend` (`fake_test.go`), plus an in-process `x/crypto/ssh` server (`sshserver_test.go`) and go-libvirt's own mock for the adapter | `fakeAPI` behind `api` (`fake_test.go`), plus `httptest.NewTLSServer` replaying the Phase 0 captures for the real client (`client_test.go`) | +| Proxy software | generated Squid config (`pod.go`) | from `cloudInit`; the image must run cloud-init | from `cloudInit`, plus a vendor-data script that installs the guest agent (`seed.go`) | from `cloudInit`, plus a vendor-data script that installs the guest agent (`nocloud.GuestAgentScript`) | from `cloudInit`, plus a vendor-data script that installs the guest agent (`nocloud.GuestAgentScript`) | +| IP reachable from the operator | Pod network, so in-cluster only (not from `make run-dev` on a laptop) | external IP, once a firewall rule allows the cluster's egress IP | LAN address on the host's bridge; the operator needs a route to that LAN | LAN address on the host's bridge; the operator needs a route to that LAN | address on the proxy network (bridge or VLAN from the config); the operator needs a route to it | +| Distinct egress IP per proxy | no: the cluster's egress | yes: ephemeral external IP | whatever the host's LAN routes through; not per VM by itself | whatever the host's LAN routes through; not per VM by itself | whatever the proxy network routes through; not per VM by itself | ## 3. Observability status -| Layer | kubernetes | gcp | kvm | kvm2 | -|---|---|---|---|---| -| 1. Operation spans and metrics | yes: decorators | yes: decorators | yes: decorators | yes: decorators | -| 2. Child spans per backend call | yes: `tracing.RestConfigWrapper()`, parent-gated (`cmd/main.go`, `NewWithTransportWrapper`) | yes: the SDK's own `otelhttp`; not parent-gated | **no** | yes: `kvm2.` from the `observe` decorator, `kvm2.dial`, `kvm2.download`; always under the caller's span, since every call carries the caller's ctx | -| 3. Logs carry `traceID` and `spanID` | n/a: the package does not log | yes, including the V(5) wire log | yes: every `libvirt call` line at V(1) | yes: every `libvirt call` line at V(1), written inside its RPC's span | -| Tested against a real collector | yes: the kind e2e checks `provider.create` and `provider.delete` in Tempo (`test/e2e/tracing_test.go`), operation spans only | no | no | no; the integration test asserts the parent-child topology in an in-memory recorder (`assertLifecycleSpans`) | +| Layer | kubernetes | gcp | kvm | kvm2 | proxmox | +|---|---|---|---|---|---| +| 1. Operation spans and metrics | yes: decorators | yes: decorators | yes: decorators | yes: decorators | yes: decorators | +| 2. Child spans per backend call | yes: `tracing.RestConfigWrapper()`, parent-gated (`cmd/main.go`, `NewWithTransportWrapper`) | yes: the SDK's own `otelhttp`; not parent-gated | **no** | yes: `kvm2.` from the `observe` decorator, `kvm2.dial`, `kvm2.download`; always under the caller's span, since every call carries the caller's ctx | yes: `tracing.RestConfigWrapper()` on the transport, parent-gated, one `HTTP ` per request | +| 3. Logs carry `traceID` and `spanID` | n/a: the package does not log | yes, including the V(5) wire log | yes: every `libvirt call` line at V(1) | yes: every `libvirt call` line at V(1), written inside its RPC's span | yes: every `proxmox api call` line at V(1), with the operation span's `spanID` | +| Tested against a real collector | yes: the kind e2e checks `provider.create` and `provider.delete` in Tempo (`test/e2e/tracing_test.go`), operation spans only | no | no | no; the integration test asserts the parent-child topology in an in-memory recorder (`assertLifecycleSpans`) | no; `TestTracing_httpSpansUnderOperationSpan` asserts the parent-child topology in an in-memory recorder | ## 4. kubernetes (`internal/provider/kubernetes`) @@ -528,20 +531,209 @@ same libvirt. its backing image or size. - The sample cloud-init installs Squid with `http_access allow all`. -## 8. Gaps shared by the existing providers +## 8. proxmox (`internal/provider/proxmox`) -- **Adoption is not verified in three of the four.** kubernetes, gcp and +The full user guide is [proxmox-provider.md](proxmox-provider.md) and the +design it was built from is +[proxmox-provider-spec.md](proxmox-provider-spec.md); this section is the +author's view. It is the second provider built from the contract, and the +first on a REST API of the hypervisor's own: no SSH, no libvirt. Every +error text it matches was captured from a PVE 9.2.20 host before the code +was written. + +- **Package and seam.** `proxmox.go` (`Provider`, `New`, `withDefaults`, + `node`, `wrapErr`, the per-name lock, the budgets), `create.go`, + `get.go`, `delete.go`, `list.go` (the four methods), `api.go` (the + seam), `client.go` (the production adapter `httpAPI`, the token and CA + loading), `observe.go` (the logging decorator), `task.go` (task, lock + and running waits), `seed.go`, `mark.go`, `id.go`, `state.go`, + `errors.go`, `check.go`; the seed ISO comes from the shared + `internal/provider/nocloud` and the name lock from + `internal/provider/keyedlock`. The seam is `api`: one method per + endpoint, sixteen in all, each with `ctx` first. `New` wraps `httpAPI` in + `observe` once, so provider code never holds an undecorated seam; + `newProvider` is the test constructor. +- **Config.** `ProviderConfig.Proxmox`, the `ProxmoxConfig` type in + `internal/provider/config_proxmox.go`, validated by `validateProxmox` + (the URL shape, node names and storage ids by regexp because they end + up in request paths, `vlanTag`, `vmidRange`, the two catalogues). + `isoStorage` has no default on purpose: the token needs + `Datastore.Allocate` there. Defaults (`vmidRange` 100000 to 199999, + `requestTimeoutSeconds` 30, `installGuestAgent` true) are filled by + `withDefaults`. `New` never makes a request; it warns when the token + file does not exist, or when `insecureSkipTLSVerify` is on. +- **ID.** `formatProviderID(vmid, name)` gives `vms//`, with no + node, because VMs migrate; the node is resolved from + `/cluster/resources` on every call (`resolve`). `parseProviderID` + accepts exactly three segments, the literal `vms`, a decimal VMID + without leading zeros between 100 and 999999999 and a non-empty name, + else `ErrPermanent`. The name is there so `Get` and `Delete` can tell + our VM from another that later got the same VMID. +- **Create.** `placement` checks `Zone` (`node`; optional with one node), + `MachineType` and `Image` (both catalogue keys) before any request, and + logs the resolved parameters with `cloudInitBytes` at V(2). Then, under + the per-name lock, `create`: one `ClusterResources` read; + `adoptByName` (two VMs with the name are Permanent, one goes to + `inspect`); with none, `template` reads the template's config and + refuses a non-template, a cloud-init drive, a missing `net0` when the + config rewrites it, and an unsized boot disk when `diskGiB` is set, all + before any clone; `allocate` probes `vmidFor(uid, attempt)` up to 32 + times, cloning onto a free VMID and running `inspect` on a taken one or + on the clone's `config file already exists` answer. `inspect` waits out + a lock (`waitUnlocked`; a clone in progress is also nameless), then + adopts our name with our mark and a matching or empty UID, probes past + another name, and is Permanent for our name without the mark, with + another UID, or hibernated. An adopted VM gets one `VMStatus`: running + returns at once. Otherwise `provision`: `uploadSeed` (rebuilt and + overwritten every attempt), `SetVMConfig` (`configParams`: cores, + memory, `agent=1`, the tag, `ide2`, and `net0` through `rewriteNet0` + when a bridge or VLAN is configured), the resize task when + `resizeTarget` says the boot disk is smaller than `diskGiB`, the start + task, and `waitRunning` (every second up to 60 s, Transient on expiry + or on a stop). `Create` returns only when the VM runs. +- **Get and state.** `get` checks in order: absent from the resources is + NotFound; a lock other than `suspended` is Transient (checked before the + name, because a mid-clone entry has none); `status: unknown` is + Transient; the config's `does not exist` is NotFound, a config lock + Transient; another name or no mark is NotFound; hibernated is Stopped + without further calls. Then `VMStatus` and `mapState(status, + qmpstatus, ip)`: + + | `status` | `qmpstatus` | agent | Provider state | + |---|---|---|---| + | running | running | an IPv4 | Running | + | running | running | 200 without an IPv4, or `QEMU guest agent is not running` | Provisioning | + | running | running | `No QEMU guest agent configured` | Stopped | + | running | prelaunch | not asked | Provisioning | + | running | paused, suspended, io-error, anything else | not asked | Stopped | + | stopped, anything else | any | not asked | Stopped | + + Any other agent answer, including `VM is not running`, is + Transient. `UID` and `CreatedAt` are read from the description on every + call. A running `Get` is four calls: resources, config, status, agent. +- **Delete.** Under the per-name lock, `destroy`: `resolve`; a lock other + than `suspended` or `status: unknown` is Transient; the config's `does + not exist`, another name or no mark leave the VM alone; our VM + hibernated is Permanent, because the destroy refuses it and the lock + never clears. Otherwise stop (unless stopped) and destroy with + `purge=1&destroy-unreferenced-disks=1`, each a task waited on through + `runTask`, where `does not exist` means gone. Then `deleteSeed` on the + VM's node, where a failure fails the call (a 403 for + `Datastore.Allocate` is Permanent), or, with no VM of ours destroyed, + `deleteSeedEverywhere` over every configured node, where a node that + fails is logged and skipped unless ctx ended. A missing volume still + ends in a successful task, so no message is matched. Nothing is left + when nil is returned. +- **ListByTag.** One `ClusterResources` covers every node, unreachable + ones included (their guests are listed with `status: unknown`). + `isCandidate` keeps QEMU VMs that are not templates and carry the tag + `proxy_operator` or a name starting `proxy-`; a nameless entry (a clone + in progress) is not a candidate. One `VMConfig` per candidate; the mark + in the description decides, and the `Instance` is `ID`, `UID` and + `CreatedAt` only. A candidate whose config cannot be read is skipped at + V(1), unless ctx ended, which fails the call; the resources call failing + fails the whole call. +- **Errors.** `classify(op, err)`: an error already carrying a class keeps + it; then `apiError`s by `classifyAPI` (every 4xx Permanent; the 500 + `does not exist` NotFound, Transient when `op == "create"`; every other + status Transient, including the lock, the taken VMID and pveproxy's 595 + and 596); TLS verification failures (`tls.CertificateVerificationError` + and the three `x509` types) Permanent; everything else Transient. The + 500 matchers (`isNotExist`, `isLocked`, `isAlreadyExists`, the three + agent ones) are anchored to the 500 status and each is backed by a + captured response. `permanentf` and `transientf` classify at the + source: placement, the template checks, the ownership check, a + malformed token file or CA file (Permanent); a missing token or CA file, + an expired wait, a failed task (Transient). Nothing is Quota. +- **Check.** `Check` runs `checkNode` per node concurrently on the shared + client. `probeNode` reads the token file for `tokenID` (no request when + it is unusable), then `Version`, `NodeStatus`, `checkStorage` (a 500 + whatever its text, inactive, or no `iso` content is `StorageNotFound`) + and `checkTemplates` (each catalogue VMID exists and is a template, + else `TemplateNotFound`). `checkNode` answers `Timeout` whenever the + round's ctx ended; otherwise `checkReason`: the two failures above, + `errCredentials` `CredentialsUnavailable`, `errCAFile` or a TLS + verification failure `CertificateInvalid`, 401 `Unauthorized`, 403 + `Forbidden`, 595 and 596 `Unreachable`, any other status `APIError`, a + `net.OpError` `Unreachable` (before the deadline, so a dial out of its + own budget is a node that did not answer), a deadline or cancellation + `Timeout`, a cut connection `Unreachable`. +- **Connections.** One `http.Client`, built on first use once the CA + loads (`httpClient`), shared by every call and the Checker: dial and + TLS handshake 10 s each, `Client.Timeout` `requestTimeoutSeconds`, 4 + idle connections per host kept 90 s, HTTP/2 disabled. A cancelled + request ends only itself. The token file is read by `readToken` on + every request. +- **Logging.** Logger from the context, named `proxmox`, with `provider`, + and `vmid`, `name` and `node` on lines about an instance. V(1): one line + per operation outcome (`proxmox vm ensured running`, `proxmox vm + fetched`, `proxmox vm deleted` with `destroyed`, `proxmox vms listed`, + `proxmox node checked`), one `proxmox api call` line per seam call from + `observe` (call, identifiers, UPID, duration, `httpStatus` and `error`), + `proxmox task failed`, and the no-op lines. V(2): request parameters, + config summaries, agent addresses, the resolved Create parameters. + Cloud-init, the token, the CA and the upload body are never logged. +- **Tracing.** No span of its own: `newHTTPAPI` wraps the transport in + `tracing.RestConfigWrapper(opts...)`, so each request is an + `HTTP ` client span under the caller's span, and none without + one. `observe` only logs, so nothing is counted twice. +- **Tests.** Pure pieces (`id_test.go`, `mark_test.go`, `state_test.go`, + `seed_test.go`, `errors_test.go` with the classify table over the + captures, `api_test.go`); `client_test.go` (the real client over + `httptest.NewTLSServer`: the token never in errors or logs, x509 + failures Permanent, cancellation, the child span); `create_test.go`, + `get_test.go`, `delete_test.go`, `list_test.go`, `task_test.go`, + `proxmox_test.go`, `check_test.go`, `observe_test.go` (every seam + method logs exactly one line, by reflection) and `provider_test.go` + (the contract §11 sweep, named test by test in the spec's entry 21). + The double is `fakeAPI` in `fake_test.go`: VMs, tasks and ISOs in + memory, a clone that leaves a nameless locked entry until its task + ends, per-method errors and blocking, a call log; its error texts are + checked against the captures. Real backend: + `proxmox_integration_test.go` behind `//go:build integration`, driven + by `PROXMOX_TEST_*` variables and `make test-integration-proxmox`, + described in + [testing.md](testing.md#the-proxmox-integration-test-real-proxmox-ve-host); + it compiles but has not run against a host yet. +- **Deployment.** The optional `proxmox-token` Secret (keys `token` and + `ca.pem`) mounted at `/etc/proxy-operator/proxmox` + (`config/manager/manager.yaml`), `make proxmox-token-secret`, a + commented `proxmox-home` entry in both providers configs (loaded by + `TestLoadConfig_proxmoxSampleEntry`), `config/samples/proxy_proxmox.yaml`, + the `proxmox` type in the `status.type` comment of + `api/v1alpha1/providerhealth_types.go`. No RBAC, and no + `hack/providers-dev.yaml` entry, since it needs credentials. +- **Known gaps** (see also [proxmox-provider.md § Known gaps](proxmox-provider.md#known-gaps)). + - `Create` blocks until the VM runs. + - A clone lock that never clears makes every Create for that Proxy wait + out the 10-minute clone budget; `qm unlock` by hand. + - GC blind spot: a seed ISO whose VM was destroyed outside the operator + is invisible to GC, and a seed on node-local storage stays behind + when its VM migrated. + - The VMID range is shared with humans; a hand-made VM with a proxy's + name is Permanent. + - Cross-node clones (`target`) need shared storage and are untested. + - A hibernated VM of ours latches its Proxy in Create and in Delete. + - The CA is cached after first use; the token is not. + - Never run against a real host yet; three answers were never captured + (the storage status for a missing storage, a destroy on a hibernated + VM, a stop on a paused VM). + +## 9. Gaps shared by the existing providers + +- **Adoption is not verified in three of the five.** kubernetes, gcp and kvm adopt any existing instance with the right name without checking - `LabelUID`; kvm2 returns `ErrPermanent` when the UID differs. -- **The Create comment does not match kvm and kvm2.** The interface comment - in `internal/provider/provider.go` says `Create` returns as soon as the - request is submitted; both libvirt providers block until the domain has - started. + `LabelUID`; kvm2 and proxmox return `ErrPermanent` when the UID differs. +- **The Create comment does not match kvm, kvm2 and proxmox.** The + interface comment in `internal/provider/provider.go` says `Create` + returns as soon as the request is submitted; the three VM providers on + your own hypervisors block until the VM has started. - **`Stopped` and `Terminated` are produced inconsistently.** kubernetes - never produces Stopped, kvm and kvm2 never produce Terminated, and the - reconciler treats them the same. + never produces Stopped, kvm, kvm2 and proxmox never produce Terminated, + and the reconciler treats them the same. - **Different `ListByTag` partial-failure policies.** gcp leaves failed - parts out, kvm and kvm2 fail the whole list. Both are safe. + parts out, kvm and kvm2 fail the whole list, and proxmox fails it when + its one cluster-wide listing fails. Both are safe. - **Stale `mock` comments** in `internal/provider/provider.go`, `internal/provider/registry/registry.go` and `api/v1alpha1/proxy_types.go` (and so the generated CRD's `spec.provider` description). @@ -555,10 +747,10 @@ same libvirt. log lines. kvm2 has them, but only an in-memory recorder checks their topology. Only kubernetes is tested against a real collector, and that test checks only the operation spans; nothing checks that gcp's SDK spans, - the kubernetes API spans or kvm2's spans nest under `provider.*` in a - real collector. + the kubernetes API spans, kvm2's spans or proxmox's HTTP spans nest under + `provider.*` in a real collector. -## 9. Proposed features +## 10. Proposed features Listed only. Nothing below is designed or implemented, except where a bullet says kvm2 already does it. @@ -576,7 +768,7 @@ bullet says kvm2 already does it. backend, so a new provider gets the contract tests for free. - **Verify the UID when `Create` adopts an instance.** Return `ErrPermanent` on a mismatch instead of adopting an instance that belongs to someone else. - kvm2 does this; the other three do not. + kvm2 and proxmox do this; the other three do not. - **Report asynchronous create failures:** gcp keeps the operation name, and `Get` reports the failure instead of NotFound. - **Capacity or quota signals** for kvm, such as pool free space and host diff --git a/docs/providers.md b/docs/providers.md index c8431d9..8a96803 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -14,12 +14,13 @@ the existing ones happen to work. open. Copy it to `docs/-provider-spec.md`, fill it in, and hand it to the implementer with the other two files. - **[providers-existing.md](providers-existing.md)**: how kubernetes, gcp, - kvm and kvm2 meet the contract, a table mapping each of them to every + kvm, kvm2 and proxmox meet the contract, a table mapping each of them to every profile, their per-provider reference notes, shared gaps and proposed features. Per-provider user guides: [kvm-provider.md](kvm-provider.md), [kvm2-provider.md](kvm2-provider.md), +[proxmox-provider.md](proxmox-provider.md), [gcp-vm-validation.md](gcp-vm-validation.md) and [gcp-in-specific-project.md](gcp-in-specific-project.md). diff --git a/docs/proxmox-provider.md b/docs/proxmox-provider.md new file mode 100644 index 0000000..e152118 --- /dev/null +++ b/docs/proxmox-provider.md @@ -0,0 +1,646 @@ +# The proxmox provider + +Runs Managed proxies as VMs on a Proxmox VE cluster. It is the third +VM-style provider and the second built from +[provider-contract.md](provider-contract.md), from the design in +[proxmox-provider-spec.md](proxmox-provider-spec.md). The operator talks to +the cluster's REST API only (`https://:8006/api2/json`) with an API +token. It needs no SSH, no shell on the nodes, and no library beyond Go's +standard `net/http`. + +Each VM is a **full clone of a template** you prepare once. The operator +builds the Proxy's cloud-init into a NoCloud seed ISO in-process, uploads +it to a storage on the VM's node, attaches it as a CD-ROM, sets the size, +starts the VM and learns its address from the QEMU guest agent. Everything +here was proven against **Proxmox VE 9.2.20** (QEMU 11.0) on the single +node `pv1-1`; every error text the provider matches was captured from that +host. + +## Proxmox prerequisites + +Once per cluster, as root on a node: + +1. **Proxmox VE 9.x with the API reachable** from the operator's pods on + port 8006. The operator's pods and the crawler clients must also route to + the network the proxies land on (see step 5). +2. **A template without a Proxmox cloud-init drive.** The provider brings + its own seed on `ide2`; a Proxmox cloud-init drive would compete with it + and the Proxy's user-data would be ignored. Debian 13's generic cloud + image boots UEFI (under BIOS it hangs in GRUB on a headless VM), so the + template does too: + + ```sh + wget https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2 + qm create 9001 --name debian-13-template --memory 2048 --cores 2 --ostype l26 \ + --machine q35 --bios ovmf --efidisk0 local-lvm:1,efitype=4m,pre-enrolled-keys=0 \ + --scsihw virtio-scsi-pci --net0 virtio,bridge=vmbr0 \ + --serial0 socket --vga serial0 --agent 1 + qm set 9001 --scsi0 local-lvm:0,import-from=/root/debian-13-generic-amd64.qcow2 + qm set 9001 --boot order=scsi0 + qm template 9001 + ``` + + The template must boot from its first disk, run cloud-init with the + NoCloud datasource (the Debian and Ubuntu cloud images do), and either + ship `qemu-guest-agent` or be able to install it at first boot. Before + the first clone for a Proxy, `Create` reads the template's config and + refuses it with a permanent error when: + + - it is not a template (`qm template` was not run); + - any disk slot holds a Proxmox cloud-init drive (a value containing + `cloudinit`), such as the one `qm set --ide2 local-lvm:cloudinit` + adds for Proxmox's own cloud-init support; + - it has no `net0` while the config sets `bridge` or `vlanTag`; + - the machine type sets `diskGiB` and the template has no boot disk, or + one without a readable `size=`. + + These are checked on the template because the clone copies its config: + a VM cloned from a bad template could not be fixed by fixing the + template. The template is checked only when a VM is about to be cloned; + a VM that already exists no longer depends on it. The health check sees + only whether each template exists and is a template. +3. **A storage for the seed ISOs**, holding `iso` content. A dedicated + directory storage is best, because the token needs `Datastore.Allocate` + on it to delete seeds, and on a dedicated storage that privilege cannot + touch backups or other ISOs: + + ```sh + pvesm add dir proxy-seeds --path /var/lib/proxy-seeds --content iso + ``` + + A seed is tens of KiB. Proxmox stages uploads under `/var/tmp`, not the + node's tmpfs `/tmp`. On a cluster, a storage that is not shared keeps + each seed on the node the VM was created on (see *Known gaps*). +4. **A user, two roles and a token.** `VM.Monitor` no longer exists in PVE + 9 and `pveum` rejects it; the agent read needs `VM.GuestAgent.Audit` + alone. `VM.Config.Cloudinit` is not needed for the seed on `ide2`. + `SDN.Use` on `/` is what lets the config write put a VLAN tag on + `vmbr0`: + + ```sh + pveum user add proxy-operator@pve --comment "egress-proxies-operator" + pveum role add ProxyOperator -privs "VM.Allocate VM.Clone VM.Audit VM.Config.CDROM VM.Config.CPU VM.Config.Memory VM.Config.Disk VM.Config.Network VM.Config.Options VM.PowerMgmt VM.GuestAgent.Audit Datastore.Audit Datastore.AllocateSpace Datastore.AllocateTemplate Sys.Audit SDN.Use" + pveum aclmod / -user proxy-operator@pve -role ProxyOperator + pveum role add ProxyOperatorSeeds -privs "Datastore.Allocate Datastore.AllocateTemplate Datastore.AllocateSpace Datastore.Audit" + pveum aclmod /storage/proxy-seeds -user proxy-operator@pve -role ProxyOperatorSeeds + pveum user token add proxy-operator@pve operator --privsep 0 + ``` + + The second role repeats three privileges of the first on purpose: an + ACL on a more specific path **replaces** what the user inherits from + `/` for that path, so a role with `Datastore.Allocate` alone would take + the upload's `Datastore.AllocateTemplate` away. The role on `/` covers + clone, config, resize, start, stop, the agent read, the upload and the + destroy; what it lacks is `Datastore.Allocate`, without which deleting + a seed is a 403 (`Permission check failed (/storage/, + Datastore.Allocate)`). With `--privsep 0` the token carries the user's + privileges. + + The token's secret is printed once. Save it on your workstation as one + line, `proxy-operator@pve!operator=`, in + `~/.config/proxmox-token` with mode 0600. Copy the node's CA, + `/etc/pve/pve-root-ca.pem`, to `~/.config/proxmox-ca.pem`. +5. **A proxy network with DHCP and a route to the internet.** Each VM + keeps the template's one NIC, optionally moved to another bridge or + VLAN by the config, and takes its address from that network's DHCP. The + network needs internet twice over: the vendor-data script installs + `qemu-guest-agent` from the distro mirrors at first boot, and the agent + is the only way the operator learns the VM's address; and an egress + proxy needs egress anyway. Baking `qemu-guest-agent` into the template + and setting `installGuestAgent: false` removes the mirror from IP + discovery. The operator's pods and the clients must route to that + network: the health engine probes `http://:` through the + proxy, and clients get the same address from the discovery API. The + provider does not use `spec.port` and configures no Proxmox firewall. +6. **A reserved VMID range.** VMIDs are allocated from `vmidRange` + (default 100000 to 199999). Keep hand-made VMs out of it: one there + costs a probe, and one named like a proxy is refused (see *What lands + on Proxmox*). + +Check from the operator's point of view before wiring it in. This sends +the token through a file descriptor, so it never reaches the command line: + +```sh +pve() { + curl -s --cacert ~/.config/proxmox-ca.pem \ + -H @<(printf 'Authorization: PVEAPIToken=%s\n' "$(tr -d '[:space:]' < ~/.config/proxmox-token)") \ + "https://192.168.0.8:8006/api2/json/$1"; echo +} +pve version # the token works, TLS verifies +pve nodes/pv1-1/storage/proxy-seeds/status # active, content includes iso +pve 'cluster/resources?type=vm' # the template is listed with "template":1 +``` + +## Cluster-side setup + +1. Create the Secret. It holds two keys, `token` (the one-line token file) + and `ca.pem` (the node's CA): + + ```sh + make proxmox-token-secret + # or by hand: + kubectl -n egress-proxies-operator-system create secret generic proxmox-token \ + --from-file=token=$HOME/.config/proxmox-token \ + --from-file=ca.pem=$HOME/.config/proxmox-ca.pem + ``` + + The make target runs that command as an apply, so it also updates an + existing Secret. Override the paths with + `PROXMOX_TOKEN_FILE=... PROXMOX_CA_FILE=...`. Before it applies, it + checks that both files exist, that the token file is exactly one line + of `@!=` (without printing it), and that + the CA file holds a PEM certificate. With `insecureSkipTLSVerify` + instead of a CA, create the Secret by hand with the `token` key only. + + `config/manager/manager.yaml` mounts the Secret (optional, read-only) + at `/etc/proxy-operator/proxmox`, so the files are + `/etc/proxy-operator/proxmox/token` and + `/etc/proxy-operator/proxmox/ca.pem`. The deployment starts fine + without it, even with a proxmox provider configured: `New` makes no + request, only checks that the token file exists, and logs a `WARNING:` + line when it does not. The provider's proxies then fail transiently and + recover once the Secret lands; the health check reports + `CredentialsUnavailable` meanwhile. + + **Rotating the token needs no restart.** The token file is read on + every request, so the next request after the kubelet refreshes the + mount uses the new token. **Rotating the CA does**: it is loaded on + first use and cached for the life of the process. A missing CA file is + retried until it lands; one without a certificate is a permanent error. + +2. Add a provider entry to the providers config + (`config/manager/providers_config.yaml` in-cluster; the full sample, + commented out, is in `config/samples/providers-config.yaml`): + + ```yaml + - name: proxmox-home + type: proxmox + proxmox: + url: https://192.168.0.8:8006 + tokenFile: /etc/proxy-operator/proxmox/token + caFile: /etc/proxy-operator/proxmox/ca.pem + nodes: [pv1-1] + isoStorage: proxy-seeds + vlanTag: 4 + templates: + debian-13: {vmid: 9001} + machineTypes: + small: {cores: 1, memoryMiB: 1024} + ``` + + **The URL's host must be in the node certificate.** The node's + self-signed certificate lists its own addresses and names, and TLS + verification checks the URL's host against them. On `pv1-1` it lists + `192.168.0.8`, `pv1-1` and `pv1-1.home.lab`, so + `https://192.168.2.8:8006` reaches the same node and still fails + verification (`CertificateInvalid` in the health check, a permanent + error on every call). See what yours lists with: + + ```sh + openssl s_client -connect 192.168.0.8:8006 /dev/null | openssl x509 -noout -ext subjectAltName + ``` + + If the address the operator must use is not there, either give the + node a certificate that names it (a custom certificate in the node's + Certificates panel) and put that certificate's CA in `caFile`, or set + `insecureSkipTLSVerify: true`, which `New` logs as a `WARNING:` line. + + `nodes` holds node names as Proxmox knows them, not addresses: the one + URL reaches the whole cluster. The providers config is read once per + process, so adding a node, a template or a machine type needs a manager + restart. + +3. Apply a Proxy (`config/samples/proxy_proxmox.yaml`) and watch it: + + ```sh + kubectl get px -w + # NAME MODE PROVIDER PHASE IP HEALTHY + # proxy-proxmox-sample Managed proxmox-home Provisioning + # proxy-proxmox-sample Managed proxmox-home Running 192.168.4.147 + # proxy-proxmox-sample Managed proxmox-home Ready 192.168.4.147 True + ``` + + (Illustrative.) `Create` returns once the VM runs, a few seconds after + the clone; the address follows when the guest agent is up, about 50 + seconds after the start on `pv1-1` with the agent installed at first + boot. + +## Config reference + +The block is `proxmox:` and its Go type is `ProxmoxConfig` in +`internal/provider/config_proxmox.go`, validated at load by +`validateProxmox`; unknown keys are rejected. Defaults are applied by +`withDefaults` in `internal/provider/proxmox/proxmox.go`. + +| Field | Required | Default | Meaning | +|---|---|---|---| +| `url` | yes | | `https://:8006`, with no path (the client appends `/api2/json`). The host must be in the node certificate | +| `tokenFile` | yes | | file holding one line, `@!=`, read on every request and sent as `Authorization: PVEAPIToken=` | +| `caFile` | | system roots | PEM bundle trusted for the API's certificate, typically the node's `/etc/pve/pve-root-ca.pem`. Loaded on first use and cached | +| `insecureSkipTLSVerify` | | `false` | skip certificate verification; `New` logs a `WARNING:` line when it is on | +| `nodes` | yes, at least one | | node names `placement.zone` selects from; unique, lowercase letters, digits and dashes. They end up in request paths and as the clone's `target` | +| `isoStorage` | yes | | storage id the seeds are uploaded to and deleted from. Must hold `iso` content; the token needs `Datastore.Allocate` on it. No default on purpose | +| `storage` | | the template's disk storage | storage id the full clone's disks go to | +| `bridge` | | as cloned | replaces `bridge=` in the cloned `net0` | +| `vlanTag` | | as cloned | 1 to 4094; replaces or appends `tag=` in the cloned `net0` | +| `vmidRange.min`, `vmidRange.max` | | `100000`, `199999` | where VMIDs are allocated; at least 100, at most 999999999, min below max | +| `templates..vmid` | at least one template | | the key names the template in `placement.image`; the VMID is at least 100 | +| `machineTypes.` | at least one | | `{cores, memoryMiB, diskGiB}` size profile. `cores` and `memoryMiB` above zero. `diskGiB` is optional: when larger than the cloned boot disk the disk is grown to it; it is never shrunk | +| `requestTimeoutSeconds` | | `30` | bound on one API request, response included. The seed upload is one request, so a multi-megabyte `vendorData` needs more | +| `vendorData` | | | extra cloud-init vendor-data placed on every seed next to the Proxy's user-data; a MIME multipart part beside the agent script when `installGuestAgent` is on. Cloud-init drops any top-level key of a vendor-data cloud-config that the user-data also defines | +| `installGuestAgent` | | `true` | add the vendor-data script that installs and starts `qemu-guest-agent`. Turn it off for templates that ship the agent | + +## Placement + +All three catalogue fields are checked before any request. An unknown +value is `ErrPermanent`, which the reconciler latches as `Failed` with the +message below until the spec changes. + +| Proxy field | Meaning for proxmox | Error when unknown | +|---|---|---| +| `placement.zone` | a node from `nodes`. May be omitted when exactly one node is configured; otherwise required | `placement.zone "x" is not in proxmox.nodes`; with several nodes and no zone, `placement.zone is required when proxmox.nodes lists more than one node` | +| `placement.machineType` | a key in `machineTypes` | `placement.machineType "x" is not in proxmox.machineTypes` | +| `placement.image` | a key in `templates` | `placement.image "x" is not in proxmox.templates` | +| `placement.region` | ignored. It is still part of the spec hash, so changing it replaces the VM and does nothing else | | + +The messages come from `placement` in `internal/provider/proxmox/create.go` +and `node` in `proxmox.go`. A key that exists but names a VMID that is +missing or not a template fails later, before any clone: +`placement.image "x": template VM 9001 does not exist` or `... is not a +template`. + +**Catalogue edits affect only new VMs.** The spec hash covers the +placement values, not what the config maps them to, so repointing +`templates.debian-13.vmid` at another template changes the VMs created +afterwards and leaves the running ones alone. Add a new key rather than +repointing one in use, so the Proxies that name a key keep a meaning you +can reason about. A template may be edited, replaced or deleted without +affecting existing VMs: they are full clones and share nothing with it. + +## What lands on Proxmox + +For a Proxy whose deterministic name is `proxy-abc…`: + +| Object | Name | Lifetime | +|---|---|---| +| VM | name `proxy-abc…`, a VMID from `vmidRange` | with the Proxy | +| its disks | the template's disks, fully copied onto `storage` (or the template's storage) | destroyed with the VM (`purge=1&destroy-unreferenced-disks=1`) | +| seed ISO | `:iso/proxy-abc…-seed.iso` on the VM's node | with the Proxy | + +**The VMID** is derived from the Proxy's UID, so a retried Create lands on +the same number: an FNV-1a hash of the UID placed in `vmidRange`, moving +forward one at a time on collision, up to 32 probes (`vmidFor` in +`id.go`). Before allocating, `Create` looks the name up in the cluster's +resources. A VMID that is taken, or a VM that already has the name, goes +through an ownership check (`inspect` in `create.go`): a VM with a lock +(a clone still running, which also has no name yet) is waited on; then + +- another name: someone else's VM, so the next VMID is probed; +- our name and our mark with our UID or an empty one: **adopted**, and + Create carries on from where the previous attempt stopped; +- our name without the mark, or with another UID: **permanent error** + naming the VMID and both UIDs. The operator never takes over a VM it did + not create; +- our name, our mark, hibernated (`lock: suspended`): **permanent error**, + `resume or destroy it in Proxmox` (see *Known gaps*). + +Two VMs with the proxy's name are a permanent error naming every VMID; +remove all but one. + +**The providerID** is `vms//`. It names no node, because VMs +migrate: the node is looked up in `GET /cluster/resources` on every call. +`Get` and `Delete` check that the VM at that VMID still has that name and +the mark. Another name means ours is gone and the VMID was reused: `Get` +answers NotFound, `Delete` succeeds without touching it. + +**The mark** is three lines in the VM's description (Notes in the UI), +passed in the clone call itself, so a VM is never unmarked: + +``` +proxy-operator-managed=true +proxy-operator-uid=0f3a… +proxy-operator-created-at=2026-09-23T17:42:11Z +``` + +That is how orphan GC recognises its own VMs and how a repeat `Create` +knows whether a VM of the same name belongs to this Proxy. Lines an admin +adds to the description are kept and ignored. The clone's own +`meta: ctime=` is useless for this: Proxmox copies it from the template. + +**The config write.** After the clone and the seed upload, one +`PUT /nodes/{node}/qemu/{vmid}/config` sets: + +| Key | Value | +|---|---| +| `cores`, `memory` | from the machine type | +| `agent` | `1` | +| `tags` | `proxy_operator` | +| `ide2` | `:iso/-seed.iso,media=cdrom` | +| `net0` | only when `bridge` or `vlanTag` is set: the cloned value with `bridge=` and `tag=` replaced or appended, the model, MAC and every other option kept | + +The write **replaces the template's `agent` option and tags**: a +template's `agent: 1,fstrim_cloned_disks=1` becomes `agent: 1`, and its +tags give way to `proxy_operator`. The tag is for filtering in the +Proxmox UI and narrows GC's candidates; the mark decides. Proxmox appends +`ide2` to the boot order on its own (`order=scsi0;ide2`), which is +harmless because the boot disk stays first. When the machine type's +`diskGiB` is larger than the boot disk (the first disk in `boot: order=` +that is not a CD-ROM), a resize task grows it; cloud-init's growpart +expands the root filesystem at first boot. + +**The seed** (`nocloud.BuildSeedISO`, shared with kvm2 in +`internal/provider/nocloud`) is an ISO 9660 image labelled `cidata`. +`user-data` is the Proxy's `cloudInit` **byte for byte**; the operator +never parses or re-renders it. `meta-data` holds `instance-id: ` and +`local-hostname: `. `vendor-data` carries the guest-agent script +(a shell script, not a cloud-config, so a Proxy's own `packages:` cannot +shadow it), joined with `vendorData` as a MIME multipart when both are +set. It is staged under `/tmp` (the manager's emptyDir), built in memory +and uploaded with `POST /nodes/{node}/storage/{isoStorage}/upload`. An +upload overwrites a file of the same name, so it is rebuilt and +re-uploaded on every Create attempt with no delete first: a previous +attempt may have been cut off. Only the API is used; the storage upload +endpoint does not accept `snippets`, so Proxmox's own `cicustom` would +need shell access to the node. + +**What a retried Create finds.** Every step is safe to repeat. After a +crash mid-clone, the retry lands on the same VMID, waits for the clone's +lock, and adopts the VM. A stopped VM of ours gets the seed, the config +and the resize again (the upload overwrites, the config is a replace, the +resize is skipped once large enough), then the start. A running one is +returned at once. + +**A clone lock that never clears.** A node that crashes mid-clone can +leave the VM locked with `lock: clone` for good. Every Create for that +Proxy then waits out the 10 minute clone budget and returns a transient +error, and the reconciler keeps retrying rather than latching. Clear it +on the node: + +```sh +qm unlock # then let the next Create adopt and finish it +qm destroy --purge # or remove it and let Create clone again +``` + +**Seeds without a VM.** A VM destroyed outside the operator leaves its +seed ISO, which GC cannot see (it lists VMs). Seeds are named +`proxy-*-seed.iso` on `isoStorage`; any without a matching VM may be +removed by hand: + +```sh +pvesm list proxy-seeds --content iso +pvesm free proxy-seeds:iso/proxy-abc…-seed.iso +``` + +**Delete** runs under a per-name lock shared with Create: stop (a paused +VM too), destroy with `purge=1&destroy-unreferenced-disks=1`, then delete +the seed on the VM's node. Each is a task that is waited on. A VM that is +already gone, not ours, or unmarked is left alone, and the seed is then +looked for on every configured node, where a node that fails is logged +and skipped. On the VM's own node a failed seed delete fails the call: +**without `Datastore.Allocate` on `isoStorage` the 403 makes Delete +permanent**, and the Proxy's deletion stays stuck with that message until +the privilege is granted. When Delete returns nil, nothing of the Proxy +is left. + +## Logging + +The provider logs through controller-runtime's logger, named `proxmox`, +with `provider` (the configured name) on every line, and `vmid`, `name` +and `node` on every line about an instance. Every line inside a traced +operation also carries `traceID` and `spanID`. + +- At startup (always on): `WARNING: token file is not readable; every + call fails until it is` with `path` and `reason`, and `WARNING: + insecureSkipTLSVerify is set`. +- `--zap-log-level=debug` (logr V(1)): + - one line per operation outcome: `proxmox vm ensured running`, + `proxmox vm adopted` with `status` and `qmpstatus`, + `proxmox vm fetched` with `state`, `ip`, `uid`, `status` and + `qmpstatus`, `proxmox vm not found` with `reason`, + `proxmox vm deleted` with `destroyed` (false when no VM of ours was + left to destroy), `proxmox vms listed` with `vms`, `candidates`, + `managed` and `skipped` counts, and `proxmox node checked` with + `ready`, `reason` and `latency`; + - **one `proxmox api call` line per API request**, written by the + decorator in `internal/provider/proxmox/observe.go`: `call` (the seam + method, for example `Clone`), the identifiers it was given (`node`, + `vmid`, `storage`), `upid` for a task, `newid` and `name` on a clone, + `filename` on an upload, `volid` on a volume delete, `duration`, and + on failure `httpStatus` and `error`. A running `Get` is four calls; a + fresh Create is a dozen or more, depending on how many polls each + task takes; + - `proxmox task failed` with `upid` and `exitstatus`, and + `proxmox task finished with warnings`; + - the no-op lines: `proxmox vm already gone`, + `proxmox vm at this vmid is not ours, left alone` with `vmName`, + `proxmox vm carries no proxy-operator mark, left alone`, + `proxmox seed delete failed on a node, skipped`, + `skipping unreadable proxmox vm`, and the probe's + `proxmox vmid taken by another vm, probing on`. +- `--zap-log-level=2`: `proxmox create resolved` with the resolved + parameters (node, machine type, cores, memory, disk, template VMID, + storages, bridge, VLAN tag) and `cloudInitBytes`; the `proxmox api + call` lines also carry the config write's parameters, a summary of each + config read, the clone's description, storage and target, the resize + size, VM and task status, the agent's addresses and the upload's byte + count; `proxmox guest agent answered`; `proxmox listed instance` per + listed VM. +- Never logged at any level: the cloud-init content, the token, the CA + content and the upload body. There is no raw wire tier. + +## Tracing + +With an OTLP exporter configured ([architecture.md +§10](architecture.md#10-tracing-internaltracing)), every API request is a +child span of the operation span the decorators open (`provider.create`, +`provider.get`, `provider.delete`, `provider.list`) or of the monitor's +`providerhealth.check`: + +| Span | Around | Attributes | +|---|---|---| +| `HTTP GET`, `HTTP POST`, `HTTP PUT`, `HTTP DELETE` | one API request, from `otelhttp` on the client's transport | otelhttp's standard HTTP client attributes: method, URL, server address and port, response status | + +The transport is `tracing.RestConfigWrapper()`, the same parent-gated +wrapper the kubernetes provider uses, so a request without a parent span +starts none. The logging decorator starts no span of its own, so nothing +is counted twice; its `proxmox api call` line carries the operation +span's `spanID`, not the HTTP span's. The token travels in a header, +which otelhttp does not record, and no attribute carries cloud-init. + +## Timeouts + +No caller puts a deadline on the context, so every request and every wait +carries its own budget (`defaultBudgets` in +`internal/provider/proxmox/proxmox.go`, and the transport in `client.go`): + +| Bound | Budget | +|---|---| +| TCP connect | 10 s | +| TLS handshake | 10 s | +| one request, response included | `requestTimeoutSeconds`, default 30 s | +| clone task, and the wait for a VM's lock in the ownership check | 10 m | +| upload task, resize task, start task, stop task | 2 m each | +| destroy task | 5 m | +| seed delete task | 1 m | +| wait for `running` after the start | 60 s | +| task and status poll interval | 1 s | + +A task wait that runs out is a transient error naming the UPID; the task +keeps running on Proxmox, and the next attempt finds its result through +the ownership check or the lock. A task that ends with any `exitstatus` +other than `OK` or `WARNINGS: ` is transient too, with Proxmox's text. + +Requests share one HTTP client and its idle connections (4 per host, +closed after 90 s idle). A cancelled or overlong request ends only +itself: nothing else in flight is disturbed, and there is no connection +to kill as there is for kvm2. HTTP/2 is disabled, because pveproxy speaks +HTTP/1.1 only. Create and Delete for the same name are serialised on a +per-name lock, so the reconciler's Delete and GC's Delete never race on +stop and destroy; Get and ListByTag take no lock. + +## Health check + +The operator checks every configured node every +`--provider-check-interval` (default 1m) and publishes the result in the +`ProviderHealth` object named after the provider +(`kubectl get ph proxmox-home -o yaml`) and in the +`proxy_operator_provider_target_up{provider,target}` gauge. One target per +`nodes` entry, named after the node with the URL as its address, in +config order, all checked concurrently. + +Each check (`checkNode` in `internal/provider/proxmox/check.go`) first +reads the token file for its id, then makes four requests on the shared +client: `GET /version`, `GET /nodes/{node}/status`, +`GET /nodes/{node}/storage/{isoStorage}/status` (the storage must exist, +be active and hold `iso` content), and `GET /cluster/resources?type=vm` +(every configured template must exist and be a template). A check that +runs out of time cancels only its own requests. + +Every target's `details` carry `isoStorage`. As facts arrive they gain +`tokenID` (the part of the token before `=`, never the secret), +`pveVersion`, `kernel`, `isoStorageAvailGiB` and `templates` +(`/`), so a target that fails late still says what the +node answered before. There is no quota signal; the storage's free space +is here instead. + +| Reason | Meaning | Look at | +|---|---|---| +| `Reachable` | ready | | +| `CredentialsUnavailable` | the token file is missing or not `@!=`; no request was made | the `proxmox-token` Secret; fixed in the cluster, not in Proxmox | +| `CertificateInvalid` | TLS verification failed, or `caFile` is missing or holds no certificate | `url` (its host must be in the node certificate) or `caFile`; not the node | +| `Unreachable` | the TCP connect or TLS handshake failed for a network reason, the connection was cut mid-response, or pveproxy answered 595 or 596 (it cannot reach the node behind it) | routing and firewalls between the cluster and `url`; `pveproxy` running; the node up | +| `Unauthorized` | 401: the token id or secret is wrong | the token file; `pveum user token list proxy-operator@pve` | +| `Forbidden` | 403; the message names the path and the privilege | the roles in *Proxmox prerequisites* | +| `Timeout` | the monitor's `--provider-check-timeout` (default 15s) ran out, whichever request was running, or a request ran out of `requestTimeoutSeconds` while the round still had time | a slow or hung pveproxy, a saturated link | +| `StorageNotFound` | `isoStorage` answered with a 500 on that node (it does not exist, is not enabled there, or is offline), is not active, or holds no `iso` content; the message carries Proxmox's text | `pvesm status`; the storage's nodes and content types | +| `TemplateNotFound` | a configured template VMID does not exist or is not a template | `qm list`; the `templates` catalogue | +| `APIError` | any other failure | the message carries Proxmox's own text | + +The check does not look inside the templates: a cloud-init drive, a +missing `net0` or an unreadable disk size surface only when a Create is +about to clone. + +## Error classes + +Every error is classified once, in `classify` in +`internal/provider/proxmox/errors.go`, with errors that carry a class from +their source (validation, the token and CA files, the ownership check) +keeping it. The reconciler latches `ErrPermanent` as `Failed` until the +spec changes, recreates on `ErrNotFound` from `Get`, and retries +everything else with backoff. The provider never returns +`ErrQuotaExceeded`: the thin `local-lvm` storage overprovisions, so no +out-of-space answer could be produced to match, and capacity failures are +transient. + +| Class | What lands there | +|---|---| +| `ErrPermanent` | placement values not in the catalogue; a template VMID that is missing or not a template, has a cloud-init drive, or lacks `net0` or a sized boot disk when the config needs them; a malformed providerID; any 4xx (401 `Authentication failed!` or `No ticket`: the token; 403 `Permission check failed (, )`: a missing privilege, including `Datastore.Allocate` for the seed delete; 400 `Parameter verification failed.` with the field errors); a TLS verification failure; a token file that exists but is not `@!=`; a CA file with no certificate; a VM with the proxy's name that carries no mark or another UID; two VMs with the proxy's name; our VM hibernated (`lock: suspended`), in both Create and Delete | +| `ErrNotFound` | from `Get`: the VMID absent from the cluster's resources, the 500 `Configuration file 'nodes//qemu-server/.conf' does not exist`, the VM at the VMID now named differently or unmarked. Inside `Create` the `does not exist` answer is `ErrTransient`, because the contract forbids NotFound from Create; `Delete` treats all of these as done | +| `ErrTransient` | everything else: a missing token or CA file (the Secret has not landed yet); network errors and cancellation; 500 `VM is locked ()` and any lock seen in the resources or config; a VM whose node does not answer (`status: unknown`); the agent's answers that `Get` cannot place, such as `VM is not running`; any other 5xx, including pveproxy's 595 and 596; a task that fails or runs out of its budget; no free VMID after 32 probes; the wait for running expiring, or a VM that stops right after its start | + +`Get` turns two agent answers into states rather than errors: +`QEMU guest agent is not running` is Provisioning (booting, or the agent +install has not finished), and `No QEMU guest agent configured` is +Stopped (someone removed `agent: 1`; the reconciler replaces the VM). + +Error strings name VMIDs, nodes, storages, UPIDs and Proxmox's own text. +They never contain the token, the CA content or cloud-init. + +## Known gaps + +- **Create blocks** until the VM runs, holding one of the three reconcile + workers for its duration. On `pv1-1` that is about five seconds plus the + clone; large templates on slow storage take longer. Other Proxies keep + reconciling on the remaining workers. +- **A clone lock that never clears** (a node crash mid-clone) makes every + Create for that Proxy wait out the 10 minute clone budget and return a + transient error. Clear it with `qm unlock ` or destroy the VM (see + *What lands on Proxmox*). +- **Seed ISOs of VMs destroyed outside the operator are invisible to + GC.** Remove `proxy-*-seed.iso` files without a matching VM by hand. + On a storage that is not shared, a VM migrated to another node leaves + its seed on the original node; Delete then looks on the VM's current + node, finds nothing, and the seed stays. Shared ISO storage avoids it. +- **The VMID range is shared with humans.** A VM created by hand inside + it costs a probe; a hand-made VM named like a proxy is refused as + `ErrPermanent`. Reserve the range. +- **Cross-node clones need shared storage and are untested.** When the + zone's node differs from the template's, the clone is sent with + `target`, which Proxmox allows only when the template's disks are on + shared storage. The host this was built on has one node. +- **A VM whose agent was switched off is replaced**, not repaired: with + `agent: 1` removed it can never report an address, so `Get` reports it + Stopped. +- **The proxy network needs internet** for the agent install, unless the + template ships the agent and `installGuestAgent` is `false`. Without it + the VM boots, gets an address, and stays `Provisioning` forever, + because the agent is the only address source. Check + `systemctl status qemu-guest-agent` and + `/var/log/cloud-init-output.log` on the guest (`qm terminal ` on + the node, or the console in the UI). +- **The CA is cached** after first use; rotating it needs a manager + restart. The token rotates freely. +- **A node that stops answering** makes `Get` and `Delete` for its VMs + transient until it returns, so deleting such a Proxy waits. Proxmox + lists an unreachable node's guests with `status: unknown` rather than + dropping them, so they are never mistaken for gone. That is Proxmox's + documented behaviour; a single-node host could not show it. +- **Contract limits that apply unchanged** ([provider-contract.md + §13](provider-contract.md)): the providers config is read once per + process; GC has no installation identity, so two operators sharing a + Proxmox cluster and a VMID range delete each other's VMs; and a Proxy + naming an unconfigured provider latches until its spec changes. + +Beyond the ten limits accepted in the design: + +- **A hibernated VM latches its Proxy.** A VM of ours with + `lock: suspended` (hibernated to disk) makes both Create and Delete fail + permanently with `resume or destroy it in Proxmox`: the lock never + clears on its own, and the destroy refuses a locked VM. `Get` reports it + Stopped. Resume it by starting it (`qm start ` restores the + hibernated state) or destroy it on the node, then touch the Proxy's + spec, or let the pending Delete retry. +- **The seed upload is one request**, bounded by `requestTimeoutSeconds`. + A seed is tens of KiB, but a multi-megabyte `vendorData` would need a + larger timeout. +- **The sample cloud-init is an open proxy.** + `config/samples/proxy_proxmox.yaml` installs Squid with + `http_access allow all`; restrict it before exposing the proxy network + to anything but the cluster. +- **Not yet proven end to end.** The unit tests run over captured + responses from `pv1-1` and an in-memory fake; the integration tests + below compile but have not run against a host yet. Three answers were + never captured: the storage status for a missing storage, a destroy on + a hibernated VM, and a stop on a paused VM. + +## Testing + +Unit tests run over an in-memory fake of the API seam, and the HTTP +client over a TLS test server replaying the responses captured from +`pv1-1` (`internal/provider/proxmox/testdata/`). The optional integration +test against a real host is described in +[testing.md](testing.md#the-proxmox-integration-test-real-proxmox-ve-host), +which also lists the variables it needs; run it with +`make test-integration-proxmox`. `TestProxmoxCheck_realHost` runs the +health check against the same host and creates nothing: +`go test -tags=integration -count=1 -v -run TestProxmoxCheck ./internal/provider/proxmox/`. diff --git a/internal/provider/config_proxmox_test.go b/internal/provider/config_proxmox_test.go index d623928..8585fdd 100644 --- a/internal/provider/config_proxmox_test.go +++ b/internal/provider/config_proxmox_test.go @@ -1,8 +1,12 @@ package provider import ( + "os" + "slices" "strings" "testing" + + "sigs.k8s.io/yaml" ) const validProxmoxYAML = ` @@ -130,3 +134,117 @@ machineTypes: {small: {cores: 1, memoryMiB: 1024}} }) } } + +// commentedEntry cuts the commented-out providers entry named name out of a +// config file and strips its "# " at the entry's own indent, so a sample +// that is shipped commented out is still loaded by a test. +func commentedEntry(t *testing.T, path, name string) string { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("reading %s: %v", path, err) + } + var out []string + var prefix string + for line := range strings.SplitSeq(string(data), "\n") { + if prefix == "" { + if strings.TrimSpace(line) == "# - name: "+name { + prefix = line[:strings.Index(line, "#")] + out = append(out, prefix+"- name: "+name) + } + continue + } + if !strings.HasPrefix(line, prefix+"# ") { + break + } + out = append(out, prefix+strings.TrimPrefix(line, prefix+"# ")) + } + if prefix == "" { + t.Fatalf("%s has no commented entry %q", path, name) + } + return strings.Join(out, "\n") + "\n" +} + +func TestLoadConfig_proxmoxSampleEntry(t *testing.T) { + t.Parallel() + for name, path := range map[string]string{ + "sample": "../../config/samples/providers-config.yaml", + "configmap": "../../config/manager/providers_config.yaml", + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + cfg, err := LoadConfig([]byte("providers:\n" + commentedEntry(t, path, "proxmox-home"))) + if err != nil { + t.Fatalf("LoadConfig(proxmox-home from %s) error = %v, want nil", path, err) + } + p := cfg.Providers[0] + // LoadConfig already refused a proxmox block under another type. + if p.Name != "proxmox-home" || p.Proxmox == nil { + t.Fatalf("provider %q has proxmox block %v, want proxmox-home with one", p.Name, p.Proxmox) + } + c := p.Proxmox + if c.URL != "https://192.168.0.8:8006" { + t.Errorf("url = %q, want https://192.168.0.8:8006 (the address in the node certificate)", c.URL) + } + if c.TokenFile != "/etc/proxy-operator/proxmox/token" || c.CAFile != "/etc/proxy-operator/proxmox/ca.pem" { + t.Errorf("tokenFile, caFile = %q, %q, want the proxmox-token Secret's mount paths", c.TokenFile, c.CAFile) + } + if c.InsecureSkipTLSVerify { + t.Error("insecureSkipTLSVerify = true, want it left commented out") + } + if len(c.Nodes) != 1 || c.Nodes[0] != "pv1-1" { + t.Errorf("nodes = %v, want [pv1-1]", c.Nodes) + } + if c.ISOStorage != "proxy-seeds" || c.Storage != "" || c.Bridge != "" || c.VLANTag != 4 { + t.Errorf("isoStorage, storage, bridge, vlanTag = %q, %q, %q, %d, want proxy-seeds, \"\", \"\", 4", c.ISOStorage, c.Storage, c.Bridge, c.VLANTag) + } + if len(c.Templates) != 1 || c.Templates["debian-13"].VMID != 9001 { + t.Errorf("templates = %v, want {debian-13: {vmid: 9001}}", c.Templates) + } + if mt, ok := c.MachineTypes["small"]; !ok || mt.Cores != 1 || mt.MemoryMiB != 1024 || mt.DiskGiB != 0 { + t.Errorf("machineTypes[small] = %+v (present %v), want {1 1024 0}", mt, ok) + } + if c.VMIDRange != nil || c.RequestTimeoutSeconds != 0 || c.InstallGuestAgent != nil || c.VendorData != "" { + t.Errorf("optional fields set: %+v, want vmidRange, requestTimeoutSeconds, installGuestAgent and vendorData left commented out", c) + } + }) + } +} + +func TestLoadConfig_proxmoxSampleProxyMatchesSampleEntry(t *testing.T) { + t.Parallel() + cfg, err := LoadConfig([]byte("providers:\n" + commentedEntry(t, "../../config/samples/providers-config.yaml", "proxmox-home"))) + if err != nil { + t.Fatalf("LoadConfig() error = %v, want nil", err) + } + data, err := os.ReadFile("../../config/samples/proxy_proxmox.yaml") + if err != nil { + t.Fatalf("reading the sample Proxy: %v", err) + } + var px struct { + Spec struct { + Provider string `json:"provider"` + Placement struct { + Zone string `json:"zone"` + MachineType string `json:"machineType"` + Image string `json:"image"` + } `json:"placement"` + } `json:"spec"` + } + if err := yaml.Unmarshal(data, &px); err != nil { + t.Fatalf("parsing the sample Proxy: %v", err) + } + c := cfg.Providers[0].Proxmox + if px.Spec.Provider != cfg.Providers[0].Name { + t.Errorf("spec.provider = %q, want %q", px.Spec.Provider, cfg.Providers[0].Name) + } + if !slices.Contains(c.Nodes, px.Spec.Placement.Zone) { + t.Errorf("placement.zone = %q, not in nodes %v", px.Spec.Placement.Zone, c.Nodes) + } + if _, ok := c.MachineTypes[px.Spec.Placement.MachineType]; !ok { + t.Errorf("placement.machineType = %q, not in machineTypes", px.Spec.Placement.MachineType) + } + if _, ok := c.Templates[px.Spec.Placement.Image]; !ok { + t.Errorf("placement.image = %q, not in templates", px.Spec.Placement.Image) + } +}