2026-09-24 13:12:45 +02:00
14 changed files with 1407 additions and 67 deletions
@@ -226,3 +226,6 @@ A lightweight `check` job (`go vet` / `go build` / `go test -short`) gates the b
 - `+kubebuilder:rbac` markers are package-level. Keep them in their own comment block,
  never inside a type's or function's doc comment: controller-gen silently ignores
  them there. After `make manifests`, check `git diff config/rbac/role.yaml`.
 - `make test-integration-proxmox` runs the proxmox provider's integration tests against
  a real Proxmox VE host. They skip unless `PROXMOX_TEST_URL`, `_TOKEN_FILE`, `_NODE`,
  `_TEMPLATE_VMID` and `_ISO_STORAGE` are set; see docs/testing.md for the full env block.
@@ -200,6 +200,23 @@ kvm-ssh-secret: ## Create or update the kvm-ssh Secret from KVM_SSH_KEY (private
 		--from-file=known_hosts="$(KVM_KNOWN_HOSTS)" \
 		--dry-run=client -o yaml | "$(KUBECTL)" apply -f -
 PROXMOX_TOKEN_FILE ?= $(HOME)/.config/proxmox-token
 PROXMOX_CA_FILE ?= $(HOME)/.config/proxmox-ca.pem
 PROXMOX_NAMESPACE ?= egress-proxies-operator-system
 .PHONY: proxmox-token-secret
 proxmox-token-secret: ## Create or update the proxmox-token Secret from PROXMOX_TOKEN_FILE (user@realm!tokenid=secret) and PROXMOX_CA_FILE.
 	@test -f "$(PROXMOX_TOKEN_FILE)" || { echo "PROXMOX_TOKEN_FILE=$(PROXMOX_TOKEN_FILE) not found"; exit 1; }
 	@test -f "$(PROXMOX_CA_FILE)" || { echo "PROXMOX_CA_FILE=$(PROXMOX_CA_FILE) not found"; exit 1; }
 	@test "$$(grep -cv '^[[:space:]]*$$' "$(PROXMOX_TOKEN_FILE)")" = 1 && \
 		grep -qxE '[[:space:]]*[^@[:space:]]+@[^![:space:]]+![^=[:space:]]+=[^[:space:]]+[[:space:]]*' "$(PROXMOX_TOKEN_FILE)" || \
 		{ echo "PROXMOX_TOKEN_FILE=$(PROXMOX_TOKEN_FILE) is not one line of user@realm!tokenid=secret"; exit 1; }
 	@grep -q 'BEGIN CERTIFICATE' "$(PROXMOX_CA_FILE)" || { echo "PROXMOX_CA_FILE=$(PROXMOX_CA_FILE) holds no PEM certificate"; exit 1; }
 	"$(KUBECTL)" -n $(PROXMOX_NAMESPACE) create secret generic proxmox-token \
 		--from-file=token="$(PROXMOX_TOKEN_FILE)" \
 		--from-file=ca.pem="$(PROXMOX_CA_FILE)" \
 		--dry-run=client -o yaml | "$(KUBECTL)" apply -f -
 .PHONY: deploy
 deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
 	cd config/manager && "$(KUSTOMIZE)" edit set image controller=${IMG}
@@ -192,6 +192,7 @@ can be two differently-configured instances of the same `gcp` type (see
 | `gcp` | A Compute Engine VM with an ephemeral external IP | `project` (required), `network`, `networkTag`, `diskSizeGb` | The real egress fleet. Deliberately uses only four API calls (Insert / Get / Delete / AggregatedList), all fire-and-forget: `Create` returns as soon as the operation is submitted and the reconciler discovers progress by polling `Get`. Auth is Application Default Credentials — workload identity in-cluster, `gcloud` ADC locally; no key-file plumbing. |
 | `kvm` | A libvirt VM on one of your own hypervisors, reached over SSH | `sshKeyFile`, `knownHostsFile` (unless `insecureIgnoreHostKey`), `machineTypes`, `hosts` (required); `images`, `diskSizeGb`, `vendorData`, `installGuestAgent` | Home-lab or on-prem egress. The base image is downloaded into each host's pool on first use; `Create` waits for the VM to start. See [docs/kvm-provider.md](docs/kvm-provider.md). |
 | `kvm2` | The same: a libvirt VM on one of your own hypervisors, reached over SSH | the same block shape as `kvm`, under `kvm2:` | The second-generation libvirt provider, built from the provider contract. Adds a child span per libvirt RPC, a pool-space precheck reported as quota (the reconciler waits instead of failing), SSH credentials re-read on every dial, and image downloads and health checks on connections of their own. It does not adopt `kvm`'s VMs. See [docs/kvm2-provider.md](docs/kvm2-provider.md). |
 | `proxmox` | A VM on a Proxmox VE cluster: a full clone of a template you prepare, reached through the Proxmox REST API with an API token | `url`, `tokenFile`, `nodes`, `isoStorage`, `templates`, `machineTypes` (required); `caFile` or `insecureSkipTLSVerify`, `storage`, `bridge`, `vlanTag`, `vmidRange`, `requestTimeoutSeconds`, `vendorData`, `installGuestAgent` | Home-lab or on-prem egress without SSH. Cloud-init goes on a seed ISO uploaded through the API; the address comes from the QEMU guest agent, so the proxy network needs a route to a package mirror unless the template ships the agent. `Create` waits for the VM to run. The token is read on every request, from the optional `proxmox-token` Secret (`make proxmox-token-secret`). Built from the provider contract; not yet run against a real host end to end. See [docs/proxmox-provider.md](docs/proxmox-provider.md). |
 ### Adding a provider
@@ -166,6 +166,18 @@ spec:
        - name: kvm-ssh
          mountPath: /etc/proxy-operator/kvm
          readOnly: true
        # API token and CA for the proxmox provider (tokenFile / caFile in
        # the providers config point here). Optional: the mount is empty when
        # the Secret does not exist, and only proxmox entries need it. The
        # token file is read on every request; the CA once, on first use.
        # Create it with:
        #   kubectl -n egress-proxies-operator-system create secret generic proxmox-token \
        #     --from-file=token=<file holding user@realm!tokenid=secret> \
        #     --from-file=ca.pem=<the node's /etc/pve/pve-root-ca.pem>
        # or `make proxmox-token-secret`.
        - name: proxmox-token
          mountPath: /etc/proxy-operator/proxmox
          readOnly: true
        # Scratch space: the root filesystem is read-only, and the kvm
        # provider builds each VM's cloud-init seed ISO through a temp
        # directory (a few hundred KB, deleted right after upload).
@@ -180,6 +192,11 @@ spec:
        secret:
          secretName: kvm-ssh
          optional: true
      # The proxmox provider's token and CA; see the mount above.
      - name: proxmox-token
        secret:
          secretName: proxmox-token
          optional: true
      - name: tmp
        emptyDir:
          sizeLimit: 64Mi
@@ -84,4 +84,32 @@ data:
              bridge: br0
              # socket: /var/run/libvirt/libvirt-sock
              # pool: default
      # proxmox clones a prepared template on a Proxmox VE cluster through its
      # REST API and seeds it with a NoCloud ISO uploaded to isoStorage; it needs
      # no SSH. Both files come from the optional proxmox-token Secret mounted by
      # config/manager/manager.yaml. The url's host must be a name or address in
      # the node certificate. See docs/proxmox-provider.md.
      # - name: proxmox-home
      #   type: proxmox
      #   proxmox:
      #     url: https://192.168.0.8:8006
      #     tokenFile: /etc/proxy-operator/proxmox/token
      #     caFile: /etc/proxy-operator/proxmox/ca.pem
      #     # insecureSkipTLSVerify: true   # instead of caFile; logged as a warning
      #     nodes: [pv1-1]                  # placement.zone names a node; optional with one node
      #     isoStorage: proxy-seeds         # iso content; the token needs Datastore.Allocate here
      #     # storage: local-lvm            # clone target; default is the template's disk storage
      #     # bridge: vmbr0                 # replaces the cloned net0's bridge
      #     vlanTag: 4                      # set on the cloned net0, MAC kept
      #     # vmidRange: {min: 100000, max: 199999}
      #     # requestTimeoutSeconds: 30
      #     # installGuestAgent: false
      #     # vendorData: |
      #     #   #cloud-config
      #     #   ...
      #     templates:                      # placement.image names a key; add keys, never repoint one
      #       debian-13: {vmid: 9001}
      #     machineTypes:                   # placement.machineType names a key here
      #       small:  {cores: 1, memoryMiB: 1024}
      #       medium: {cores: 2, memoryMiB: 2048, diskGiB: 10}
@@ -9,5 +9,6 @@ resources:
 - proxy_gcp.yaml
 - proxy_kvm.yaml
 - proxy_kvm2.yaml
 - proxy_proxmox.yaml
 - proxy_external.yaml
 # +kubebuilder:scaffold:manifestskustomizesamples
@@ -88,3 +88,31 @@ providers:
  #         bridge: br0
  #         # socket: /var/run/libvirt/libvirt-sock
  #         # pool: default
  # proxmox clones a prepared template on a Proxmox VE cluster through its
  # REST API and seeds it with a NoCloud ISO uploaded to isoStorage; it needs
  # no SSH. Both files come from the optional proxmox-token Secret mounted by
  # config/manager/manager.yaml. The url's host must be a name or address in
  # the node certificate. See docs/proxmox-provider.md.
  # - name: proxmox-home
  #   type: proxmox
  #   proxmox:
  #     url: https://192.168.0.8:8006
  #     tokenFile: /etc/proxy-operator/proxmox/token
  #     caFile: /etc/proxy-operator/proxmox/ca.pem
  #     # insecureSkipTLSVerify: true   # instead of caFile; logged as a warning
  #     nodes: [pv1-1]                  # placement.zone names a node; optional with one node
  #     isoStorage: proxy-seeds         # iso content; the token needs Datastore.Allocate here
  #     # storage: local-lvm            # clone target; default is the template's disk storage
  #     # bridge: vmbr0                 # replaces the cloned net0's bridge
  #     vlanTag: 4                      # set on the cloned net0, MAC kept
  #     # vmidRange: {min: 100000, max: 199999}
  #     # requestTimeoutSeconds: 30
  #     # installGuestAgent: false
  #     # vendorData: |
  #     #   #cloud-config
  #     #   ...
  #     templates:                      # placement.image names a key; add keys, never repoint one
  #       debian-13: {vmid: 9001}
  #     machineTypes:                   # placement.machineType names a key here
  #       small:  {cores: 1, memoryMiB: 1024}
  #       medium: {cores: 2, memoryMiB: 2048, diskGiB: 10}
@@ -0,0 +1,46 @@
 # A Managed proxy backed by a Proxmox VE cluster through the proxmox
 # provider: the operator full-clones a prepared template on the named node,
 # attaches a NoCloud seed ISO carrying this cloud-init, and starts the VM on
 # the configured network (here VLAN 4 on vmbr0, IP from that network's
 # DHCP), where cloud-init installs Squid. Requires a providers-config entry
 # named "proxmox-home" (see providers-config.yaml) and the proxmox-token
 # Secret (docs/proxmox-provider.md).
 #
 # placement.zone is a node name from proxmox.nodes (optional when only one
 # node is configured); machineType and image are keys in the provider's
 # machineTypes and templates maps. The address comes from the guest agent,
 # which a vendor-data script installs at first boot, so the VM's network
 # needs a route to a package mirror.
 apiVersion: crawl.example.com/v1alpha1
 kind: Proxy
 metadata:
  name: proxy-proxmox-sample
 spec:
  mode: Managed
  provider: proxmox-home
  placement:
    zone: pv1-1
    machineType: small
    image: debian-13
  port: 3128
  cloudInit:
    inline: |
      #cloud-config
      packages:
        - squid
      write_files:
        - path: /etc/squid/conf.d/proxy-operator.conf
          content: |
            http_port 3128
            http_access allow all
            via off
            forwarded_for off
      runcmd:
        - systemctl restart squid
      # Optional: log in with `ssh debian@<ip>` (the Debian image's default
      # user) to look around; `qm terminal <vmid>` on the node works too.
      # ssh_authorized_keys:
      #   - ssh-ed25519 AAAA... you@laptop
  attributes:
    geo: home
    purpose: crawl
@@ -11,6 +11,7 @@
 | gcp provider | `internal/provider/gcp` | library | — | Compute Engine VMs, four API calls, fire-and-forget ops |
 | kvm provider | `internal/provider/kvm` | library | — | VMs on libvirt hosts over SSH-forwarded sockets; image cache, overlay disks, NoCloud seeds |
 | kvm2 provider | `internal/provider/kvm2` | library | — | the same backend, built clean-room from the provider contract as a second type: per-RPC child spans, pool-space quota, per-call budgets, its own metadata namespace, credentials re-read on every dial |
 | proxmox provider | `internal/provider/proxmox` | library | — | VMs on a Proxmox VE cluster through its REST API: full clones of a template, a NoCloud seed uploaded through the API, the VMID derived from the Proxy UID, every task waited on, HTTP child spans |
 | Shared provider helpers | `internal/provider/nocloud`, `internal/provider/keyedlock` | library | — | the NoCloud seed ISO and vendor-data builder, and one-slot semaphores by key; used by kvm2 and proxmox |
 | Health engine | `internal/health` | Runnable | yes | through-the-proxy probes, thresholds, transition events |
 | Lease store | `internal/lease` | Runnable (expiry sweep) | no | in-memory leases + cooldowns, single mutex |
@@ -204,6 +205,37 @@ outlives its ctx or budget kills the pooled connection (the only way to unblock
 go-libvirt), so other calls in flight on that host fail once with Transient and
 retry; the next call redials. Each RPC is a child span kvm2.<Method> under
 provider.*, plus kvm2.dial and kvm2.download.
 proxmox provider (internal/provider/proxmox/) - the Proxmox VE REST API, one URL per cluster
 prov.Create ──► placement checked against the catalogue (Permanent before any request)
                GET /cluster/resources: a VM with the name → ownership check      ─┐ every step
                else: template config checked (not a template, cloud-init drive,   │ idempotent,
                        no net0 or boot disk it needs = Permanent)                 │ under a
                      VMID = hash(UID) in vmidRange, probing on; a taken VMID →    │ per-name lock
                        ownership check: wait out a lock, adopt our name + mark +  │ shared with
                        UID, probe past another name, Permanent otherwise          │ Delete; each
                      POST .../clone (full, description = the mark) → wait task    │ task's UPID
                upload seed ISO to isoStorage (overwrites) → wait task             │ polled every
                PUT .../config: cores, memory, agent=1, tag, ide2 = seed,          │ 1s to its end
                  net0 bridge/tag when configured                                  │
                resize if diskGiB is larger → wait task                            │
                start → wait task → poll status/current until running (60s)       ─┘
 prov.Get    ──► /cluster/resources (absent = NotFound; lock or node unknown = Transient)
                  + config (name and mark must match, else NotFound: VMID reused)
                  + status/current (paused and hibernated = Stopped)
                  + agent network-get-interfaces (first routable IPv4; agent not
                    up yet = Provisioning)
 prov.Delete ──► resources + config: gone, not ours or unmarked = leave it;
                  hibernated = Permanent; stop → destroy (purge) → delete the seed
                  on the VM's node (or on every node when no VM of ours was left)
 prov.ListByTag ─► one /cluster/resources for the whole cluster, one config read per
                    candidate (tag or proxy- name); the description's mark decides
 Check       ──► per node: /version, node status, isoStorage status, templates in
                  /cluster/resources
 providerID = vms/<vmid>/<name>: no node, because VMs migrate; the node is looked
 up in /cluster/resources on every call. Token read from its file on every
 request. Each request is an HTTP <METHOD> child span from the parent-gated
 transport wrapper; a cancelled request ends only itself.
 ```
 The reconciler never watches provider-side resources (Pods or GCP VMs).
@@ -388,6 +420,12 @@ incoming trace context):
  from a span decorator over the seam, applied once in the dialer, so no
  RPC is unspanned. The `libvirt call` log line of an RPC is written
  inside its span. The `kvm` provider has no child spans.
 - proxmox child spans: one `HTTP <METHOD>` client span per API request,
  from `tracing.RestConfigWrapper()` on the provider's own HTTP transport,
  under `provider.*` or `providerhealth.check`. Parent-gated like the
  Kubernetes API spans, so a request without a parent starts none. The
  provider's logging decorator starts no span; its `proxmox api call` line
  carries the operation span's IDs.
 - Kubernetes API calls become client child spans via a wrapped
  `rest.Config` transport — gated on an existing parent span, so informer
  list/watch long-polls and leader-election renewals never create root
@@ -413,12 +451,13 @@ the slog handler's ctx.
 Answers "can this cluster reach each provider's backend right now?"
 without waiting for a Proxy to fail. A provider opts in by implementing
 `provider.Checker`: `Check(ctx) []TargetStatus` returns one verdict per
-target and never fails as a whole. All four providers do:
+target and never fails as a whole. All five providers do:
 | Provider | Target | What one check does |
 |---|---|---|
 | kvm | each `kvm.hosts` entry | over the pooled SSH+libvirt connection: libvirt version and hostname, then the configured pool must exist and be running; reports pool capacity and free space ([details](kvm-provider.md#health-check)) |
 | kvm2 | each `kvm2.hosts` entry | on a fresh SSH+libvirt connection dialled for the check and closed after it: libvirt version and hostname, then the configured pool must exist and be running; `details` carry `user`, `pool`, `hostname`, `libvirtVersion`, `poolState`, `poolCapacityGiB` and `poolAvailableGiB` ([details](kvm2-provider.md#health-check)) |
 | proxmox | each `proxmox.nodes` entry | four requests on the shared HTTP client: API version, node status, the `isoStorage` status (it must be active and hold `iso` content), and the cluster's resources (every configured template must exist and be a template); `details` carry `isoStorage`, `tokenID`, `pveVersion`, `kernel`, `isoStorageAvailGiB` and `templates` ([details](proxmox-provider.md#health-check)) |
 | kubernetes | `apiserver` | lists at most one managed Pod cluster-wide: the verb and scope `ListByTag` and GC rely on |
 | gcp | the project | one page of an aggregated instance list with `maxResults=1`: credentials, Compute API enabled, `compute.instances.list` |
@@ -704,3 +743,29 @@ they read as choices, not accidents. Chronological by build step.
  the Secret needs no restart; nothing is cached. The constructor only
  checks that the files exist, to warn early without failing the provider
  set.
 - **proxmox derives the VMID from the Proxy UID and checks ownership
  before using a taken one.** Proxmox addresses VMs by VMID, and names are
  not unique, so the name cannot be the key. A hash of the UID placed in
  `vmidRange` makes a retry land on the same VMID; a taken one is waited
  on while locked, then adopted when it has our name, mark and UID,
  probed past when it has another name, and refused otherwise. The wait
  is what stops a retry after a crash mid-clone from cloning a duplicate:
  a clone in progress has no name yet, so the by-name lookup misses it.
 - **proxmox writes its mark into the VM's description**, in the clone
  call itself, so no VM of ours exists unmarked. Tags cannot be passed to
  the clone, and the clone's own creation time is copied from the
  template. The `proxy_operator` tag is set afterwards for the Proxmox UI
  and to narrow GC's candidates; the mark decides.
 - **proxmox delivers cloud-init as a seed ISO uploaded through the API**,
  not as Proxmox's own cloud-init drive or a `cicustom` snippet. The
  upload endpoint does not accept snippets, and the operator has no shell
  on the nodes; the seed code is kvm2's, extracted into
  `internal/provider/nocloud`. Templates must therefore carry no Proxmox
  cloud-init drive, and seeds of VMs destroyed outside the operator are a
  documented GC blind spot.
 - **proxmox resolves a VM's node on every call** from
  `/cluster/resources`, and its providerID names none, because VMs migrate
  between nodes. The cost is one extra request per operation; the gain is
  that a migrated VM stays addressable and the listing, which includes the
  guests of an unreachable node as `status: unknown`, never mistakes a
  down node for a deleted VM.
@@ -13,7 +13,7 @@ Pairs with [docs/plans/2026-09-18-1502-proxmox-provider.md](../plans/2026-09-18-
 - [x] Phase 6 — Create
 - [x] Phase 7 — Delete, Checker, CRD comment
 - [x] Phase 8 — Contract and observability sweep, integration test, testing.md
- [ ] Phase 9 — Deployment and documentation
+- [x] Phase 9 — Deployment and documentation
 - [ ] Phase 10 — Review, real-host validation, changelog, MR
 ## Phase 0 — Plan file, branch, host recon, fixtures
@@ -1121,3 +1121,180 @@ Worth noting:
  `go test -short ./...` run of this phase.
 - `.claude/settings.json` stays out of the commit. Nothing in this phase
  talked to the host.
 ## Phase 9 — Deployment and documentation
 Deployment: the optional `proxmox-token` Secret (keys `token` and `ca.pem`)
 is mounted read-only at `/etc/proxy-operator/proxmox` in
 `config/manager/manager.yaml`, next to `kvm-ssh` and commented the same way.
 The Makefile gains `proxmox-token-secret`. Both providers configs gain a
 commented `proxmox-home` entry after kvm2's. `config/samples/proxy_proxmox.yaml`
 is listed in the samples kustomization. There is no RBAC change and no
 `hack/providers-dev.yaml` entry. Docs: `docs/proxmox-provider.md` follows
 kvm2's guide heading for heading and has no `—`. The rows and sections
 named below went into `providers-existing.md`, `architecture.md`, the
 README, `providers.md` and `CLAUDE.md`. `docs/testing.md` and the
 integration Makefile target were done in Phase 8 and were not touched; the
 guide's Testing section links to them.
 Where the spec and the plan's Phase 9 text disagree, the spec won. The URL
 is `https://192.168.0.8:8006`, not 192.168.2.8, which is not in the node
 certificate. The Secret has two keys, and the target takes
 `PROXMOX_CA_FILE` as well as `PROXMOX_TOKEN_FILE`.
 Judgment calls:
 - **The target also checks both files' content.** The token file must be
  exactly one non-blank line matching
  `<user>@<realm>!<tokenid>=<secret>`. That is the shape `readToken`
  accepts: `TrimSpace`, then an anchored regexp. The check uses `grep -c`
  and `grep -qx`, so the secret is never printed. The CA check mirrors
  `kvm-ssh-secret`'s `PRIVATE KEY` check: the file must contain
  `BEGIN CERTIFICATE`. With `insecureSkipTLSVerify` the guide says to
  create the Secret by hand with the `token` key only.
 - **The sample-entry test covers both files.** `commentedEntry` finds the
  `# - name: proxmox-home` line and takes its indent. It keeps the lines
  that continue at `<indent>#   ` and strips `# ` at that indent. The same
  helper then works for the ConfigMap, whose entry sits six spaces deep
  inside a `|` block. So `TestLoadConfig_proxmoxSampleEntry` has a
  `sample` subtest and a `configmap` subtest. It checks every field the
  spec names, and that the optional fields stay commented out.
 - **A second test goes beyond the plan:**
  `TestLoadConfig_proxmoxSampleProxyMatchesSampleEntry`. The sample Proxy's
  provider, zone, machine type and image must resolve in the sample
  entry. kvm has exactly this bug as a known gap (`proxy_kvm.yaml` places
  on `nas1`).
 - **The sample entry also has `medium: {cores: 2, memoryMiB: 2048,
  diskGiB: 10}`**, next to the `small` the spec asks for, so `diskGiB`
  appears somewhere. The optional fields are commented out with a short
  note each, because no live proxmox entry exists to explain them. The
  kvm2 entry can rely on kvm's.
 - **In the ConfigMap the proxmox entry is commented out**, although kvm2's
  is live there. It needs a host and a token.
 - **providers-existing.md needed more than the column and the section.**
  The intro now says five providers and links the guide. In the shared
  gaps, six sentences would otherwise be false, and each now names
  proxmox where it applies: adoption verified in two of the five, Create
  blocking in all three VM providers, no Terminated, the list policy, the
  collector gap, and the "kvm2 does this" in proposed features. The new
  section is §8, so the shared gaps and proposed features move to §9 and
  §10. No doc links those sections by anchor (grep).
 - **architecture.md §11 now says "All five providers do".** The sentence
  counts Checkers, and proxmox has one. The four Decisions bullets are
  entry 23's: the VMID from the UID with the ownership check, the mark in
  the description, the seed uploaded through the API, and the node
  resolved per call.
 - **The guide lists the ten design limits first, then four more.** They
  are the hibernated-VM latch, the one-request seed upload, the open-proxy
  sample (kvm2's guide lists it too), and "not yet proven end to end" with
  the three answers never captured. The Phase 6 to 8 guide items are
  each in the section where a reader looks for them. The config write
  replacing `agent` and `tags` is in *What lands on Proxmox*, and so are
  the seed cleanup, `qm unlock` and the 403 that makes Delete Permanent.
  The certificate rule is in *Cluster-side setup*. CA caching versus
  token rotation is in both.
 - **The guide resumes a hibernated VM with `qm start`, not
  `qm resume`.** A VM suspended to disk resumes from its state file on
  its next start; `qm resume` is for a VM paused in RAM.
 - **The guide's `pvesm add dir proxy-seeds --path /var/lib/proxy-seeds
  --content iso` is the standard form, not a recorded command.** Phase 0
  says only that the user created `proxy-seeds` as a `dir` storage with
  ISO content; the path is illustrative.
 The token-file check, run without touching a cluster: `KUBECTL=echo`
 prints only the apply stage, and `KUBECTL=true` runs nothing. Scratch
 files covered one good line, two lines, a bare secret, blank lines around
 a good line, a non-PEM CA and a missing file. The real files on this
 workstation pass:
 ```bash
 make -n proxmox-token-secret
 make -s proxmox-token-secret PROXMOX_TOKEN_FILE=$D/<case> PROXMOX_CA_FILE=$D/ca KUBECTL=echo
 # good, ws: "apply -f -"; two, bad: "... is not one line of user@realm!tokenid=secret"; noca: "... holds no PEM certificate"
 make -s proxmox-token-secret KUBECTL=true && echo "real files pass"   # real files pass
 ```
 The mutation check for the sample tests edited one line per run,
 asserting a single match. It ran the two tests, restored the file from a
 copy, and at the end `sha256sum -c` confirmed the three files. All five
 mutations failed the tests: `vlanTag: 5000` and `isoStorag:` in the
 sample, `192.168.2.8` and `type: kvm2` in the ConfigMap, and
 `image: debian-12` in the sample Proxy. `type: kvm2` was rerun after lint
 removed the test's own type comparison. It still fails, because
 `LoadConfig` refuses a proxmox block under another type.
 The link sweep is a short script. It takes the docs as arguments, so it
 can be rerun after any later edit. It skips fenced code and inline code
 spans. For each relative link it checks that the file exists, and that
 the `#anchor` matches a GitHub-style slug of a heading in the target file
 (or the same file). Before the real run it was checked on a scratch copy
 of the guide with five links added. It reported exactly the three bad
 ones: a missing file, a missing anchor in `testing.md`, and a missing
 same-file anchor. It checked 78 relative links in the six edited docs,
 plus this log's, and found 0 broken:
 ```bash
 cat > doclinks.py <<'PY'
 import os, re, sys
 def slug(h, seen):
    s = re.sub(r"[^\w\- ]", "", h.strip().lower()).replace(" ", "-")
    n = seen.get(s, 0); seen[s] = n + 1
    return s if n == 0 else f"{s}-{n}"
 def prose(path):
    out, fence = [], False
    for line in open(path, encoding="utf-8"):
        if line.lstrip().startswith("```"): fence = not fence; continue
        if not fence: out.append(line)
    return out
 def anchors(path):
    seen = {}
    return {slug(re.sub(r"`", "", l.lstrip("#").strip()), seen) for l in prose(path) if re.match(r"#{1,6} ", l)}
 bad = 0
 for doc in sys.argv[1:]:
    for n, line in enumerate(prose(doc), 1):
        for target in re.findall(r"\]\(([^)\s]+)\)", re.sub(r"`[^`]*`", "", line)):
            if re.match(r"[a-z]+:", target): continue
            f, _, a = target.partition("#")
            p = os.path.normpath(os.path.join(os.path.dirname(doc), f)) if f else doc
            if not os.path.exists(p): print(f"{doc}: missing file {target}"); bad += 1
            elif a and p.endswith(".md") and a not in anchors(p): print(f"{doc}: missing anchor {target}"); bad += 1
 print(f"{bad} broken"); sys.exit(1 if bad else 0)
 PY
 python3 doclinks.py docs/proxmox-provider.md docs/providers-existing.md docs/architecture.md \
  README.md docs/providers.md CLAUDE.md docs/plans-executions/2026-09-18-1502-proxmox-provider.md   # 0 broken
 ```
 Verification:
 ```bash
 bin/kustomize build config/default >/dev/null && bin/kustomize build config/samples >/dev/null   # ok
 go test -count=1 -run 'TestLoadConfig_proxmoxSample' -v ./internal/provider/                     # 3 PASS
 grep -c '—' docs/proxmox-provider.md                                                             # 0
 go build ./... && go vet ./... && go vet -tags=integration ./... && go test -short -count=1 ./...   # all ok
 bin/golangci-lint run --disable staticcheck --max-same-issues 0 --max-issues-per-linter 0 ./internal/provider/
 git status --short config/   # the four edited files named above, plus proxy_proxmox.yaml
 ```
 Worth noting:
 - Lint on `internal/provider` is at its baseline of three goconst hits.
  The first version of the new test pushed it to five: its
  `p.Type != "proxmox"` made `"proxmox"` a three-time literal, flagged in
  both `config.go` and the test. That comparison added nothing, so it
  went. `internal/provider/proxmox` was not touched.
 - The `tmp` volume comment in `manager.yaml` still says only the kvm
  provider stages seeds there. kvm2 and proxmox do too. It was already
  stale for kvm2, so it was left out of this phase's diff.
 - The architecture §5 block's right-hand bracket was aligned by
  character, not byte (`→` is three bytes), so every line puts `│` in
  column 83.
 - Nothing in this phase talked to the Proxmox host or a cluster.
  `.claude/settings.json` stays out of the commit. The parked kvm2 items
  (S5.1, and the 200 ms `TestCheck_hostsCheckedConcurrently`) did not
  bite: `go test -short ./...` passed.
 - Next is Phase 10: the first real run of `TestProxmox_lifecycle` and
  `TestProxmoxCheck_realHost` on pv1-1, with the env block in
  `docs/testing.md`. The three answers never captured go with it, and the
  guide's "Not yet proven end to end" gap and the README's matching
  clause come out once they pass. Then the CHANGELOG entry after the user
  confirms, and the MR.
@@ -1,17 +1,20 @@
 # The existing providers
 This is the appendix to [provider-contract.md](provider-contract.md): how the
-four providers that exist today (kubernetes, gcp, kvm, kvm2) meet the
+five providers that exist today (kubernetes, gcp, kvm, kvm2, proxmox) meet
-contract, which profile each one chose on every axis, and what is still
+the contract, which profile each one chose on every axis, and what is still
 missing. It is reference material for whoever writes the next provider, not
 a user guide; the user guides are [kvm-provider.md](kvm-provider.md),
 [kvm2-provider.md](kvm2-provider.md),
 [proxmox-provider.md](proxmox-provider.md),
 [gcp-vm-validation.md](gcp-vm-validation.md) and
 [gcp-in-specific-project.md](gcp-in-specific-project.md). The README's
-"Implemented providers" table stays the short public summary. kvm2 is the
+"Implemented providers" table stays the short public summary. kvm2 and
-one provider built from the contract itself (clean-room, from
+proxmox are the two providers built from the contract itself (kvm2
-[kvm2-provider-spec.md](kvm2-provider-spec.md), without reading kvm's code),
+clean-room from [kvm2-provider-spec.md](kvm2-provider-spec.md) without
-so its column is also a record of what the contract produced.
+reading kvm's code, proxmox from
 [proxmox-provider-spec.md](proxmox-provider-spec.md)), so their columns are
 also a record of what the contract produced.
 The per-provider call flows are drawn in
 [architecture.md §5](architecture.md#5-what-provider-calls-do-in-the-outside-world),
@@ -27,59 +30,59 @@ Code references are a file path plus a function or symbol name.
 One row per axis of contract §4, in the same order, with the profile name
 and the symbol that shows it.
-| Axis | kubernetes | gcp | kvm | kvm2 |
+| Axis | kubernetes | gcp | kvm | kvm2 | proxmox |
-|---|---|---|---|---|
+|---|---|---|---|---|---|
-| `readiness` | immediate: a Pod is scheduled within seconds (`stateFromPod`) | provisioning-window: minutes of boot (`mapState`) | provisioning-window: boot plus, on first use, an image download (`imageCache.ensure`) | provisioning-window: boot plus, on first use, an image download (`ensureImage`) |
+| `readiness` | immediate: a Pod is scheduled within seconds (`stateFromPod`) | provisioning-window: minutes of boot (`mapState`) | provisioning-window: boot plus, on first use, an image download (`imageCache.ensure`) | provisioning-window: boot plus, on first use, an image download (`ensureImage`) | provisioning-window: boot plus the guest agent's install at first boot, about 50 s to an address on the test host (`agentAddress`) |
-| `create-return` | submit: returns after `client.Create` | submit: `Insert` is never awaited, `Operation.Wait` is not called | **blocking**: waits through `DomainCreate` (`ensureRunning`), and the first image fetch runs inside (`imageCache.fetch`) | **blocking**: waits through `DomainCreate` and a 15-second poll for running (`ensureRunning`, `waitRunning`), and the first image fetch runs inside (`fetchImage`) |
+| `create-return` | submit: returns after `client.Create` | submit: `Insert` is never awaited, `Operation.Wait` is not called | **blocking**: waits through `DomainCreate` (`ensureRunning`), and the first image fetch runs inside (`imageCache.fetch`) | **blocking**: waits through `DomainCreate` and a 15-second poll for running (`ensureRunning`, `waitRunning`), and the first image fetch runs inside (`fetchImage`) | **blocking**: waits through the clone, upload, resize and start tasks and a 60-second poll for running (`provision`, `waitRunning`) |
-| `delete-return` | returns-early: `client.Delete`, Pod terminates on its own | returns-early: `Delete` submits an operation each time | waits: destroy, undefine and volume removal complete before return (`removeDomain`, `removeVolumes`) | waits: destroy, undefine and volume removal complete before return (`removeDomain`, `removeVolumes`) |
+| `delete-return` | returns-early: `client.Delete`, Pod terminates on its own | returns-early: `Delete` submits an operation each time | waits: destroy, undefine and volume removal complete before return (`removeDomain`, `removeVolumes`) | waits: destroy, undefine and volume removal complete before return (`removeDomain`, `removeVolumes`) | waits: the stop, destroy and seed delete tasks complete before return (`destroy`, `deleteSeed`) |
-| `ip-source` | assigned-at-create: `PodIP` (`instanceFromPod`) | assigned-at-create: the external NAT IP of the first NIC (`toInstance`) | discovered: ARP table first, then the guest agent, first routable IPv4 (`address`, `firstIPv4`) | discovered: the guest agent first, then the ARP table, first routable IPv4 (`addressSources`, `address`, `firstIPv4`) |
+| `ip-source` | assigned-at-create: `PodIP` (`instanceFromPod`) | assigned-at-create: the external NAT IP of the first NIC (`toInstance`) | discovered: ARP table first, then the guest agent, first routable IPv4 (`address`, `firstIPv4`) | discovered: the guest agent first, then the ARP table, first routable IPv4 (`addressSources`, `address`, `firstIPv4`) | discovered: the guest agent only, first routable IPv4 in the agent's order (`agentAddress`, `firstIPv4`) |
-| `gc-mark` | native-labels: Pod labels (`buildPod`) | native-labels: instance labels (`buildInsertRequest`) | written-metadata: a namespaced `<metadata>` element written in `DomainDefineXML` (`metadata.go`) | written-metadata: an element in kvm2's own namespace written in `DomainDefineXML` (`metadataURI`, `newInstanceMetadata`) |
+| `gc-mark` | native-labels: Pod labels (`buildPod`) | native-labels: instance labels (`buildInsertRequest`) | written-metadata: a namespaced `<metadata>` element written in `DomainDefineXML` (`metadata.go`) | written-metadata: an element in kvm2's own namespace written in `DomainDefineXML` (`metadataURI`, `newInstanceMetadata`) | written-metadata: three `key=value` lines in the VM's description, passed in the clone call itself (`formatDescription`, `parseDescription`) |
-| `created-at` | backend-timestamp: `CreationTimestamp` | backend-timestamp: `CreationTimestamp`, parse error gives zero | written-at-create: `createdAt` in the metadata element (`newInstanceMetadata`) | written-at-create: `createdAt` in the metadata element, parse error gives zero (`newInstanceMetadata`, `createdAt`) |
+| `created-at` | backend-timestamp: `CreationTimestamp` | backend-timestamp: `CreationTimestamp`, parse error gives zero | written-at-create: `createdAt` in the metadata element (`newInstanceMetadata`) | written-at-create: `createdAt` in the metadata element, parse error gives zero (`newInstanceMetadata`, `createdAt`) | written-at-create: `proxy-operator-created-at` in the description, parse error gives zero (`parseDescription`) |
-| `list-partial` | n/a: one call, all namespaces | omit-unreachable: `ReturnPartialSuccess` on `AggregatedList` | fail-whole per host, one unreadable domain skipped (`ListByTag`, `managedInstance`) | fail-whole per host, one unreadable domain skipped unless ctx ended; `ID`, `UID` and `CreatedAt` from the name and the mark only, no state or address RPC (`ListByTag`) |
+| `list-partial` | n/a: one call, all namespaces | omit-unreachable: `ReturnPartialSuccess` on `AggregatedList` | fail-whole per host, one unreadable domain skipped (`ListByTag`, `managedInstance`) | fail-whole per host, one unreadable domain skipped unless ctx ended; `ID`, `UID` and `CreatedAt` from the name and the mark only, no state or address RPC (`ListByTag`) | fail-whole for the one cluster-wide `/cluster/resources` call, one unreadable VM skipped unless ctx ended; `ID`, `UID` and `CreatedAt` from the description only, no status or agent call (`ListByTag`) |
-| `quota` | absent: a quota 403 is Permanent (`classify`) | detected: 429, and 403 with reason `quotaExceeded` or `rateLimitExceeded` (`classify`, `hasReason`) | absent (`classify`) | detected: a pool-space precheck before any volume is created (`checkPoolSpace`, `neededBytes`) |
+| `quota` | absent: a quota 403 is Permanent (`classify`) | detected: 429, and 403 with reason `quotaExceeded` or `rateLimitExceeded` (`classify`, `hasReason`) | absent (`classify`) | detected: a pool-space precheck before any volume is created (`checkPoolSpace`, `neededBytes`) | absent: thin storage overprovisions, so no out-of-space answer exists to match; free space goes to the Checker's details (`classify`) |
-| `not-found` | status-404: `apierrors.IsNotFound` | status-404: `isNotFound` | error-code: `ErrNoDomain`, `ErrNoStorageVol` (`hasCode`) | error-code: `ErrNoDomain`, `ErrNoStorageVol`, Transient inside Create (`classifyLibvirt`) |
+| `not-found` | status-404: `apierrors.IsNotFound` | status-404: `isNotFound` | error-code: `ErrNoDomain`, `ErrNoStorageVol` (`hasCode`) | error-code: `ErrNoDomain`, `ErrNoStorageVol`, Transient inside Create (`classifyLibvirt`) | error-code: the 500 `Configuration file '…' does not exist`, plus list-and-miss on `/cluster/resources`; Transient inside Create (`isNotExist`, `classifyAPI`) |
-| `primary-key` | name-is-key: `namespace/name` | name-is-key: `zones/<zone>/instances/<name>` | name-is-key: `hosts/<host>/domains/<name>` | name-is-key: `hosts/<host>/domains/<name>`, with a UID guard on adoption (`create`) |
+| `primary-key` | name-is-key: `namespace/name` | name-is-key: `zones/<zone>/instances/<name>` | name-is-key: `hosts/<host>/domains/<name>` | name-is-key: `hosts/<host>/domains/<name>`, with a UID guard on adoption (`create`) | **allocated-id**: `vms/<vmid>/<name>`, the VMID from an FNV-1a hash of the UID with forward probing, each taken VMID put through an ownership check with a UID guard (`vmidFor`, `allocate`, `inspect`) |
-| `provisioning-state` | native: `PodPending` | native: `PROVISIONING`, `STAGING`, `REPAIRING` | create-blocks: a domain is defined and started in one `Create`; `nostate` maps to Provisioning | create-blocks: defined, started and polled to running in one `Create`; `nostate` maps to Provisioning (`mapState`) |
+| `provisioning-state` | native: `PodPending` | native: `PROVISIONING`, `STAGING`, `REPAIRING` | create-blocks: a domain is defined and started in one `Create`; `nostate` maps to Provisioning | create-blocks: defined, started and polled to running in one `Create`; `nostate` maps to Provisioning (`mapState`) | create-blocks: cloned, configured, started and polled to running in one `Create`; `qmpstatus: prelaunch` maps to Provisioning (`mapState`) |
-| `async-ops` | none | task-handle, but the handle is discarded: an insert that fails after acceptance is never read | none | none |
+| `async-ops` | none | task-handle, but the handle is discarded: an insert that fails after acceptance is never read | none | none | task-handle and backend-lock: every clone, upload, resize, start, stop, destroy and volume delete returns a UPID polled to its end within the call (`waitTask`); a VM's lock is waited on in Create and Transient in Get and Delete (`waitUnlocked`) |
-| `secondary-resources` | none | cascade: `AutoDelete` on the boot disk (`buildInsertRequest`) | explicit: overlay and seed removed by name (`removeVolumes`); base image kept | explicit: overlay and seed removed by name (`removeVolumes`); base image kept |
+| `secondary-resources` | none | cascade: `AutoDelete` on the boot disk (`buildInsertRequest`) | explicit: overlay and seed removed by name (`removeVolumes`); base image kept | explicit: overlay and seed removed by name (`removeVolumes`); base image kept | explicit: the seed ISO deleted by name (`deleteSeed`, `deleteSeedEverywhere`); the disks go with the destroy's `purge=1&destroy-unreferenced-disks=1` |
-| `credentials` | workload-identity: in-cluster config or kubeconfig (`NewWithTransportWrapper`) | environment: Application Default Credentials | mounted-files: SSH key and known_hosts from the optional `kvm-ssh` Secret, read lazily (`newLazySSH`) | mounted-files: the same `kvm-ssh` Secret, read on every dial and never cached (`loadCredentials`) |
+| `credentials` | workload-identity: in-cluster config or kubeconfig (`NewWithTransportWrapper`) | environment: Application Default Credentials | mounted-files: SSH key and known_hosts from the optional `kvm-ssh` Secret, read lazily (`newLazySSH`) | mounted-files: the same `kvm-ssh` Secret, read on every dial and never cached (`loadCredentials`) | mounted-files: the token from the optional `proxmox-token` Secret, read on every request (`readToken`); the CA from the same Secret, loaded once and cached (`httpClient`) |
-| `client-kind` | vendor-sdk: controller-runtime `client.Client`, no seam interface | vendor-sdk: `cloud.google.com/go/compute` behind `instancesAPI` | rpc-over-ssh: `go-libvirt` over an SSH-forwarded unix socket, behind `libvirtAPI` (`conn.go`) | rpc-over-ssh: `go-libvirt` over an SSH-forwarded unix socket, behind the ctx-first `backend` seam (`backend.go`) |
+| `client-kind` | vendor-sdk: controller-runtime `client.Client`, no seam interface | vendor-sdk: `cloud.google.com/go/compute` behind `instancesAPI` | rpc-over-ssh: `go-libvirt` over an SSH-forwarded unix socket, behind `libvirtAPI` (`conn.go`) | rpc-over-ssh: `go-libvirt` over an SSH-forwarded unix socket, behind the ctx-first `backend` seam (`backend.go`) | thin-client: `net/http` and `encoding/json` behind the ctx-first `api` seam, one method per endpoint (`api.go`, `httpAPI`) |
-| `timeouts` | ctx-only: no `rest.Config.Timeout` is set | ctx-only: SDK defaults, with SDK retries on Get and List | per-call for the dial only (`defaultHandshakeTimeout` in `dialSSH`); RPCs are ctx-only and a cancelled `call` closes the connection | per-call: a budget around the dial, every RPC, the uploads and the download (`defaultBudgets`, `call`); a cancelled or overlong call kills the pooled connection |
+| `timeouts` | ctx-only: no `rest.Config.Timeout` is set | ctx-only: SDK defaults, with SDK retries on Get and List | per-call for the dial only (`defaultHandshakeTimeout` in `dialSSH`); RPCs are ctx-only and a cancelled `call` closes the connection | per-call: a budget around the dial, every RPC, the uploads and the download (`defaultBudgets`, `call`); a cancelled or overlong call kills the pooled connection | per-call: dial and TLS handshake 10 s, each request `requestTimeoutSeconds`, a budget per task wait (`defaultBudgets`); a cancelled request ends only itself |
-| `logging` | none | tiered: V(1) outcomes and `logAPIError`, V(2) detail, V(5) wire log (`wirelog.go`) | tiered: V(1) outcomes and one `libvirt call` line per RPC (`trace.go`), V(2) XML and addresses; no wire tier | tiered: V(1) outcomes and one `libvirt call` line per RPC (`observe.go`), V(2) XML and addresses; no wire tier |
+| `logging` | none | tiered: V(1) outcomes and `logAPIError`, V(2) detail, V(5) wire log (`wirelog.go`) | tiered: V(1) outcomes and one `libvirt call` line per RPC (`trace.go`), V(2) XML and addresses; no wire tier | tiered: V(1) outcomes and one `libvirt call` line per RPC (`observe.go`), V(2) XML and addresses; no wire tier | tiered: V(1) outcomes and one `proxmox api call` line per request (`observe.go`), V(2) parameters, config summaries and addresses; no wire tier |
-| `child-spans` | transport-wrapper: `tracing.RestConfigWrapper()` on its own `rest.Config`, parent-gated | sdk-instrumentation: the SDK adds `otelhttp` itself; not parent-gated, but every call site has a parent | **none**: `trace.go` is a log decorator, not spans | span-decorator: `kvm2.<Method>` per RPC (`observe`), plus `kvm2.dial` (`libvirtDialer`) and `kvm2.download` (`fetchImage`) |
+| `child-spans` | transport-wrapper: `tracing.RestConfigWrapper()` on its own `rest.Config`, parent-gated | sdk-instrumentation: the SDK adds `otelhttp` itself; not parent-gated, but every call site has a parent | **none**: `trace.go` is a log decorator, not spans | span-decorator: `kvm2.<Method>` per RPC (`observe`), plus `kvm2.dial` (`libvirtDialer`) and `kvm2.download` (`fetchImage`) | transport-wrapper: `tracing.RestConfigWrapper()` on the client's transport, parent-gated, one `HTTP <METHOD>` span per request (`newHTTPAPI`) |
-| `backend-test` | cluster-e2e: kind (`test/e2e`) | manual-runbook: [gcp-vm-validation.md](gcp-vm-validation.md) | integration-tag: `kvm_integration_test.go`, `make test-integration-kvm` | integration-tag: `kvm2_integration_test.go`, `make test-integration-kvm2` |
+| `backend-test` | cluster-e2e: kind (`test/e2e`) | manual-runbook: [gcp-vm-validation.md](gcp-vm-validation.md) | integration-tag: `kvm_integration_test.go`, `make test-integration-kvm` | integration-tag: `kvm2_integration_test.go`, `make test-integration-kvm2` | integration-tag: `proxmox_integration_test.go`, `make test-integration-proxmox`; compiled, not yet run against a host |
 ## 2. Other differences
 Rows that are not profile axes but that a new provider author compares
 against.
-| Feature | kubernetes | gcp | kvm | kvm2 |
+| Feature | kubernetes | gcp | kvm | kvm2 | proxmox |
-|---|---|---|---|---|
+|---|---|---|---|---|---|
-| Create verifies the adopted instance's UID | no | no | no | **yes**: a domain without the kvm2 mark or with a different non-empty UID is Permanent; a marked one with a matching or empty UID is adopted (`create`) |
+| Create verifies the adopted instance's UID | no | no | no | **yes**: a domain without the kvm2 mark or with a different non-empty UID is Permanent; a marked one with a matching or empty UID is adopted (`create`) | **yes**: a VM with the proxy's name and no mark, or with a different non-empty UID, is Permanent; a marked one with a matching or empty UID is adopted (`inspect`) |
-| Placement used | none | Zone, MachineType, Image | Zone (host), MachineType, Image | Zone (host), MachineType, Image |
+| Placement used | none | Zone, MachineType, Image | Zone (host), MachineType, Image | Zone (host), MachineType, Image | Zone (node), MachineType, Image (a template key) |
-| CloudInit delivered | **no**: ignored | yes: `user-data` metadata | yes: NoCloud seed ISO built in-process (`buildSeedISO`) | yes: NoCloud seed ISO built in-process (`nocloud.BuildSeedISO`) |
+| CloudInit delivered | **no**: ignored | yes: `user-data` metadata | yes: NoCloud seed ISO built in-process (`buildSeedISO`) | yes: NoCloud seed ISO built in-process (`nocloud.BuildSeedISO`) | yes: NoCloud seed ISO built in-process (`nocloud.BuildSeedISO`) and uploaded through the API (`uploadSeed`) |
-| Port used | yes: Squid config and containerPort (`squidConf`) | no: firewall is manual | no | no |
+| Port used | yes: Squid config and containerPort (`squidConf`) | no: firewall is manual | no | no | no |
-| States produced | Provisioning, Running, Terminated | all four | Provisioning, Running, Stopped | Provisioning, Running, Stopped |
+| States produced | Provisioning, Running, Terminated | all four | Provisioning, Running, Stopped | Provisioning, Running, Stopped | Provisioning, Running, Stopped |
-| Deleting instance still reports its state | yes: a Pod being deleted reports its phase | n/a | n/a | n/a: Delete waits, and a concurrent Get sees the domain until it is undefined |
+| Deleting instance still reports its state | yes: a Pod being deleted reports its phase | n/a | n/a | n/a: Delete waits, and a concurrent Get sees the domain until it is undefined | n/a: Delete waits, and a concurrent Get sees the VM (Stopped once stopped) until the destroy removes its config |
-| Error.Provider is the configured name | **no**: hardcoded `"kubernetes"` | yes | yes | yes (`wrapErr`) |
+| Error.Provider is the configured name | **no**: hardcoded `"kubernetes"` | yes | yes | yes (`wrapErr`) | yes (`wrapErr`) |
-| Checker target | `apiserver` | the project | each host, concurrently | each host, concurrently, on a fresh connection |
+| Checker target | `apiserver` | the project | each host, concurrently | each host, concurrently, on a fresh connection | each node, concurrently, on the shared HTTP client |
-| Checker Details | none | none | `user`, `pool`, `hostname`, `libvirtVersion`, `poolState`, `poolCapacity`, `poolAvailable` | `user`, `pool`, `hostname`, `libvirtVersion`, `poolState`, `poolCapacityGiB`, `poolAvailableGiB`, filled in as far as the check got |
+| Checker Details | none | none | `user`, `pool`, `hostname`, `libvirtVersion`, `poolState`, `poolCapacity`, `poolAvailable` | `user`, `pool`, `hostname`, `libvirtVersion`, `poolState`, `poolCapacityGiB`, `poolAvailableGiB`, filled in as far as the check got | `isoStorage`, `tokenID`, `pveVersion`, `kernel`, `isoStorageAvailGiB`, `templates`, filled in as far as the check got |
-| Credential rotation without restart | n/a | via ADC | no: cached after first successful load | yes: both files read on every dial (`loadCredentials`) |
+| Credential rotation without restart | n/a | via ADC | no: cached after first successful load | yes: both files read on every dial (`loadCredentials`) | the token yes, read on every request (`readToken`); the CA no, cached after first use |
-| Connection handling | client-go | SDK client, never closed | one pooled SSH plus libvirt connection per host, 15s keepalive (`hostPool`) | one pooled SSH plus libvirt connection per host, 15s keepalive (`hostTable`); dedicated connections for the image upload and the check (`dialFresh`) |
+| Connection handling | client-go | SDK client, never closed | one pooled SSH plus libvirt connection per host, 15s keepalive (`hostPool`) | one pooled SSH plus libvirt connection per host, 15s keepalive (`hostTable`); dedicated connections for the image upload and the check (`dialFresh`) | one `net/http` client, 4 idle connections per host, HTTP/1.1 only; a cancelled request ends only itself |
-| Unit-test double | controller-runtime fake client | `fakeAPI` behind `instancesAPI`, plus `httptest` for the real SDK client in `check_test.go` | `fakeAPI` behind `libvirtAPI`, plus an in-process `x/crypto/ssh` server in `conn_test.go` | `fakeBackend` behind `backend` (`fake_test.go`), plus an in-process `x/crypto/ssh` server (`sshserver_test.go`) and go-libvirt's own mock for the adapter |
+| Unit-test double | controller-runtime fake client | `fakeAPI` behind `instancesAPI`, plus `httptest` for the real SDK client in `check_test.go` | `fakeAPI` behind `libvirtAPI`, plus an in-process `x/crypto/ssh` server in `conn_test.go` | `fakeBackend` behind `backend` (`fake_test.go`), plus an in-process `x/crypto/ssh` server (`sshserver_test.go`) and go-libvirt's own mock for the adapter | `fakeAPI` behind `api` (`fake_test.go`), plus `httptest.NewTLSServer` replaying the Phase 0 captures for the real client (`client_test.go`) |
-| Proxy software | generated Squid config (`pod.go`) | from `cloudInit`; the image must run cloud-init | from `cloudInit`, plus a vendor-data script that installs the guest agent (`seed.go`) | from `cloudInit`, plus a vendor-data script that installs the guest agent (`nocloud.GuestAgentScript`) |
+| Proxy software | generated Squid config (`pod.go`) | from `cloudInit`; the image must run cloud-init | from `cloudInit`, plus a vendor-data script that installs the guest agent (`seed.go`) | from `cloudInit`, plus a vendor-data script that installs the guest agent (`nocloud.GuestAgentScript`) | from `cloudInit`, plus a vendor-data script that installs the guest agent (`nocloud.GuestAgentScript`) |
-| IP reachable from the operator | Pod network, so in-cluster only (not from `make run-dev` on a laptop) | external IP, once a firewall rule allows the cluster's egress IP | LAN address on the host's bridge; the operator needs a route to that LAN | LAN address on the host's bridge; the operator needs a route to that LAN |
+| IP reachable from the operator | Pod network, so in-cluster only (not from `make run-dev` on a laptop) | external IP, once a firewall rule allows the cluster's egress IP | LAN address on the host's bridge; the operator needs a route to that LAN | LAN address on the host's bridge; the operator needs a route to that LAN | address on the proxy network (bridge or VLAN from the config); the operator needs a route to it |
-| Distinct egress IP per proxy | no: the cluster's egress | yes: ephemeral external IP | whatever the host's LAN routes through; not per VM by itself | whatever the host's LAN routes through; not per VM by itself |
+| Distinct egress IP per proxy | no: the cluster's egress | yes: ephemeral external IP | whatever the host's LAN routes through; not per VM by itself | whatever the host's LAN routes through; not per VM by itself | whatever the proxy network routes through; not per VM by itself |
 ## 3. Observability status
-| Layer | kubernetes | gcp | kvm | kvm2 |
+| Layer | kubernetes | gcp | kvm | kvm2 | proxmox |
-|---|---|---|---|---|
+|---|---|---|---|---|---|
-| 1. Operation spans and metrics | yes: decorators | yes: decorators | yes: decorators | yes: decorators |
+| 1. Operation spans and metrics | yes: decorators | yes: decorators | yes: decorators | yes: decorators | yes: decorators |
-| 2. Child spans per backend call | yes: `tracing.RestConfigWrapper()`, parent-gated (`cmd/main.go`, `NewWithTransportWrapper`) | yes: the SDK's own `otelhttp`; not parent-gated | **no** | yes: `kvm2.<Method>` from the `observe` decorator, `kvm2.dial`, `kvm2.download`; always under the caller's span, since every call carries the caller's ctx |
+| 2. Child spans per backend call | yes: `tracing.RestConfigWrapper()`, parent-gated (`cmd/main.go`, `NewWithTransportWrapper`) | yes: the SDK's own `otelhttp`; not parent-gated | **no** | yes: `kvm2.<Method>` from the `observe` decorator, `kvm2.dial`, `kvm2.download`; always under the caller's span, since every call carries the caller's ctx | yes: `tracing.RestConfigWrapper()` on the transport, parent-gated, one `HTTP <METHOD>` per request |
-| 3. Logs carry `traceID` and `spanID` | n/a: the package does not log | yes, including the V(5) wire log | yes: every `libvirt call` line at V(1) | yes: every `libvirt call` line at V(1), written inside its RPC's span |
+| 3. Logs carry `traceID` and `spanID` | n/a: the package does not log | yes, including the V(5) wire log | yes: every `libvirt call` line at V(1) | yes: every `libvirt call` line at V(1), written inside its RPC's span | yes: every `proxmox api call` line at V(1), with the operation span's `spanID` |
-| Tested against a real collector | yes: the kind e2e checks `provider.create` and `provider.delete` in Tempo (`test/e2e/tracing_test.go`), operation spans only | no | no | no; the integration test asserts the parent-child topology in an in-memory recorder (`assertLifecycleSpans`) |
+| Tested against a real collector | yes: the kind e2e checks `provider.create` and `provider.delete` in Tempo (`test/e2e/tracing_test.go`), operation spans only | no | no | no; the integration test asserts the parent-child topology in an in-memory recorder (`assertLifecycleSpans`) | no; `TestTracing_httpSpansUnderOperationSpan` asserts the parent-child topology in an in-memory recorder |
 ## 4. kubernetes (`internal/provider/kubernetes`)
@@ -528,20 +531,209 @@ same libvirt.
    its backing image or size.
  - The sample cloud-init installs Squid with `http_access allow all`.
-## 8. Gaps shared by the existing providers
+## 8. proxmox (`internal/provider/proxmox`)
- **Adoption is not verified in three of the four.** kubernetes, gcp and
+The full user guide is [proxmox-provider.md](proxmox-provider.md) and the
 design it was built from is
 [proxmox-provider-spec.md](proxmox-provider-spec.md); this section is the
 author's view. It is the second provider built from the contract, and the
 first on a REST API of the hypervisor's own: no SSH, no libvirt. Every
 error text it matches was captured from a PVE 9.2.20 host before the code
 was written.
 - **Package and seam.** `proxmox.go` (`Provider`, `New`, `withDefaults`,
  `node`, `wrapErr`, the per-name lock, the budgets), `create.go`,
  `get.go`, `delete.go`, `list.go` (the four methods), `api.go` (the
  seam), `client.go` (the production adapter `httpAPI`, the token and CA
  loading), `observe.go` (the logging decorator), `task.go` (task, lock
  and running waits), `seed.go`, `mark.go`, `id.go`, `state.go`,
  `errors.go`, `check.go`; the seed ISO comes from the shared
  `internal/provider/nocloud` and the name lock from
  `internal/provider/keyedlock`. The seam is `api`: one method per
  endpoint, sixteen in all, each with `ctx` first. `New` wraps `httpAPI` in
  `observe` once, so provider code never holds an undecorated seam;
  `newProvider` is the test constructor.
 - **Config.** `ProviderConfig.Proxmox`, the `ProxmoxConfig` type in
  `internal/provider/config_proxmox.go`, validated by `validateProxmox`
  (the URL shape, node names and storage ids by regexp because they end
  up in request paths, `vlanTag`, `vmidRange`, the two catalogues).
  `isoStorage` has no default on purpose: the token needs
  `Datastore.Allocate` there. Defaults (`vmidRange` 100000 to 199999,
  `requestTimeoutSeconds` 30, `installGuestAgent` true) are filled by
  `withDefaults`. `New` never makes a request; it warns when the token
  file does not exist, or when `insecureSkipTLSVerify` is on.
 - **ID.** `formatProviderID(vmid, name)` gives `vms/<vmid>/<name>`, with no
  node, because VMs migrate; the node is resolved from
  `/cluster/resources` on every call (`resolve`). `parseProviderID`
  accepts exactly three segments, the literal `vms`, a decimal VMID
  without leading zeros between 100 and 999999999 and a non-empty name,
  else `ErrPermanent`. The name is there so `Get` and `Delete` can tell
  our VM from another that later got the same VMID.
 - **Create.** `placement` checks `Zone` (`node`; optional with one node),
  `MachineType` and `Image` (both catalogue keys) before any request, and
  logs the resolved parameters with `cloudInitBytes` at V(2). Then, under
  the per-name lock, `create`: one `ClusterResources` read;
  `adoptByName` (two VMs with the name are Permanent, one goes to
  `inspect`); with none, `template` reads the template's config and
  refuses a non-template, a cloud-init drive, a missing `net0` when the
  config rewrites it, and an unsized boot disk when `diskGiB` is set, all
  before any clone; `allocate` probes `vmidFor(uid, attempt)` up to 32
  times, cloning onto a free VMID and running `inspect` on a taken one or
  on the clone's `config file already exists` answer. `inspect` waits out
  a lock (`waitUnlocked`; a clone in progress is also nameless), then
  adopts our name with our mark and a matching or empty UID, probes past
  another name, and is Permanent for our name without the mark, with
  another UID, or hibernated. An adopted VM gets one `VMStatus`: running
  returns at once. Otherwise `provision`: `uploadSeed` (rebuilt and
  overwritten every attempt), `SetVMConfig` (`configParams`: cores,
  memory, `agent=1`, the tag, `ide2`, and `net0` through `rewriteNet0`
  when a bridge or VLAN is configured), the resize task when
  `resizeTarget` says the boot disk is smaller than `diskGiB`, the start
  task, and `waitRunning` (every second up to 60 s, Transient on expiry
  or on a stop). `Create` returns only when the VM runs.
 - **Get and state.** `get` checks in order: absent from the resources is
  NotFound; a lock other than `suspended` is Transient (checked before the
  name, because a mid-clone entry has none); `status: unknown` is
  Transient; the config's `does not exist` is NotFound, a config lock
  Transient; another name or no mark is NotFound; hibernated is Stopped
  without further calls. Then `VMStatus` and `mapState(status,
  qmpstatus, ip)`:
  | `status` | `qmpstatus` | agent | Provider state |
  |---|---|---|---|
  | running | running | an IPv4 | Running |
  | running | running | 200 without an IPv4, or `QEMU guest agent is not running` | Provisioning |
  | running | running | `No QEMU guest agent configured` | Stopped |
  | running | prelaunch | not asked | Provisioning |
  | running | paused, suspended, io-error, anything else | not asked | Stopped |
  | stopped, anything else | any | not asked | Stopped |
  Any other agent answer, including `VM <vmid> is not running`, is
  Transient. `UID` and `CreatedAt` are read from the description on every
  call. A running `Get` is four calls: resources, config, status, agent.
 - **Delete.** Under the per-name lock, `destroy`: `resolve`; a lock other
  than `suspended` or `status: unknown` is Transient; the config's `does
  not exist`, another name or no mark leave the VM alone; our VM
  hibernated is Permanent, because the destroy refuses it and the lock
  never clears. Otherwise stop (unless stopped) and destroy with
  `purge=1&destroy-unreferenced-disks=1`, each a task waited on through
  `runTask`, where `does not exist` means gone. Then `deleteSeed` on the
  VM's node, where a failure fails the call (a 403 for
  `Datastore.Allocate` is Permanent), or, with no VM of ours destroyed,
  `deleteSeedEverywhere` over every configured node, where a node that
  fails is logged and skipped unless ctx ended. A missing volume still
  ends in a successful task, so no message is matched. Nothing is left
  when nil is returned.
 - **ListByTag.** One `ClusterResources` covers every node, unreachable
  ones included (their guests are listed with `status: unknown`).
  `isCandidate` keeps QEMU VMs that are not templates and carry the tag
  `proxy_operator` or a name starting `proxy-`; a nameless entry (a clone
  in progress) is not a candidate. One `VMConfig` per candidate; the mark
  in the description decides, and the `Instance` is `ID`, `UID` and
  `CreatedAt` only. A candidate whose config cannot be read is skipped at
  V(1), unless ctx ended, which fails the call; the resources call failing
  fails the whole call.
 - **Errors.** `classify(op, err)`: an error already carrying a class keeps
  it; then `apiError`s by `classifyAPI` (every 4xx Permanent; the 500
  `does not exist` NotFound, Transient when `op == "create"`; every other
  status Transient, including the lock, the taken VMID and pveproxy's 595
  and 596); TLS verification failures (`tls.CertificateVerificationError`
  and the three `x509` types) Permanent; everything else Transient. The
  500 matchers (`isNotExist`, `isLocked`, `isAlreadyExists`, the three
  agent ones) are anchored to the 500 status and each is backed by a
  captured response. `permanentf` and `transientf` classify at the
  source: placement, the template checks, the ownership check, a
  malformed token file or CA file (Permanent); a missing token or CA file,
  an expired wait, a failed task (Transient). Nothing is Quota.
 - **Check.** `Check` runs `checkNode` per node concurrently on the shared
  client. `probeNode` reads the token file for `tokenID` (no request when
  it is unusable), then `Version`, `NodeStatus`, `checkStorage` (a 500
  whatever its text, inactive, or no `iso` content is `StorageNotFound`)
  and `checkTemplates` (each catalogue VMID exists and is a template,
  else `TemplateNotFound`). `checkNode` answers `Timeout` whenever the
  round's ctx ended; otherwise `checkReason`: the two failures above,
  `errCredentials` `CredentialsUnavailable`, `errCAFile` or a TLS
  verification failure `CertificateInvalid`, 401 `Unauthorized`, 403
  `Forbidden`, 595 and 596 `Unreachable`, any other status `APIError`, a
  `net.OpError` `Unreachable` (before the deadline, so a dial out of its
  own budget is a node that did not answer), a deadline or cancellation
  `Timeout`, a cut connection `Unreachable`.
 - **Connections.** One `http.Client`, built on first use once the CA
  loads (`httpClient`), shared by every call and the Checker: dial and
  TLS handshake 10 s each, `Client.Timeout` `requestTimeoutSeconds`, 4
  idle connections per host kept 90 s, HTTP/2 disabled. A cancelled
  request ends only itself. The token file is read by `readToken` on
  every request.
 - **Logging.** Logger from the context, named `proxmox`, with `provider`,
  and `vmid`, `name` and `node` on lines about an instance. V(1): one line
  per operation outcome (`proxmox vm ensured running`, `proxmox vm
  fetched`, `proxmox vm deleted` with `destroyed`, `proxmox vms listed`,
  `proxmox node checked`), one `proxmox api call` line per seam call from
  `observe` (call, identifiers, UPID, duration, `httpStatus` and `error`),
  `proxmox task failed`, and the no-op lines. V(2): request parameters,
  config summaries, agent addresses, the resolved Create parameters.
  Cloud-init, the token, the CA and the upload body are never logged.
 - **Tracing.** No span of its own: `newHTTPAPI` wraps the transport in
  `tracing.RestConfigWrapper(opts...)`, so each request is an
  `HTTP <METHOD>` client span under the caller's span, and none without
  one. `observe` only logs, so nothing is counted twice.
 - **Tests.** Pure pieces (`id_test.go`, `mark_test.go`, `state_test.go`,
  `seed_test.go`, `errors_test.go` with the classify table over the
  captures, `api_test.go`); `client_test.go` (the real client over
  `httptest.NewTLSServer`: the token never in errors or logs, x509
  failures Permanent, cancellation, the child span); `create_test.go`,
  `get_test.go`, `delete_test.go`, `list_test.go`, `task_test.go`,
  `proxmox_test.go`, `check_test.go`, `observe_test.go` (every seam
  method logs exactly one line, by reflection) and `provider_test.go`
  (the contract §11 sweep, named test by test in the spec's entry 21).
  The double is `fakeAPI` in `fake_test.go`: VMs, tasks and ISOs in
  memory, a clone that leaves a nameless locked entry until its task
  ends, per-method errors and blocking, a call log; its error texts are
  checked against the captures. Real backend:
  `proxmox_integration_test.go` behind `//go:build integration`, driven
  by `PROXMOX_TEST_*` variables and `make test-integration-proxmox`,
  described in
  [testing.md](testing.md#the-proxmox-integration-test-real-proxmox-ve-host);
  it compiles but has not run against a host yet.
 - **Deployment.** The optional `proxmox-token` Secret (keys `token` and
  `ca.pem`) mounted at `/etc/proxy-operator/proxmox`
  (`config/manager/manager.yaml`), `make proxmox-token-secret`, a
  commented `proxmox-home` entry in both providers configs (loaded by
  `TestLoadConfig_proxmoxSampleEntry`), `config/samples/proxy_proxmox.yaml`,
  the `proxmox` type in the `status.type` comment of
  `api/v1alpha1/providerhealth_types.go`. No RBAC, and no
  `hack/providers-dev.yaml` entry, since it needs credentials.
 - **Known gaps** (see also [proxmox-provider.md § Known gaps](proxmox-provider.md#known-gaps)).
  - `Create` blocks until the VM runs.
  - A clone lock that never clears makes every Create for that Proxy wait
    out the 10-minute clone budget; `qm unlock` by hand.
  - GC blind spot: a seed ISO whose VM was destroyed outside the operator
    is invisible to GC, and a seed on node-local storage stays behind
    when its VM migrated.
  - The VMID range is shared with humans; a hand-made VM with a proxy's
    name is Permanent.
  - Cross-node clones (`target`) need shared storage and are untested.
  - A hibernated VM of ours latches its Proxy in Create and in Delete.
  - The CA is cached after first use; the token is not.
  - Never run against a real host yet; three answers were never captured
    (the storage status for a missing storage, a destroy on a hibernated
    VM, a stop on a paused VM).
 ## 9. Gaps shared by the existing providers
 - **Adoption is not verified in three of the five.** kubernetes, gcp and
  kvm adopt any existing instance with the right name without checking
-  `LabelUID`; kvm2 returns `ErrPermanent` when the UID differs.
+  `LabelUID`; kvm2 and proxmox return `ErrPermanent` when the UID differs.
- **The Create comment does not match kvm and kvm2.** The interface comment
+- **The Create comment does not match kvm, kvm2 and proxmox.** The
-  in `internal/provider/provider.go` says `Create` returns as soon as the
+  interface comment in `internal/provider/provider.go` says `Create`
-  request is submitted; both libvirt providers block until the domain has
+  returns as soon as the request is submitted; the three VM providers on
-  started.
+  your own hypervisors block until the VM has started.
 - **`Stopped` and `Terminated` are produced inconsistently.** kubernetes
-  never produces Stopped, kvm and kvm2 never produce Terminated, and the
+  never produces Stopped, kvm, kvm2 and proxmox never produce Terminated,
-  reconciler treats them the same.
+  and the reconciler treats them the same.
 - **Different `ListByTag` partial-failure policies.** gcp leaves failed
-  parts out, kvm and kvm2 fail the whole list. Both are safe.
+  parts out, kvm and kvm2 fail the whole list, and proxmox fails it when
  its one cluster-wide listing fails. Both are safe.
 - **Stale `mock` comments** in `internal/provider/provider.go`,
  `internal/provider/registry/registry.go` and `api/v1alpha1/proxy_types.go`
  (and so the generated CRD's `spec.provider` description).
@@ -555,10 +747,10 @@ same libvirt.
  log lines. kvm2 has them, but only an in-memory recorder checks their
  topology. Only kubernetes is tested against a real collector, and that
  test checks only the operation spans; nothing checks that gcp's SDK spans,
-  the kubernetes API spans or kvm2's spans nest under `provider.*` in a
+  the kubernetes API spans, kvm2's spans or proxmox's HTTP spans nest under
-  real collector.
+  `provider.*` in a real collector.
-## 9. Proposed features
+## 10. Proposed features
 Listed only. Nothing below is designed or implemented, except where a
 bullet says kvm2 already does it.
@@ -576,7 +768,7 @@ bullet says kvm2 already does it.
  backend, so a new provider gets the contract tests for free.
 - **Verify the UID when `Create` adopts an instance.** Return `ErrPermanent`
  on a mismatch instead of adopting an instance that belongs to someone else.
-  kvm2 does this; the other three do not.
+  kvm2 and proxmox do this; the other three do not.
 - **Report asynchronous create failures:** gcp keeps the operation name, and
  `Get` reports the failure instead of NotFound.
 - **Capacity or quota signals** for kvm, such as pool free space and host
@@ -14,12 +14,13 @@ the existing ones happen to work.
  open. Copy it to `docs/<type>-provider-spec.md`, fill it in, and hand it to
  the implementer with the other two files.
 - **[providers-existing.md](providers-existing.md)**: how kubernetes, gcp,
-  kvm and kvm2 meet the contract, a table mapping each of them to every
+  kvm, kvm2 and proxmox meet the contract, a table mapping each of them to every
  profile, their per-provider reference notes, shared gaps and proposed
  features.
 Per-provider user guides: [kvm-provider.md](kvm-provider.md),
 [kvm2-provider.md](kvm2-provider.md),
 [proxmox-provider.md](proxmox-provider.md),
 [gcp-vm-validation.md](gcp-vm-validation.md) and
 [gcp-in-specific-project.md](gcp-in-specific-project.md).
@@ -0,0 +1,646 @@
 # The proxmox provider
 Runs Managed proxies as VMs on a Proxmox VE cluster. It is the third
 VM-style provider and the second built from
 [provider-contract.md](provider-contract.md), from the design in
 [proxmox-provider-spec.md](proxmox-provider-spec.md). The operator talks to
 the cluster's REST API only (`https://<host>:8006/api2/json`) with an API
 token. It needs no SSH, no shell on the nodes, and no library beyond Go's
 standard `net/http`.
 Each VM is a **full clone of a template** you prepare once. The operator
 builds the Proxy's cloud-init into a NoCloud seed ISO in-process, uploads
 it to a storage on the VM's node, attaches it as a CD-ROM, sets the size,
 starts the VM and learns its address from the QEMU guest agent. Everything
 here was proven against **Proxmox VE 9.2.20** (QEMU 11.0) on the single
 node `pv1-1`; every error text the provider matches was captured from that
 host.
 ## Proxmox prerequisites
 Once per cluster, as root on a node:
 1. **Proxmox VE 9.x with the API reachable** from the operator's pods on
   port 8006. The operator's pods and the crawler clients must also route to
   the network the proxies land on (see step 5).
 2. **A template without a Proxmox cloud-init drive.** The provider brings
   its own seed on `ide2`; a Proxmox cloud-init drive would compete with it
   and the Proxy's user-data would be ignored. Debian 13's generic cloud
   image boots UEFI (under BIOS it hangs in GRUB on a headless VM), so the
   template does too:
   ```sh
   wget https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
   qm create 9001 --name debian-13-template --memory 2048 --cores 2 --ostype l26 \
     --machine q35 --bios ovmf --efidisk0 local-lvm:1,efitype=4m,pre-enrolled-keys=0 \
     --scsihw virtio-scsi-pci --net0 virtio,bridge=vmbr0 \
     --serial0 socket --vga serial0 --agent 1
   qm set 9001 --scsi0 local-lvm:0,import-from=/root/debian-13-generic-amd64.qcow2
   qm set 9001 --boot order=scsi0
   qm template 9001
   ```
   The template must boot from its first disk, run cloud-init with the
   NoCloud datasource (the Debian and Ubuntu cloud images do), and either
   ship `qemu-guest-agent` or be able to install it at first boot. Before
   the first clone for a Proxy, `Create` reads the template's config and
   refuses it with a permanent error when:
   - it is not a template (`qm template` was not run);
   - any disk slot holds a Proxmox cloud-init drive (a value containing
     `cloudinit`), such as the one `qm set <vmid> --ide2 local-lvm:cloudinit`
     adds for Proxmox's own cloud-init support;
   - it has no `net0` while the config sets `bridge` or `vlanTag`;
   - the machine type sets `diskGiB` and the template has no boot disk, or
     one without a readable `size=`.
   These are checked on the template because the clone copies its config:
   a VM cloned from a bad template could not be fixed by fixing the
   template. The template is checked only when a VM is about to be cloned;
   a VM that already exists no longer depends on it. The health check sees
   only whether each template exists and is a template.
 3. **A storage for the seed ISOs**, holding `iso` content. A dedicated
   directory storage is best, because the token needs `Datastore.Allocate`
   on it to delete seeds, and on a dedicated storage that privilege cannot
   touch backups or other ISOs:
   ```sh
   pvesm add dir proxy-seeds --path /var/lib/proxy-seeds --content iso
   ```
   A seed is tens of KiB. Proxmox stages uploads under `/var/tmp`, not the
   node's tmpfs `/tmp`. On a cluster, a storage that is not shared keeps
   each seed on the node the VM was created on (see *Known gaps*).
 4. **A user, two roles and a token.** `VM.Monitor` no longer exists in PVE
   9 and `pveum` rejects it; the agent read needs `VM.GuestAgent.Audit`
   alone. `VM.Config.Cloudinit` is not needed for the seed on `ide2`.
   `SDN.Use` on `/` is what lets the config write put a VLAN tag on
   `vmbr0`:
   ```sh
   pveum user add proxy-operator@pve --comment "egress-proxies-operator"
   pveum role add ProxyOperator -privs "VM.Allocate VM.Clone VM.Audit VM.Config.CDROM VM.Config.CPU VM.Config.Memory VM.Config.Disk VM.Config.Network VM.Config.Options VM.PowerMgmt VM.GuestAgent.Audit Datastore.Audit Datastore.AllocateSpace Datastore.AllocateTemplate Sys.Audit SDN.Use"
   pveum aclmod / -user proxy-operator@pve -role ProxyOperator
   pveum role add ProxyOperatorSeeds -privs "Datastore.Allocate Datastore.AllocateTemplate Datastore.AllocateSpace Datastore.Audit"
   pveum aclmod /storage/proxy-seeds -user proxy-operator@pve -role ProxyOperatorSeeds
   pveum user token add proxy-operator@pve operator --privsep 0
   ```
   The second role repeats three privileges of the first on purpose: an
   ACL on a more specific path **replaces** what the user inherits from
   `/` for that path, so a role with `Datastore.Allocate` alone would take
   the upload's `Datastore.AllocateTemplate` away. The role on `/` covers
   clone, config, resize, start, stop, the agent read, the upload and the
   destroy; what it lacks is `Datastore.Allocate`, without which deleting
   a seed is a 403 (`Permission check failed (/storage/<id>,
   Datastore.Allocate)`). With `--privsep 0` the token carries the user's
   privileges.
   The token's secret is printed once. Save it on your workstation as one
   line, `proxy-operator@pve!operator=<secret>`, in
   `~/.config/proxmox-token` with mode 0600. Copy the node's CA,
   `/etc/pve/pve-root-ca.pem`, to `~/.config/proxmox-ca.pem`.
 5. **A proxy network with DHCP and a route to the internet.** Each VM
   keeps the template's one NIC, optionally moved to another bridge or
   VLAN by the config, and takes its address from that network's DHCP. The
   network needs internet twice over: the vendor-data script installs
   `qemu-guest-agent` from the distro mirrors at first boot, and the agent
   is the only way the operator learns the VM's address; and an egress
   proxy needs egress anyway. Baking `qemu-guest-agent` into the template
   and setting `installGuestAgent: false` removes the mirror from IP
   discovery. The operator's pods and the clients must route to that
   network: the health engine probes `http://<ip>:<port>` through the
   proxy, and clients get the same address from the discovery API. The
   provider does not use `spec.port` and configures no Proxmox firewall.
 6. **A reserved VMID range.** VMIDs are allocated from `vmidRange`
   (default 100000 to 199999). Keep hand-made VMs out of it: one there
   costs a probe, and one named like a proxy is refused (see *What lands
   on Proxmox*).
 Check from the operator's point of view before wiring it in. This sends
 the token through a file descriptor, so it never reaches the command line:
 ```sh
 pve() {
  curl -s --cacert ~/.config/proxmox-ca.pem \
    -H @<(printf 'Authorization: PVEAPIToken=%s\n' "$(tr -d '[:space:]' < ~/.config/proxmox-token)") \
    "https://192.168.0.8:8006/api2/json/$1"; echo
 }
 pve version                                  # the token works, TLS verifies
 pve nodes/pv1-1/storage/proxy-seeds/status   # active, content includes iso
 pve 'cluster/resources?type=vm'              # the template is listed with "template":1
 ```
 ## Cluster-side setup
 1. Create the Secret. It holds two keys, `token` (the one-line token file)
   and `ca.pem` (the node's CA):
   ```sh
   make proxmox-token-secret
   # or by hand:
   kubectl -n egress-proxies-operator-system create secret generic proxmox-token \
     --from-file=token=$HOME/.config/proxmox-token \
     --from-file=ca.pem=$HOME/.config/proxmox-ca.pem
   ```
   The make target runs that command as an apply, so it also updates an
   existing Secret. Override the paths with
   `PROXMOX_TOKEN_FILE=... PROXMOX_CA_FILE=...`. Before it applies, it
   checks that both files exist, that the token file is exactly one line
   of `<user>@<realm>!<tokenid>=<secret>` (without printing it), and that
   the CA file holds a PEM certificate. With `insecureSkipTLSVerify`
   instead of a CA, create the Secret by hand with the `token` key only.
   `config/manager/manager.yaml` mounts the Secret (optional, read-only)
   at `/etc/proxy-operator/proxmox`, so the files are
   `/etc/proxy-operator/proxmox/token` and
   `/etc/proxy-operator/proxmox/ca.pem`. The deployment starts fine
   without it, even with a proxmox provider configured: `New` makes no
   request, only checks that the token file exists, and logs a `WARNING:`
   line when it does not. The provider's proxies then fail transiently and
   recover once the Secret lands; the health check reports
   `CredentialsUnavailable` meanwhile.
   **Rotating the token needs no restart.** The token file is read on
   every request, so the next request after the kubelet refreshes the
   mount uses the new token. **Rotating the CA does**: it is loaded on
   first use and cached for the life of the process. A missing CA file is
   retried until it lands; one without a certificate is a permanent error.
 2. Add a provider entry to the providers config
   (`config/manager/providers_config.yaml` in-cluster; the full sample,
   commented out, is in `config/samples/providers-config.yaml`):
   ```yaml
   - name: proxmox-home
     type: proxmox
     proxmox:
       url: https://192.168.0.8:8006
       tokenFile: /etc/proxy-operator/proxmox/token
       caFile: /etc/proxy-operator/proxmox/ca.pem
       nodes: [pv1-1]
       isoStorage: proxy-seeds
       vlanTag: 4
       templates:
         debian-13: {vmid: 9001}
       machineTypes:
         small: {cores: 1, memoryMiB: 1024}
   ```
   **The URL's host must be in the node certificate.** The node's
   self-signed certificate lists its own addresses and names, and TLS
   verification checks the URL's host against them. On `pv1-1` it lists
   `192.168.0.8`, `pv1-1` and `pv1-1.home.lab`, so
   `https://192.168.2.8:8006` reaches the same node and still fails
   verification (`CertificateInvalid` in the health check, a permanent
   error on every call). See what yours lists with:
   ```sh
   openssl s_client -connect 192.168.0.8:8006 </dev/null 2>/dev/null | openssl x509 -noout -ext subjectAltName
   ```
   If the address the operator must use is not there, either give the
   node a certificate that names it (a custom certificate in the node's
   Certificates panel) and put that certificate's CA in `caFile`, or set
   `insecureSkipTLSVerify: true`, which `New` logs as a `WARNING:` line.
   `nodes` holds node names as Proxmox knows them, not addresses: the one
   URL reaches the whole cluster. The providers config is read once per
   process, so adding a node, a template or a machine type needs a manager
   restart.
 3. Apply a Proxy (`config/samples/proxy_proxmox.yaml`) and watch it:
   ```sh
   kubectl get px -w
   # NAME                   MODE      PROVIDER       PHASE          IP              HEALTHY
   # proxy-proxmox-sample   Managed   proxmox-home   Provisioning
   # proxy-proxmox-sample   Managed   proxmox-home   Running        192.168.4.147
   # proxy-proxmox-sample   Managed   proxmox-home   Ready          192.168.4.147   True
   ```
   (Illustrative.) `Create` returns once the VM runs, a few seconds after
   the clone; the address follows when the guest agent is up, about 50
   seconds after the start on `pv1-1` with the agent installed at first
   boot.
 ## Config reference
 The block is `proxmox:` and its Go type is `ProxmoxConfig` in
 `internal/provider/config_proxmox.go`, validated at load by
 `validateProxmox`; unknown keys are rejected. Defaults are applied by
 `withDefaults` in `internal/provider/proxmox/proxmox.go`.
 | Field | Required | Default | Meaning |
 |---|---|---|---|
 | `url` | yes | | `https://<host>:8006`, with no path (the client appends `/api2/json`). The host must be in the node certificate |
 | `tokenFile` | yes | | file holding one line, `<user>@<realm>!<tokenid>=<secret>`, read on every request and sent as `Authorization: PVEAPIToken=<line>` |
 | `caFile` | | system roots | PEM bundle trusted for the API's certificate, typically the node's `/etc/pve/pve-root-ca.pem`. Loaded on first use and cached |
 | `insecureSkipTLSVerify` | | `false` | skip certificate verification; `New` logs a `WARNING:` line when it is on |
 | `nodes` | yes, at least one | | node names `placement.zone` selects from; unique, lowercase letters, digits and dashes. They end up in request paths and as the clone's `target` |
 | `isoStorage` | yes | | storage id the seeds are uploaded to and deleted from. Must hold `iso` content; the token needs `Datastore.Allocate` on it. No default on purpose |
 | `storage` | | the template's disk storage | storage id the full clone's disks go to |
 | `bridge` | | as cloned | replaces `bridge=` in the cloned `net0` |
 | `vlanTag` | | as cloned | 1 to 4094; replaces or appends `tag=` in the cloned `net0` |
 | `vmidRange.min`, `vmidRange.max` | | `100000`, `199999` | where VMIDs are allocated; at least 100, at most 999999999, min below max |
 | `templates.<key>.vmid` | at least one template | | the key names the template in `placement.image`; the VMID is at least 100 |
 | `machineTypes.<key>` | at least one | | `{cores, memoryMiB, diskGiB}` size profile. `cores` and `memoryMiB` above zero. `diskGiB` is optional: when larger than the cloned boot disk the disk is grown to it; it is never shrunk |
 | `requestTimeoutSeconds` | | `30` | bound on one API request, response included. The seed upload is one request, so a multi-megabyte `vendorData` needs more |
 | `vendorData` | | | extra cloud-init vendor-data placed on every seed next to the Proxy's user-data; a MIME multipart part beside the agent script when `installGuestAgent` is on. Cloud-init drops any top-level key of a vendor-data cloud-config that the user-data also defines |
 | `installGuestAgent` | | `true` | add the vendor-data script that installs and starts `qemu-guest-agent`. Turn it off for templates that ship the agent |
 ## Placement
 All three catalogue fields are checked before any request. An unknown
 value is `ErrPermanent`, which the reconciler latches as `Failed` with the
 message below until the spec changes.
 | Proxy field | Meaning for proxmox | Error when unknown |
 |---|---|---|
 | `placement.zone` | a node from `nodes`. May be omitted when exactly one node is configured; otherwise required | `placement.zone "x" is not in proxmox.nodes`; with several nodes and no zone, `placement.zone is required when proxmox.nodes lists more than one node` |
 | `placement.machineType` | a key in `machineTypes` | `placement.machineType "x" is not in proxmox.machineTypes` |
 | `placement.image` | a key in `templates` | `placement.image "x" is not in proxmox.templates` |
 | `placement.region` | ignored. It is still part of the spec hash, so changing it replaces the VM and does nothing else | |
 The messages come from `placement` in `internal/provider/proxmox/create.go`
 and `node` in `proxmox.go`. A key that exists but names a VMID that is
 missing or not a template fails later, before any clone:
 `placement.image "x": template VM 9001 does not exist` or `... is not a
 template`.
 **Catalogue edits affect only new VMs.** The spec hash covers the
 placement values, not what the config maps them to, so repointing
 `templates.debian-13.vmid` at another template changes the VMs created
 afterwards and leaves the running ones alone. Add a new key rather than
 repointing one in use, so the Proxies that name a key keep a meaning you
 can reason about. A template may be edited, replaced or deleted without
 affecting existing VMs: they are full clones and share nothing with it.
 ## What lands on Proxmox
 For a Proxy whose deterministic name is `proxy-abc…`:
 | Object | Name | Lifetime |
 |---|---|---|
 | VM | name `proxy-abc…`, a VMID from `vmidRange` | with the Proxy |
 | its disks | the template's disks, fully copied onto `storage` (or the template's storage) | destroyed with the VM (`purge=1&destroy-unreferenced-disks=1`) |
 | seed ISO | `<isoStorage>:iso/proxy-abc…-seed.iso` on the VM's node | with the Proxy |
 **The VMID** is derived from the Proxy's UID, so a retried Create lands on
 the same number: an FNV-1a hash of the UID placed in `vmidRange`, moving
 forward one at a time on collision, up to 32 probes (`vmidFor` in
 `id.go`). Before allocating, `Create` looks the name up in the cluster's
 resources. A VMID that is taken, or a VM that already has the name, goes
 through an ownership check (`inspect` in `create.go`): a VM with a lock
 (a clone still running, which also has no name yet) is waited on; then
 - another name: someone else's VM, so the next VMID is probed;
 - our name and our mark with our UID or an empty one: **adopted**, and
  Create carries on from where the previous attempt stopped;
 - our name without the mark, or with another UID: **permanent error**
  naming the VMID and both UIDs. The operator never takes over a VM it did
  not create;
 - our name, our mark, hibernated (`lock: suspended`): **permanent error**,
  `resume or destroy it in Proxmox` (see *Known gaps*).
 Two VMs with the proxy's name are a permanent error naming every VMID;
 remove all but one.
 **The providerID** is `vms/<vmid>/<name>`. It names no node, because VMs
 migrate: the node is looked up in `GET /cluster/resources` on every call.
 `Get` and `Delete` check that the VM at that VMID still has that name and
 the mark. Another name means ours is gone and the VMID was reused: `Get`
 answers NotFound, `Delete` succeeds without touching it.
 **The mark** is three lines in the VM's description (Notes in the UI),
 passed in the clone call itself, so a VM is never unmarked:
 ```
 proxy-operator-managed=true
 proxy-operator-uid=0f3a…
 proxy-operator-created-at=2026-09-23T17:42:11Z
 ```
 That is how orphan GC recognises its own VMs and how a repeat `Create`
 knows whether a VM of the same name belongs to this Proxy. Lines an admin
 adds to the description are kept and ignored. The clone's own
 `meta: ctime=` is useless for this: Proxmox copies it from the template.
 **The config write.** After the clone and the seed upload, one
 `PUT /nodes/{node}/qemu/{vmid}/config` sets:
 | Key | Value |
 |---|---|
 | `cores`, `memory` | from the machine type |
 | `agent` | `1` |
 | `tags` | `proxy_operator` |
 | `ide2` | `<isoStorage>:iso/<name>-seed.iso,media=cdrom` |
 | `net0` | only when `bridge` or `vlanTag` is set: the cloned value with `bridge=` and `tag=` replaced or appended, the model, MAC and every other option kept |
 The write **replaces the template's `agent` option and tags**: a
 template's `agent: 1,fstrim_cloned_disks=1` becomes `agent: 1`, and its
 tags give way to `proxy_operator`. The tag is for filtering in the
 Proxmox UI and narrows GC's candidates; the mark decides. Proxmox appends
 `ide2` to the boot order on its own (`order=scsi0;ide2`), which is
 harmless because the boot disk stays first. When the machine type's
 `diskGiB` is larger than the boot disk (the first disk in `boot: order=`
 that is not a CD-ROM), a resize task grows it; cloud-init's growpart
 expands the root filesystem at first boot.
 **The seed** (`nocloud.BuildSeedISO`, shared with kvm2 in
 `internal/provider/nocloud`) is an ISO 9660 image labelled `cidata`.
 `user-data` is the Proxy's `cloudInit` **byte for byte**; the operator
 never parses or re-renders it. `meta-data` holds `instance-id: <name>` and
 `local-hostname: <name>`. `vendor-data` carries the guest-agent script
 (a shell script, not a cloud-config, so a Proxy's own `packages:` cannot
 shadow it), joined with `vendorData` as a MIME multipart when both are
 set. It is staged under `/tmp` (the manager's emptyDir), built in memory
 and uploaded with `POST /nodes/{node}/storage/{isoStorage}/upload`. An
 upload overwrites a file of the same name, so it is rebuilt and
 re-uploaded on every Create attempt with no delete first: a previous
 attempt may have been cut off. Only the API is used; the storage upload
 endpoint does not accept `snippets`, so Proxmox's own `cicustom` would
 need shell access to the node.
 **What a retried Create finds.** Every step is safe to repeat. After a
 crash mid-clone, the retry lands on the same VMID, waits for the clone's
 lock, and adopts the VM. A stopped VM of ours gets the seed, the config
 and the resize again (the upload overwrites, the config is a replace, the
 resize is skipped once large enough), then the start. A running one is
 returned at once.
 **A clone lock that never clears.** A node that crashes mid-clone can
 leave the VM locked with `lock: clone` for good. Every Create for that
 Proxy then waits out the 10 minute clone budget and returns a transient
 error, and the reconciler keeps retrying rather than latching. Clear it
 on the node:
 ```sh
 qm unlock <vmid>      # then let the next Create adopt and finish it
 qm destroy <vmid> --purge   # or remove it and let Create clone again
 ```
 **Seeds without a VM.** A VM destroyed outside the operator leaves its
 seed ISO, which GC cannot see (it lists VMs). Seeds are named
 `proxy-*-seed.iso` on `isoStorage`; any without a matching VM may be
 removed by hand:
 ```sh
 pvesm list proxy-seeds --content iso
 pvesm free proxy-seeds:iso/proxy-abc…-seed.iso
 ```
 **Delete** runs under a per-name lock shared with Create: stop (a paused
 VM too), destroy with `purge=1&destroy-unreferenced-disks=1`, then delete
 the seed on the VM's node. Each is a task that is waited on. A VM that is
 already gone, not ours, or unmarked is left alone, and the seed is then
 looked for on every configured node, where a node that fails is logged
 and skipped. On the VM's own node a failed seed delete fails the call:
 **without `Datastore.Allocate` on `isoStorage` the 403 makes Delete
 permanent**, and the Proxy's deletion stays stuck with that message until
 the privilege is granted. When Delete returns nil, nothing of the Proxy
 is left.
 ## Logging
 The provider logs through controller-runtime's logger, named `proxmox`,
 with `provider` (the configured name) on every line, and `vmid`, `name`
 and `node` on every line about an instance. Every line inside a traced
 operation also carries `traceID` and `spanID`.
 - At startup (always on): `WARNING: token file is not readable; every
  call fails until it is` with `path` and `reason`, and `WARNING:
  insecureSkipTLSVerify is set`.
 - `--zap-log-level=debug` (logr V(1)):
  - one line per operation outcome: `proxmox vm ensured running`,
    `proxmox vm adopted` with `status` and `qmpstatus`,
    `proxmox vm fetched` with `state`, `ip`, `uid`, `status` and
    `qmpstatus`, `proxmox vm not found` with `reason`,
    `proxmox vm deleted` with `destroyed` (false when no VM of ours was
    left to destroy), `proxmox vms listed` with `vms`, `candidates`,
    `managed` and `skipped` counts, and `proxmox node checked` with
    `ready`, `reason` and `latency`;
  - **one `proxmox api call` line per API request**, written by the
    decorator in `internal/provider/proxmox/observe.go`: `call` (the seam
    method, for example `Clone`), the identifiers it was given (`node`,
    `vmid`, `storage`), `upid` for a task, `newid` and `name` on a clone,
    `filename` on an upload, `volid` on a volume delete, `duration`, and
    on failure `httpStatus` and `error`. A running `Get` is four calls; a
    fresh Create is a dozen or more, depending on how many polls each
    task takes;
  - `proxmox task failed` with `upid` and `exitstatus`, and
    `proxmox task finished with warnings`;
  - the no-op lines: `proxmox vm already gone`,
    `proxmox vm at this vmid is not ours, left alone` with `vmName`,
    `proxmox vm carries no proxy-operator mark, left alone`,
    `proxmox seed delete failed on a node, skipped`,
    `skipping unreadable proxmox vm`, and the probe's
    `proxmox vmid taken by another vm, probing on`.
 - `--zap-log-level=2`: `proxmox create resolved` with the resolved
  parameters (node, machine type, cores, memory, disk, template VMID,
  storages, bridge, VLAN tag) and `cloudInitBytes`; the `proxmox api
  call` lines also carry the config write's parameters, a summary of each
  config read, the clone's description, storage and target, the resize
  size, VM and task status, the agent's addresses and the upload's byte
  count; `proxmox guest agent answered`; `proxmox listed instance` per
  listed VM.
 - Never logged at any level: the cloud-init content, the token, the CA
  content and the upload body. There is no raw wire tier.
 ## Tracing
 With an OTLP exporter configured ([architecture.md
 §10](architecture.md#10-tracing-internaltracing)), every API request is a
 child span of the operation span the decorators open (`provider.create`,
 `provider.get`, `provider.delete`, `provider.list`) or of the monitor's
 `providerhealth.check`:
 | Span | Around | Attributes |
 |---|---|---|
 | `HTTP GET`, `HTTP POST`, `HTTP PUT`, `HTTP DELETE` | one API request, from `otelhttp` on the client's transport | otelhttp's standard HTTP client attributes: method, URL, server address and port, response status |
 The transport is `tracing.RestConfigWrapper()`, the same parent-gated
 wrapper the kubernetes provider uses, so a request without a parent span
 starts none. The logging decorator starts no span of its own, so nothing
 is counted twice; its `proxmox api call` line carries the operation
 span's `spanID`, not the HTTP span's. The token travels in a header,
 which otelhttp does not record, and no attribute carries cloud-init.
 ## Timeouts
 No caller puts a deadline on the context, so every request and every wait
 carries its own budget (`defaultBudgets` in
 `internal/provider/proxmox/proxmox.go`, and the transport in `client.go`):
 | Bound | Budget |
 |---|---|
 | TCP connect | 10 s |
 | TLS handshake | 10 s |
 | one request, response included | `requestTimeoutSeconds`, default 30 s |
 | clone task, and the wait for a VM's lock in the ownership check | 10 m |
 | upload task, resize task, start task, stop task | 2 m each |
 | destroy task | 5 m |
 | seed delete task | 1 m |
 | wait for `running` after the start | 60 s |
 | task and status poll interval | 1 s |
 A task wait that runs out is a transient error naming the UPID; the task
 keeps running on Proxmox, and the next attempt finds its result through
 the ownership check or the lock. A task that ends with any `exitstatus`
 other than `OK` or `WARNINGS: <n>` is transient too, with Proxmox's text.
 Requests share one HTTP client and its idle connections (4 per host,
 closed after 90 s idle). A cancelled or overlong request ends only
 itself: nothing else in flight is disturbed, and there is no connection
 to kill as there is for kvm2. HTTP/2 is disabled, because pveproxy speaks
 HTTP/1.1 only. Create and Delete for the same name are serialised on a
 per-name lock, so the reconciler's Delete and GC's Delete never race on
 stop and destroy; Get and ListByTag take no lock.
 ## Health check
 The operator checks every configured node every
 `--provider-check-interval` (default 1m) and publishes the result in the
 `ProviderHealth` object named after the provider
 (`kubectl get ph proxmox-home -o yaml`) and in the
 `proxy_operator_provider_target_up{provider,target}` gauge. One target per
 `nodes` entry, named after the node with the URL as its address, in
 config order, all checked concurrently.
 Each check (`checkNode` in `internal/provider/proxmox/check.go`) first
 reads the token file for its id, then makes four requests on the shared
 client: `GET /version`, `GET /nodes/{node}/status`,
 `GET /nodes/{node}/storage/{isoStorage}/status` (the storage must exist,
 be active and hold `iso` content), and `GET /cluster/resources?type=vm`
 (every configured template must exist and be a template). A check that
 runs out of time cancels only its own requests.
 Every target's `details` carry `isoStorage`. As facts arrive they gain
 `tokenID` (the part of the token before `=`, never the secret),
 `pveVersion`, `kernel`, `isoStorageAvailGiB` and `templates`
 (`<found>/<configured>`), so a target that fails late still says what the
 node answered before. There is no quota signal; the storage's free space
 is here instead.
 | Reason | Meaning | Look at |
 |---|---|---|
 | `Reachable` | ready | |
 | `CredentialsUnavailable` | the token file is missing or not `<user>@<realm>!<tokenid>=<secret>`; no request was made | the `proxmox-token` Secret; fixed in the cluster, not in Proxmox |
 | `CertificateInvalid` | TLS verification failed, or `caFile` is missing or holds no certificate | `url` (its host must be in the node certificate) or `caFile`; not the node |
 | `Unreachable` | the TCP connect or TLS handshake failed for a network reason, the connection was cut mid-response, or pveproxy answered 595 or 596 (it cannot reach the node behind it) | routing and firewalls between the cluster and `url`; `pveproxy` running; the node up |
 | `Unauthorized` | 401: the token id or secret is wrong | the token file; `pveum user token list proxy-operator@pve` |
 | `Forbidden` | 403; the message names the path and the privilege | the roles in *Proxmox prerequisites* |
 | `Timeout` | the monitor's `--provider-check-timeout` (default 15s) ran out, whichever request was running, or a request ran out of `requestTimeoutSeconds` while the round still had time | a slow or hung pveproxy, a saturated link |
 | `StorageNotFound` | `isoStorage` answered with a 500 on that node (it does not exist, is not enabled there, or is offline), is not active, or holds no `iso` content; the message carries Proxmox's text | `pvesm status`; the storage's nodes and content types |
 | `TemplateNotFound` | a configured template VMID does not exist or is not a template | `qm list`; the `templates` catalogue |
 | `APIError` | any other failure | the message carries Proxmox's own text |
 The check does not look inside the templates: a cloud-init drive, a
 missing `net0` or an unreadable disk size surface only when a Create is
 about to clone.
 ## Error classes
 Every error is classified once, in `classify` in
 `internal/provider/proxmox/errors.go`, with errors that carry a class from
 their source (validation, the token and CA files, the ownership check)
 keeping it. The reconciler latches `ErrPermanent` as `Failed` until the
 spec changes, recreates on `ErrNotFound` from `Get`, and retries
 everything else with backoff. The provider never returns
 `ErrQuotaExceeded`: the thin `local-lvm` storage overprovisions, so no
 out-of-space answer could be produced to match, and capacity failures are
 transient.
 | Class | What lands there |
 |---|---|
 | `ErrPermanent` | placement values not in the catalogue; a template VMID that is missing or not a template, has a cloud-init drive, or lacks `net0` or a sized boot disk when the config needs them; a malformed providerID; any 4xx (401 `Authentication failed!` or `No ticket`: the token; 403 `Permission check failed (<path>, <privilege>)`: a missing privilege, including `Datastore.Allocate` for the seed delete; 400 `Parameter verification failed.` with the field errors); a TLS verification failure; a token file that exists but is not `<user>@<realm>!<tokenid>=<secret>`; a CA file with no certificate; a VM with the proxy's name that carries no mark or another UID; two VMs with the proxy's name; our VM hibernated (`lock: suspended`), in both Create and Delete |
 | `ErrNotFound` | from `Get`: the VMID absent from the cluster's resources, the 500 `Configuration file 'nodes/<node>/qemu-server/<vmid>.conf' does not exist`, the VM at the VMID now named differently or unmarked. Inside `Create` the `does not exist` answer is `ErrTransient`, because the contract forbids NotFound from Create; `Delete` treats all of these as done |
 | `ErrTransient` | everything else: a missing token or CA file (the Secret has not landed yet); network errors and cancellation; 500 `VM is locked (<lock>)` and any lock seen in the resources or config; a VM whose node does not answer (`status: unknown`); the agent's answers that `Get` cannot place, such as `VM <vmid> is not running`; any other 5xx, including pveproxy's 595 and 596; a task that fails or runs out of its budget; no free VMID after 32 probes; the wait for running expiring, or a VM that stops right after its start |
 `Get` turns two agent answers into states rather than errors:
 `QEMU guest agent is not running` is Provisioning (booting, or the agent
 install has not finished), and `No QEMU guest agent configured` is
 Stopped (someone removed `agent: 1`; the reconciler replaces the VM).
 Error strings name VMIDs, nodes, storages, UPIDs and Proxmox's own text.
 They never contain the token, the CA content or cloud-init.
 ## Known gaps
 - **Create blocks** until the VM runs, holding one of the three reconcile
  workers for its duration. On `pv1-1` that is about five seconds plus the
  clone; large templates on slow storage take longer. Other Proxies keep
  reconciling on the remaining workers.
 - **A clone lock that never clears** (a node crash mid-clone) makes every
  Create for that Proxy wait out the 10 minute clone budget and return a
  transient error. Clear it with `qm unlock <vmid>` or destroy the VM (see
  *What lands on Proxmox*).
 - **Seed ISOs of VMs destroyed outside the operator are invisible to
  GC.** Remove `proxy-*-seed.iso` files without a matching VM by hand.
  On a storage that is not shared, a VM migrated to another node leaves
  its seed on the original node; Delete then looks on the VM's current
  node, finds nothing, and the seed stays. Shared ISO storage avoids it.
 - **The VMID range is shared with humans.** A VM created by hand inside
  it costs a probe; a hand-made VM named like a proxy is refused as
  `ErrPermanent`. Reserve the range.
 - **Cross-node clones need shared storage and are untested.** When the
  zone's node differs from the template's, the clone is sent with
  `target`, which Proxmox allows only when the template's disks are on
  shared storage. The host this was built on has one node.
 - **A VM whose agent was switched off is replaced**, not repaired: with
  `agent: 1` removed it can never report an address, so `Get` reports it
  Stopped.
 - **The proxy network needs internet** for the agent install, unless the
  template ships the agent and `installGuestAgent` is `false`. Without it
  the VM boots, gets an address, and stays `Provisioning` forever,
  because the agent is the only address source. Check
  `systemctl status qemu-guest-agent` and
  `/var/log/cloud-init-output.log` on the guest (`qm terminal <vmid>` on
  the node, or the console in the UI).
 - **The CA is cached** after first use; rotating it needs a manager
  restart. The token rotates freely.
 - **A node that stops answering** makes `Get` and `Delete` for its VMs
  transient until it returns, so deleting such a Proxy waits. Proxmox
  lists an unreachable node's guests with `status: unknown` rather than
  dropping them, so they are never mistaken for gone. That is Proxmox's
  documented behaviour; a single-node host could not show it.
 - **Contract limits that apply unchanged** ([provider-contract.md
  §13](provider-contract.md)): the providers config is read once per
  process; GC has no installation identity, so two operators sharing a
  Proxmox cluster and a VMID range delete each other's VMs; and a Proxy
  naming an unconfigured provider latches until its spec changes.
 Beyond the ten limits accepted in the design:
 - **A hibernated VM latches its Proxy.** A VM of ours with
  `lock: suspended` (hibernated to disk) makes both Create and Delete fail
  permanently with `resume or destroy it in Proxmox`: the lock never
  clears on its own, and the destroy refuses a locked VM. `Get` reports it
  Stopped. Resume it by starting it (`qm start <vmid>` restores the
  hibernated state) or destroy it on the node, then touch the Proxy's
  spec, or let the pending Delete retry.
 - **The seed upload is one request**, bounded by `requestTimeoutSeconds`.
  A seed is tens of KiB, but a multi-megabyte `vendorData` would need a
  larger timeout.
 - **The sample cloud-init is an open proxy.**
  `config/samples/proxy_proxmox.yaml` installs Squid with
  `http_access allow all`; restrict it before exposing the proxy network
  to anything but the cluster.
 - **Not yet proven end to end.** The unit tests run over captured
  responses from `pv1-1` and an in-memory fake; the integration tests
  below compile but have not run against a host yet. Three answers were
  never captured: the storage status for a missing storage, a destroy on
  a hibernated VM, and a stop on a paused VM.
 ## Testing
 Unit tests run over an in-memory fake of the API seam, and the HTTP
 client over a TLS test server replaying the responses captured from
 `pv1-1` (`internal/provider/proxmox/testdata/`). The optional integration
 test against a real host is described in
 [testing.md](testing.md#the-proxmox-integration-test-real-proxmox-ve-host),
 which also lists the variables it needs; run it with
 `make test-integration-proxmox`. `TestProxmoxCheck_realHost` runs the
 health check against the same host and creates nothing:
 `go test -tags=integration -count=1 -v -run TestProxmoxCheck ./internal/provider/proxmox/`.
@@ -1,8 +1,12 @@
 package provider
 import (
 	"os"
 	"slices"
 	"strings"
 	"testing"
 	"sigs.k8s.io/yaml"
 )
 const validProxmoxYAML = `
@@ -130,3 +134,117 @@ machineTypes: {small: {cores: 1, memoryMiB: 1024}}
 		})
 	}
 }
 // commentedEntry cuts the commented-out providers entry named name out of a
 // config file and strips its "# " at the entry's own indent, so a sample
 // that is shipped commented out is still loaded by a test.
 func commentedEntry(t *testing.T, path, name string) string {
 	t.Helper()
 	data, err := os.ReadFile(path)
 	if err != nil {
 		t.Fatalf("reading %s: %v", path, err)
 	}
 	var out []string
 	var prefix string
 	for line := range strings.SplitSeq(string(data), "\n") {
 		if prefix == "" {
 			if strings.TrimSpace(line) == "# - name: "+name {
 				prefix = line[:strings.Index(line, "#")]
 				out = append(out, prefix+"- name: "+name)
 			}
 			continue
 		}
 		if !strings.HasPrefix(line, prefix+"#   ") {
 			break
 		}
 		out = append(out, prefix+strings.TrimPrefix(line, prefix+"# "))
 	}
 	if prefix == "" {
 		t.Fatalf("%s has no commented entry %q", path, name)
 	}
 	return strings.Join(out, "\n") + "\n"
 }
 func TestLoadConfig_proxmoxSampleEntry(t *testing.T) {
 	t.Parallel()
 	for name, path := range map[string]string{
 		"sample":    "../../config/samples/providers-config.yaml",
 		"configmap": "../../config/manager/providers_config.yaml",
 	} {
 		t.Run(name, func(t *testing.T) {
 			t.Parallel()
 			cfg, err := LoadConfig([]byte("providers:\n" + commentedEntry(t, path, "proxmox-home")))
 			if err != nil {
 				t.Fatalf("LoadConfig(proxmox-home from %s) error = %v, want nil", path, err)
 			}
 			p := cfg.Providers[0]
 			// LoadConfig already refused a proxmox block under another type.
 			if p.Name != "proxmox-home" || p.Proxmox == nil {
 				t.Fatalf("provider %q has proxmox block %v, want proxmox-home with one", p.Name, p.Proxmox)
 			}
 			c := p.Proxmox
 			if c.URL != "https://192.168.0.8:8006" {
 				t.Errorf("url = %q, want https://192.168.0.8:8006 (the address in the node certificate)", c.URL)
 			}
 			if c.TokenFile != "/etc/proxy-operator/proxmox/token" || c.CAFile != "/etc/proxy-operator/proxmox/ca.pem" {
 				t.Errorf("tokenFile, caFile = %q, %q, want the proxmox-token Secret's mount paths", c.TokenFile, c.CAFile)
 			}
 			if c.InsecureSkipTLSVerify {
 				t.Error("insecureSkipTLSVerify = true, want it left commented out")
 			}
 			if len(c.Nodes) != 1 || c.Nodes[0] != "pv1-1" {
 				t.Errorf("nodes = %v, want [pv1-1]", c.Nodes)
 			}
 			if c.ISOStorage != "proxy-seeds" || c.Storage != "" || c.Bridge != "" || c.VLANTag != 4 {
 				t.Errorf("isoStorage, storage, bridge, vlanTag = %q, %q, %q, %d, want proxy-seeds, \"\", \"\", 4", c.ISOStorage, c.Storage, c.Bridge, c.VLANTag)
 			}
 			if len(c.Templates) != 1 || c.Templates["debian-13"].VMID != 9001 {
 				t.Errorf("templates = %v, want {debian-13: {vmid: 9001}}", c.Templates)
 			}
 			if mt, ok := c.MachineTypes["small"]; !ok || mt.Cores != 1 || mt.MemoryMiB != 1024 || mt.DiskGiB != 0 {
 				t.Errorf("machineTypes[small] = %+v (present %v), want {1 1024 0}", mt, ok)
 			}
 			if c.VMIDRange != nil || c.RequestTimeoutSeconds != 0 || c.InstallGuestAgent != nil || c.VendorData != "" {
 				t.Errorf("optional fields set: %+v, want vmidRange, requestTimeoutSeconds, installGuestAgent and vendorData left commented out", c)
 			}
 		})
 	}
 }
 func TestLoadConfig_proxmoxSampleProxyMatchesSampleEntry(t *testing.T) {
 	t.Parallel()
 	cfg, err := LoadConfig([]byte("providers:\n" + commentedEntry(t, "../../config/samples/providers-config.yaml", "proxmox-home")))
 	if err != nil {
 		t.Fatalf("LoadConfig() error = %v, want nil", err)
 	}
 	data, err := os.ReadFile("../../config/samples/proxy_proxmox.yaml")
 	if err != nil {
 		t.Fatalf("reading the sample Proxy: %v", err)
 	}
 	var px struct {
 		Spec struct {
 			Provider  string `json:"provider"`
 			Placement struct {
 				Zone        string `json:"zone"`
 				MachineType string `json:"machineType"`
 				Image       string `json:"image"`
 			} `json:"placement"`
 		} `json:"spec"`
 	}
 	if err := yaml.Unmarshal(data, &px); err != nil {
 		t.Fatalf("parsing the sample Proxy: %v", err)
 	}
 	c := cfg.Providers[0].Proxmox
 	if px.Spec.Provider != cfg.Providers[0].Name {
 		t.Errorf("spec.provider = %q, want %q", px.Spec.Provider, cfg.Providers[0].Name)
 	}
 	if !slices.Contains(c.Nodes, px.Spec.Placement.Zone) {
 		t.Errorf("placement.zone = %q, not in nodes %v", px.Spec.Placement.Zone, c.Nodes)
 	}
 	if _, ok := c.MachineTypes[px.Spec.Placement.MachineType]; !ok {
 		t.Errorf("placement.machineType = %q, not in machineTypes", px.Spec.Placement.MachineType)
 	}
 	if _, ok := c.Templates[px.Spec.Placement.Image]; !ok {
 		t.Errorf("placement.image = %q, not in templates", px.Spec.Placement.Image)
 	}
 }