proxmox: a Proxmox VE provider built from the provider contract #9
@@ -226,3 +226,6 @@ A lightweight `check` job (`go vet` / `go build` / `go test -short`) gates the b
|
||||
- `+kubebuilder:rbac` markers are package-level. Keep them in their own comment block,
|
||||
never inside a type's or function's doc comment: controller-gen silently ignores
|
||||
them there. After `make manifests`, check `git diff config/rbac/role.yaml`.
|
||||
- `make test-integration-proxmox` runs the proxmox provider's integration tests against
|
||||
a real Proxmox VE host. They skip unless `PROXMOX_TEST_URL`, `_TOKEN_FILE`, `_NODE`,
|
||||
`_TEMPLATE_VMID` and `_ISO_STORAGE` are set; see docs/testing.md for the full env block.
|
||||
|
||||
@@ -200,6 +200,23 @@ kvm-ssh-secret: ## Create or update the kvm-ssh Secret from KVM_SSH_KEY (private
|
||||
--from-file=known_hosts="$(KVM_KNOWN_HOSTS)" \
|
||||
--dry-run=client -o yaml | "$(KUBECTL)" apply -f -
|
||||
|
||||
PROXMOX_TOKEN_FILE ?= $(HOME)/.config/proxmox-token
|
||||
PROXMOX_CA_FILE ?= $(HOME)/.config/proxmox-ca.pem
|
||||
PROXMOX_NAMESPACE ?= egress-proxies-operator-system
|
||||
|
||||
.PHONY: proxmox-token-secret
|
||||
proxmox-token-secret: ## Create or update the proxmox-token Secret from PROXMOX_TOKEN_FILE (user@realm!tokenid=secret) and PROXMOX_CA_FILE.
|
||||
@test -f "$(PROXMOX_TOKEN_FILE)" || { echo "PROXMOX_TOKEN_FILE=$(PROXMOX_TOKEN_FILE) not found"; exit 1; }
|
||||
@test -f "$(PROXMOX_CA_FILE)" || { echo "PROXMOX_CA_FILE=$(PROXMOX_CA_FILE) not found"; exit 1; }
|
||||
@test "$$(grep -cv '^[[:space:]]*$$' "$(PROXMOX_TOKEN_FILE)")" = 1 && \
|
||||
grep -qxE '[[:space:]]*[^@[:space:]]+@[^![:space:]]+![^=[:space:]]+=[^[:space:]]+[[:space:]]*' "$(PROXMOX_TOKEN_FILE)" || \
|
||||
{ echo "PROXMOX_TOKEN_FILE=$(PROXMOX_TOKEN_FILE) is not one line of user@realm!tokenid=secret"; exit 1; }
|
||||
@grep -q 'BEGIN CERTIFICATE' "$(PROXMOX_CA_FILE)" || { echo "PROXMOX_CA_FILE=$(PROXMOX_CA_FILE) holds no PEM certificate"; exit 1; }
|
||||
"$(KUBECTL)" -n $(PROXMOX_NAMESPACE) create secret generic proxmox-token \
|
||||
--from-file=token="$(PROXMOX_TOKEN_FILE)" \
|
||||
--from-file=ca.pem="$(PROXMOX_CA_FILE)" \
|
||||
--dry-run=client -o yaml | "$(KUBECTL)" apply -f -
|
||||
|
||||
.PHONY: deploy
|
||||
deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
|
||||
cd config/manager && "$(KUSTOMIZE)" edit set image controller=${IMG}
|
||||
|
||||
@@ -192,6 +192,7 @@ can be two differently-configured instances of the same `gcp` type (see
|
||||
| `gcp` | A Compute Engine VM with an ephemeral external IP | `project` (required), `network`, `networkTag`, `diskSizeGb` | The real egress fleet. Deliberately uses only four API calls (Insert / Get / Delete / AggregatedList), all fire-and-forget: `Create` returns as soon as the operation is submitted and the reconciler discovers progress by polling `Get`. Auth is Application Default Credentials — workload identity in-cluster, `gcloud` ADC locally; no key-file plumbing. |
|
||||
| `kvm` | A libvirt VM on one of your own hypervisors, reached over SSH | `sshKeyFile`, `knownHostsFile` (unless `insecureIgnoreHostKey`), `machineTypes`, `hosts` (required); `images`, `diskSizeGb`, `vendorData`, `installGuestAgent` | Home-lab or on-prem egress. The base image is downloaded into each host's pool on first use; `Create` waits for the VM to start. See [docs/kvm-provider.md](docs/kvm-provider.md). |
|
||||
| `kvm2` | The same: a libvirt VM on one of your own hypervisors, reached over SSH | the same block shape as `kvm`, under `kvm2:` | The second-generation libvirt provider, built from the provider contract. Adds a child span per libvirt RPC, a pool-space precheck reported as quota (the reconciler waits instead of failing), SSH credentials re-read on every dial, and image downloads and health checks on connections of their own. It does not adopt `kvm`'s VMs. See [docs/kvm2-provider.md](docs/kvm2-provider.md). |
|
||||
| `proxmox` | A VM on a Proxmox VE cluster: a full clone of a template you prepare, reached through the Proxmox REST API with an API token | `url`, `tokenFile`, `nodes`, `isoStorage`, `templates`, `machineTypes` (required); `caFile` or `insecureSkipTLSVerify`, `storage`, `bridge`, `vlanTag`, `vmidRange`, `requestTimeoutSeconds`, `vendorData`, `installGuestAgent` | Home-lab or on-prem egress without SSH. Cloud-init goes on a seed ISO uploaded through the API; the address comes from the QEMU guest agent, so the proxy network needs a route to a package mirror unless the template ships the agent. `Create` waits for the VM to run. The token is read on every request, from the optional `proxmox-token` Secret (`make proxmox-token-secret`). Built from the provider contract; not yet run against a real host end to end. See [docs/proxmox-provider.md](docs/proxmox-provider.md). |
|
||||
|
||||
### Adding a provider
|
||||
|
||||
|
||||
@@ -166,6 +166,18 @@ spec:
|
||||
- name: kvm-ssh
|
||||
mountPath: /etc/proxy-operator/kvm
|
||||
readOnly: true
|
||||
# API token and CA for the proxmox provider (tokenFile / caFile in
|
||||
# the providers config point here). Optional: the mount is empty when
|
||||
# the Secret does not exist, and only proxmox entries need it. The
|
||||
# token file is read on every request; the CA once, on first use.
|
||||
# Create it with:
|
||||
# kubectl -n egress-proxies-operator-system create secret generic proxmox-token \
|
||||
# --from-file=token=<file holding user@realm!tokenid=secret> \
|
||||
# --from-file=ca.pem=<the node's /etc/pve/pve-root-ca.pem>
|
||||
# or `make proxmox-token-secret`.
|
||||
- name: proxmox-token
|
||||
mountPath: /etc/proxy-operator/proxmox
|
||||
readOnly: true
|
||||
# Scratch space: the root filesystem is read-only, and the kvm
|
||||
# provider builds each VM's cloud-init seed ISO through a temp
|
||||
# directory (a few hundred KB, deleted right after upload).
|
||||
@@ -180,6 +192,11 @@ spec:
|
||||
secret:
|
||||
secretName: kvm-ssh
|
||||
optional: true
|
||||
# The proxmox provider's token and CA; see the mount above.
|
||||
- name: proxmox-token
|
||||
secret:
|
||||
secretName: proxmox-token
|
||||
optional: true
|
||||
- name: tmp
|
||||
emptyDir:
|
||||
sizeLimit: 64Mi
|
||||
|
||||
@@ -84,4 +84,32 @@ data:
|
||||
bridge: br0
|
||||
# socket: /var/run/libvirt/libvirt-sock
|
||||
# pool: default
|
||||
# proxmox clones a prepared template on a Proxmox VE cluster through its
|
||||
# REST API and seeds it with a NoCloud ISO uploaded to isoStorage; it needs
|
||||
# no SSH. Both files come from the optional proxmox-token Secret mounted by
|
||||
# config/manager/manager.yaml. The url's host must be a name or address in
|
||||
# the node certificate. See docs/proxmox-provider.md.
|
||||
# - name: proxmox-home
|
||||
# type: proxmox
|
||||
# proxmox:
|
||||
# url: https://192.168.0.8:8006
|
||||
# tokenFile: /etc/proxy-operator/proxmox/token
|
||||
# caFile: /etc/proxy-operator/proxmox/ca.pem
|
||||
# # insecureSkipTLSVerify: true # instead of caFile; logged as a warning
|
||||
# nodes: [pv1-1] # placement.zone names a node; optional with one node
|
||||
# isoStorage: proxy-seeds # iso content; the token needs Datastore.Allocate here
|
||||
# # storage: local-lvm # clone target; default is the template's disk storage
|
||||
# # bridge: vmbr0 # replaces the cloned net0's bridge
|
||||
# vlanTag: 4 # set on the cloned net0, MAC kept
|
||||
# # vmidRange: {min: 100000, max: 199999}
|
||||
# # requestTimeoutSeconds: 30
|
||||
# # installGuestAgent: false
|
||||
# # vendorData: |
|
||||
# # #cloud-config
|
||||
# # ...
|
||||
# templates: # placement.image names a key; add keys, never repoint one
|
||||
# debian-13: {vmid: 9001}
|
||||
# machineTypes: # placement.machineType names a key here
|
||||
# small: {cores: 1, memoryMiB: 1024}
|
||||
# medium: {cores: 2, memoryMiB: 2048, diskGiB: 10}
|
||||
|
||||
|
||||
@@ -9,5 +9,6 @@ resources:
|
||||
- proxy_gcp.yaml
|
||||
- proxy_kvm.yaml
|
||||
- proxy_kvm2.yaml
|
||||
- proxy_proxmox.yaml
|
||||
- proxy_external.yaml
|
||||
# +kubebuilder:scaffold:manifestskustomizesamples
|
||||
|
||||
@@ -88,3 +88,31 @@ providers:
|
||||
# bridge: br0
|
||||
# # socket: /var/run/libvirt/libvirt-sock
|
||||
# # pool: default
|
||||
# proxmox clones a prepared template on a Proxmox VE cluster through its
|
||||
# REST API and seeds it with a NoCloud ISO uploaded to isoStorage; it needs
|
||||
# no SSH. Both files come from the optional proxmox-token Secret mounted by
|
||||
# config/manager/manager.yaml. The url's host must be a name or address in
|
||||
# the node certificate. See docs/proxmox-provider.md.
|
||||
# - name: proxmox-home
|
||||
# type: proxmox
|
||||
# proxmox:
|
||||
# url: https://192.168.0.8:8006
|
||||
# tokenFile: /etc/proxy-operator/proxmox/token
|
||||
# caFile: /etc/proxy-operator/proxmox/ca.pem
|
||||
# # insecureSkipTLSVerify: true # instead of caFile; logged as a warning
|
||||
# nodes: [pv1-1] # placement.zone names a node; optional with one node
|
||||
# isoStorage: proxy-seeds # iso content; the token needs Datastore.Allocate here
|
||||
# # storage: local-lvm # clone target; default is the template's disk storage
|
||||
# # bridge: vmbr0 # replaces the cloned net0's bridge
|
||||
# vlanTag: 4 # set on the cloned net0, MAC kept
|
||||
# # vmidRange: {min: 100000, max: 199999}
|
||||
# # requestTimeoutSeconds: 30
|
||||
# # installGuestAgent: false
|
||||
# # vendorData: |
|
||||
# # #cloud-config
|
||||
# # ...
|
||||
# templates: # placement.image names a key; add keys, never repoint one
|
||||
# debian-13: {vmid: 9001}
|
||||
# machineTypes: # placement.machineType names a key here
|
||||
# small: {cores: 1, memoryMiB: 1024}
|
||||
# medium: {cores: 2, memoryMiB: 2048, diskGiB: 10}
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
# A Managed proxy backed by a Proxmox VE cluster through the proxmox
|
||||
# provider: the operator full-clones a prepared template on the named node,
|
||||
# attaches a NoCloud seed ISO carrying this cloud-init, and starts the VM on
|
||||
# the configured network (here VLAN 4 on vmbr0, IP from that network's
|
||||
# DHCP), where cloud-init installs Squid. Requires a providers-config entry
|
||||
# named "proxmox-home" (see providers-config.yaml) and the proxmox-token
|
||||
# Secret (docs/proxmox-provider.md).
|
||||
#
|
||||
# placement.zone is a node name from proxmox.nodes (optional when only one
|
||||
# node is configured); machineType and image are keys in the provider's
|
||||
# machineTypes and templates maps. The address comes from the guest agent,
|
||||
# which a vendor-data script installs at first boot, so the VM's network
|
||||
# needs a route to a package mirror.
|
||||
apiVersion: crawl.example.com/v1alpha1
|
||||
kind: Proxy
|
||||
metadata:
|
||||
name: proxy-proxmox-sample
|
||||
spec:
|
||||
mode: Managed
|
||||
provider: proxmox-home
|
||||
placement:
|
||||
zone: pv1-1
|
||||
machineType: small
|
||||
image: debian-13
|
||||
port: 3128
|
||||
cloudInit:
|
||||
inline: |
|
||||
#cloud-config
|
||||
packages:
|
||||
- squid
|
||||
write_files:
|
||||
- path: /etc/squid/conf.d/proxy-operator.conf
|
||||
content: |
|
||||
http_port 3128
|
||||
http_access allow all
|
||||
via off
|
||||
forwarded_for off
|
||||
runcmd:
|
||||
- systemctl restart squid
|
||||
# Optional: log in with `ssh debian@<ip>` (the Debian image's default
|
||||
# user) to look around; `qm terminal <vmid>` on the node works too.
|
||||
# ssh_authorized_keys:
|
||||
# - ssh-ed25519 AAAA... you@laptop
|
||||
attributes:
|
||||
geo: home
|
||||
purpose: crawl
|
||||
+66
-1
@@ -11,6 +11,7 @@
|
||||
| gcp provider | `internal/provider/gcp` | library | — | Compute Engine VMs, four API calls, fire-and-forget ops |
|
||||
| kvm provider | `internal/provider/kvm` | library | — | VMs on libvirt hosts over SSH-forwarded sockets; image cache, overlay disks, NoCloud seeds |
|
||||
| kvm2 provider | `internal/provider/kvm2` | library | — | the same backend, built clean-room from the provider contract as a second type: per-RPC child spans, pool-space quota, per-call budgets, its own metadata namespace, credentials re-read on every dial |
|
||||
| proxmox provider | `internal/provider/proxmox` | library | — | VMs on a Proxmox VE cluster through its REST API: full clones of a template, a NoCloud seed uploaded through the API, the VMID derived from the Proxy UID, every task waited on, HTTP child spans |
|
||||
| Shared provider helpers | `internal/provider/nocloud`, `internal/provider/keyedlock` | library | — | the NoCloud seed ISO and vendor-data builder, and one-slot semaphores by key; used by kvm2 and proxmox |
|
||||
| Health engine | `internal/health` | Runnable | yes | through-the-proxy probes, thresholds, transition events |
|
||||
| Lease store | `internal/lease` | Runnable (expiry sweep) | no | in-memory leases + cooldowns, single mutex |
|
||||
@@ -204,6 +205,37 @@ outlives its ctx or budget kills the pooled connection (the only way to unblock
|
||||
go-libvirt), so other calls in flight on that host fail once with Transient and
|
||||
retry; the next call redials. Each RPC is a child span kvm2.<Method> under
|
||||
provider.*, plus kvm2.dial and kvm2.download.
|
||||
|
||||
proxmox provider (internal/provider/proxmox/) - the Proxmox VE REST API, one URL per cluster
|
||||
prov.Create ──► placement checked against the catalogue (Permanent before any request)
|
||||
GET /cluster/resources: a VM with the name → ownership check ─┐ every step
|
||||
else: template config checked (not a template, cloud-init drive, │ idempotent,
|
||||
no net0 or boot disk it needs = Permanent) │ under a
|
||||
VMID = hash(UID) in vmidRange, probing on; a taken VMID → │ per-name lock
|
||||
ownership check: wait out a lock, adopt our name + mark + │ shared with
|
||||
UID, probe past another name, Permanent otherwise │ Delete; each
|
||||
POST .../clone (full, description = the mark) → wait task │ task's UPID
|
||||
upload seed ISO to isoStorage (overwrites) → wait task │ polled every
|
||||
PUT .../config: cores, memory, agent=1, tag, ide2 = seed, │ 1s to its end
|
||||
net0 bridge/tag when configured │
|
||||
resize if diskGiB is larger → wait task │
|
||||
start → wait task → poll status/current until running (60s) ─┘
|
||||
prov.Get ──► /cluster/resources (absent = NotFound; lock or node unknown = Transient)
|
||||
+ config (name and mark must match, else NotFound: VMID reused)
|
||||
+ status/current (paused and hibernated = Stopped)
|
||||
+ agent network-get-interfaces (first routable IPv4; agent not
|
||||
up yet = Provisioning)
|
||||
prov.Delete ──► resources + config: gone, not ours or unmarked = leave it;
|
||||
hibernated = Permanent; stop → destroy (purge) → delete the seed
|
||||
on the VM's node (or on every node when no VM of ours was left)
|
||||
prov.ListByTag ─► one /cluster/resources for the whole cluster, one config read per
|
||||
candidate (tag or proxy- name); the description's mark decides
|
||||
Check ──► per node: /version, node status, isoStorage status, templates in
|
||||
/cluster/resources
|
||||
providerID = vms/<vmid>/<name>: no node, because VMs migrate; the node is looked
|
||||
up in /cluster/resources on every call. Token read from its file on every
|
||||
request. Each request is an HTTP <METHOD> child span from the parent-gated
|
||||
transport wrapper; a cancelled request ends only itself.
|
||||
```
|
||||
|
||||
The reconciler never watches provider-side resources (Pods or GCP VMs).
|
||||
@@ -388,6 +420,12 @@ incoming trace context):
|
||||
from a span decorator over the seam, applied once in the dialer, so no
|
||||
RPC is unspanned. The `libvirt call` log line of an RPC is written
|
||||
inside its span. The `kvm` provider has no child spans.
|
||||
- proxmox child spans: one `HTTP <METHOD>` client span per API request,
|
||||
from `tracing.RestConfigWrapper()` on the provider's own HTTP transport,
|
||||
under `provider.*` or `providerhealth.check`. Parent-gated like the
|
||||
Kubernetes API spans, so a request without a parent starts none. The
|
||||
provider's logging decorator starts no span; its `proxmox api call` line
|
||||
carries the operation span's IDs.
|
||||
- Kubernetes API calls become client child spans via a wrapped
|
||||
`rest.Config` transport — gated on an existing parent span, so informer
|
||||
list/watch long-polls and leader-election renewals never create root
|
||||
@@ -413,12 +451,13 @@ the slog handler's ctx.
|
||||
Answers "can this cluster reach each provider's backend right now?"
|
||||
without waiting for a Proxy to fail. A provider opts in by implementing
|
||||
`provider.Checker`: `Check(ctx) []TargetStatus` returns one verdict per
|
||||
target and never fails as a whole. All four providers do:
|
||||
target and never fails as a whole. All five providers do:
|
||||
|
||||
| Provider | Target | What one check does |
|
||||
|---|---|---|
|
||||
| kvm | each `kvm.hosts` entry | over the pooled SSH+libvirt connection: libvirt version and hostname, then the configured pool must exist and be running; reports pool capacity and free space ([details](kvm-provider.md#health-check)) |
|
||||
| kvm2 | each `kvm2.hosts` entry | on a fresh SSH+libvirt connection dialled for the check and closed after it: libvirt version and hostname, then the configured pool must exist and be running; `details` carry `user`, `pool`, `hostname`, `libvirtVersion`, `poolState`, `poolCapacityGiB` and `poolAvailableGiB` ([details](kvm2-provider.md#health-check)) |
|
||||
| proxmox | each `proxmox.nodes` entry | four requests on the shared HTTP client: API version, node status, the `isoStorage` status (it must be active and hold `iso` content), and the cluster's resources (every configured template must exist and be a template); `details` carry `isoStorage`, `tokenID`, `pveVersion`, `kernel`, `isoStorageAvailGiB` and `templates` ([details](proxmox-provider.md#health-check)) |
|
||||
| kubernetes | `apiserver` | lists at most one managed Pod cluster-wide: the verb and scope `ListByTag` and GC rely on |
|
||||
| gcp | the project | one page of an aggregated instance list with `maxResults=1`: credentials, Compute API enabled, `compute.instances.list` |
|
||||
|
||||
@@ -704,3 +743,29 @@ they read as choices, not accidents. Chronological by build step.
|
||||
the Secret needs no restart; nothing is cached. The constructor only
|
||||
checks that the files exist, to warn early without failing the provider
|
||||
set.
|
||||
- **proxmox derives the VMID from the Proxy UID and checks ownership
|
||||
before using a taken one.** Proxmox addresses VMs by VMID, and names are
|
||||
not unique, so the name cannot be the key. A hash of the UID placed in
|
||||
`vmidRange` makes a retry land on the same VMID; a taken one is waited
|
||||
on while locked, then adopted when it has our name, mark and UID,
|
||||
probed past when it has another name, and refused otherwise. The wait
|
||||
is what stops a retry after a crash mid-clone from cloning a duplicate:
|
||||
a clone in progress has no name yet, so the by-name lookup misses it.
|
||||
- **proxmox writes its mark into the VM's description**, in the clone
|
||||
call itself, so no VM of ours exists unmarked. Tags cannot be passed to
|
||||
the clone, and the clone's own creation time is copied from the
|
||||
template. The `proxy_operator` tag is set afterwards for the Proxmox UI
|
||||
and to narrow GC's candidates; the mark decides.
|
||||
- **proxmox delivers cloud-init as a seed ISO uploaded through the API**,
|
||||
not as Proxmox's own cloud-init drive or a `cicustom` snippet. The
|
||||
upload endpoint does not accept snippets, and the operator has no shell
|
||||
on the nodes; the seed code is kvm2's, extracted into
|
||||
`internal/provider/nocloud`. Templates must therefore carry no Proxmox
|
||||
cloud-init drive, and seeds of VMs destroyed outside the operator are a
|
||||
documented GC blind spot.
|
||||
- **proxmox resolves a VM's node on every call** from
|
||||
`/cluster/resources`, and its providerID names none, because VMs migrate
|
||||
between nodes. The cost is one extra request per operation; the gain is
|
||||
that a migrated VM stays addressable and the listing, which includes the
|
||||
guests of an unreachable node as `status: unknown`, never mistakes a
|
||||
down node for a deleted VM.
|
||||
|
||||
@@ -13,7 +13,7 @@ Pairs with [docs/plans/2026-09-18-1502-proxmox-provider.md](../plans/2026-09-18-
|
||||
- [x] Phase 6 — Create
|
||||
- [x] Phase 7 — Delete, Checker, CRD comment
|
||||
- [x] Phase 8 — Contract and observability sweep, integration test, testing.md
|
||||
- [ ] Phase 9 — Deployment and documentation
|
||||
- [x] Phase 9 — Deployment and documentation
|
||||
- [ ] Phase 10 — Review, real-host validation, changelog, MR
|
||||
|
||||
## Phase 0 — Plan file, branch, host recon, fixtures
|
||||
@@ -1121,3 +1121,180 @@ Worth noting:
|
||||
`go test -short ./...` run of this phase.
|
||||
- `.claude/settings.json` stays out of the commit. Nothing in this phase
|
||||
talked to the host.
|
||||
|
||||
## Phase 9 — Deployment and documentation
|
||||
|
||||
Deployment: the optional `proxmox-token` Secret (keys `token` and `ca.pem`)
|
||||
is mounted read-only at `/etc/proxy-operator/proxmox` in
|
||||
`config/manager/manager.yaml`, next to `kvm-ssh` and commented the same way.
|
||||
The Makefile gains `proxmox-token-secret`. Both providers configs gain a
|
||||
commented `proxmox-home` entry after kvm2's. `config/samples/proxy_proxmox.yaml`
|
||||
is listed in the samples kustomization. There is no RBAC change and no
|
||||
`hack/providers-dev.yaml` entry. Docs: `docs/proxmox-provider.md` follows
|
||||
kvm2's guide heading for heading and has no `—`. The rows and sections
|
||||
named below went into `providers-existing.md`, `architecture.md`, the
|
||||
README, `providers.md` and `CLAUDE.md`. `docs/testing.md` and the
|
||||
integration Makefile target were done in Phase 8 and were not touched; the
|
||||
guide's Testing section links to them.
|
||||
|
||||
Where the spec and the plan's Phase 9 text disagree, the spec won. The URL
|
||||
is `https://192.168.0.8:8006`, not 192.168.2.8, which is not in the node
|
||||
certificate. The Secret has two keys, and the target takes
|
||||
`PROXMOX_CA_FILE` as well as `PROXMOX_TOKEN_FILE`.
|
||||
|
||||
Judgment calls:
|
||||
|
||||
- **The target also checks both files' content.** The token file must be
|
||||
exactly one non-blank line matching
|
||||
`<user>@<realm>!<tokenid>=<secret>`. That is the shape `readToken`
|
||||
accepts: `TrimSpace`, then an anchored regexp. The check uses `grep -c`
|
||||
and `grep -qx`, so the secret is never printed. The CA check mirrors
|
||||
`kvm-ssh-secret`'s `PRIVATE KEY` check: the file must contain
|
||||
`BEGIN CERTIFICATE`. With `insecureSkipTLSVerify` the guide says to
|
||||
create the Secret by hand with the `token` key only.
|
||||
- **The sample-entry test covers both files.** `commentedEntry` finds the
|
||||
`# - name: proxmox-home` line and takes its indent. It keeps the lines
|
||||
that continue at `<indent># ` and strips `# ` at that indent. The same
|
||||
helper then works for the ConfigMap, whose entry sits six spaces deep
|
||||
inside a `|` block. So `TestLoadConfig_proxmoxSampleEntry` has a
|
||||
`sample` subtest and a `configmap` subtest. It checks every field the
|
||||
spec names, and that the optional fields stay commented out.
|
||||
- **A second test goes beyond the plan:**
|
||||
`TestLoadConfig_proxmoxSampleProxyMatchesSampleEntry`. The sample Proxy's
|
||||
provider, zone, machine type and image must resolve in the sample
|
||||
entry. kvm has exactly this bug as a known gap (`proxy_kvm.yaml` places
|
||||
on `nas1`).
|
||||
- **The sample entry also has `medium: {cores: 2, memoryMiB: 2048,
|
||||
diskGiB: 10}`**, next to the `small` the spec asks for, so `diskGiB`
|
||||
appears somewhere. The optional fields are commented out with a short
|
||||
note each, because no live proxmox entry exists to explain them. The
|
||||
kvm2 entry can rely on kvm's.
|
||||
- **In the ConfigMap the proxmox entry is commented out**, although kvm2's
|
||||
is live there. It needs a host and a token.
|
||||
- **providers-existing.md needed more than the column and the section.**
|
||||
The intro now says five providers and links the guide. In the shared
|
||||
gaps, six sentences would otherwise be false, and each now names
|
||||
proxmox where it applies: adoption verified in two of the five, Create
|
||||
blocking in all three VM providers, no Terminated, the list policy, the
|
||||
collector gap, and the "kvm2 does this" in proposed features. The new
|
||||
section is §8, so the shared gaps and proposed features move to §9 and
|
||||
§10. No doc links those sections by anchor (grep).
|
||||
- **architecture.md §11 now says "All five providers do".** The sentence
|
||||
counts Checkers, and proxmox has one. The four Decisions bullets are
|
||||
entry 23's: the VMID from the UID with the ownership check, the mark in
|
||||
the description, the seed uploaded through the API, and the node
|
||||
resolved per call.
|
||||
- **The guide lists the ten design limits first, then four more.** They
|
||||
are the hibernated-VM latch, the one-request seed upload, the open-proxy
|
||||
sample (kvm2's guide lists it too), and "not yet proven end to end" with
|
||||
the three answers never captured. The Phase 6 to 8 guide items are
|
||||
each in the section where a reader looks for them. The config write
|
||||
replacing `agent` and `tags` is in *What lands on Proxmox*, and so are
|
||||
the seed cleanup, `qm unlock` and the 403 that makes Delete Permanent.
|
||||
The certificate rule is in *Cluster-side setup*. CA caching versus
|
||||
token rotation is in both.
|
||||
- **The guide resumes a hibernated VM with `qm start`, not
|
||||
`qm resume`.** A VM suspended to disk resumes from its state file on
|
||||
its next start; `qm resume` is for a VM paused in RAM.
|
||||
- **The guide's `pvesm add dir proxy-seeds --path /var/lib/proxy-seeds
|
||||
--content iso` is the standard form, not a recorded command.** Phase 0
|
||||
says only that the user created `proxy-seeds` as a `dir` storage with
|
||||
ISO content; the path is illustrative.
|
||||
|
||||
The token-file check, run without touching a cluster: `KUBECTL=echo`
|
||||
prints only the apply stage, and `KUBECTL=true` runs nothing. Scratch
|
||||
files covered one good line, two lines, a bare secret, blank lines around
|
||||
a good line, a non-PEM CA and a missing file. The real files on this
|
||||
workstation pass:
|
||||
|
||||
```bash
|
||||
make -n proxmox-token-secret
|
||||
make -s proxmox-token-secret PROXMOX_TOKEN_FILE=$D/<case> PROXMOX_CA_FILE=$D/ca KUBECTL=echo
|
||||
# good, ws: "apply -f -"; two, bad: "... is not one line of user@realm!tokenid=secret"; noca: "... holds no PEM certificate"
|
||||
make -s proxmox-token-secret KUBECTL=true && echo "real files pass" # real files pass
|
||||
```
|
||||
|
||||
The mutation check for the sample tests edited one line per run,
|
||||
asserting a single match. It ran the two tests, restored the file from a
|
||||
copy, and at the end `sha256sum -c` confirmed the three files. All five
|
||||
mutations failed the tests: `vlanTag: 5000` and `isoStorag:` in the
|
||||
sample, `192.168.2.8` and `type: kvm2` in the ConfigMap, and
|
||||
`image: debian-12` in the sample Proxy. `type: kvm2` was rerun after lint
|
||||
removed the test's own type comparison. It still fails, because
|
||||
`LoadConfig` refuses a proxmox block under another type.
|
||||
|
||||
The link sweep is a short script. It takes the docs as arguments, so it
|
||||
can be rerun after any later edit. It skips fenced code and inline code
|
||||
spans. For each relative link it checks that the file exists, and that
|
||||
the `#anchor` matches a GitHub-style slug of a heading in the target file
|
||||
(or the same file). Before the real run it was checked on a scratch copy
|
||||
of the guide with five links added. It reported exactly the three bad
|
||||
ones: a missing file, a missing anchor in `testing.md`, and a missing
|
||||
same-file anchor. It checked 78 relative links in the six edited docs,
|
||||
plus this log's, and found 0 broken:
|
||||
|
||||
```bash
|
||||
cat > doclinks.py <<'PY'
|
||||
import os, re, sys
|
||||
def slug(h, seen):
|
||||
s = re.sub(r"[^\w\- ]", "", h.strip().lower()).replace(" ", "-")
|
||||
n = seen.get(s, 0); seen[s] = n + 1
|
||||
return s if n == 0 else f"{s}-{n}"
|
||||
def prose(path):
|
||||
out, fence = [], False
|
||||
for line in open(path, encoding="utf-8"):
|
||||
if line.lstrip().startswith("```"): fence = not fence; continue
|
||||
if not fence: out.append(line)
|
||||
return out
|
||||
def anchors(path):
|
||||
seen = {}
|
||||
return {slug(re.sub(r"`", "", l.lstrip("#").strip()), seen) for l in prose(path) if re.match(r"#{1,6} ", l)}
|
||||
bad = 0
|
||||
for doc in sys.argv[1:]:
|
||||
for n, line in enumerate(prose(doc), 1):
|
||||
for target in re.findall(r"\]\(([^)\s]+)\)", re.sub(r"`[^`]*`", "", line)):
|
||||
if re.match(r"[a-z]+:", target): continue
|
||||
f, _, a = target.partition("#")
|
||||
p = os.path.normpath(os.path.join(os.path.dirname(doc), f)) if f else doc
|
||||
if not os.path.exists(p): print(f"{doc}: missing file {target}"); bad += 1
|
||||
elif a and p.endswith(".md") and a not in anchors(p): print(f"{doc}: missing anchor {target}"); bad += 1
|
||||
print(f"{bad} broken"); sys.exit(1 if bad else 0)
|
||||
PY
|
||||
python3 doclinks.py docs/proxmox-provider.md docs/providers-existing.md docs/architecture.md \
|
||||
README.md docs/providers.md CLAUDE.md docs/plans-executions/2026-09-18-1502-proxmox-provider.md # 0 broken
|
||||
```
|
||||
|
||||
Verification:
|
||||
|
||||
```bash
|
||||
bin/kustomize build config/default >/dev/null && bin/kustomize build config/samples >/dev/null # ok
|
||||
go test -count=1 -run 'TestLoadConfig_proxmoxSample' -v ./internal/provider/ # 3 PASS
|
||||
grep -c '—' docs/proxmox-provider.md # 0
|
||||
go build ./... && go vet ./... && go vet -tags=integration ./... && go test -short -count=1 ./... # all ok
|
||||
bin/golangci-lint run --disable staticcheck --max-same-issues 0 --max-issues-per-linter 0 ./internal/provider/
|
||||
git status --short config/ # the four edited files named above, plus proxy_proxmox.yaml
|
||||
```
|
||||
|
||||
Worth noting:
|
||||
|
||||
- Lint on `internal/provider` is at its baseline of three goconst hits.
|
||||
The first version of the new test pushed it to five: its
|
||||
`p.Type != "proxmox"` made `"proxmox"` a three-time literal, flagged in
|
||||
both `config.go` and the test. That comparison added nothing, so it
|
||||
went. `internal/provider/proxmox` was not touched.
|
||||
- The `tmp` volume comment in `manager.yaml` still says only the kvm
|
||||
provider stages seeds there. kvm2 and proxmox do too. It was already
|
||||
stale for kvm2, so it was left out of this phase's diff.
|
||||
- The architecture §5 block's right-hand bracket was aligned by
|
||||
character, not byte (`→` is three bytes), so every line puts `│` in
|
||||
column 83.
|
||||
- Nothing in this phase talked to the Proxmox host or a cluster.
|
||||
`.claude/settings.json` stays out of the commit. The parked kvm2 items
|
||||
(S5.1, and the 200 ms `TestCheck_hostsCheckedConcurrently`) did not
|
||||
bite: `go test -short ./...` passed.
|
||||
- Next is Phase 10: the first real run of `TestProxmox_lifecycle` and
|
||||
`TestProxmoxCheck_realHost` on pv1-1, with the env block in
|
||||
`docs/testing.md`. The three answers never captured go with it, and the
|
||||
guide's "Not yet proven end to end" gap and the README's matching
|
||||
clause come out once they pass. Then the CHANGELOG entry after the user
|
||||
confirms, and the MR.
|
||||
|
||||
+256
-64
@@ -1,17 +1,20 @@
|
||||
# The existing providers
|
||||
|
||||
This is the appendix to [provider-contract.md](provider-contract.md): how the
|
||||
four providers that exist today (kubernetes, gcp, kvm, kvm2) meet the
|
||||
contract, which profile each one chose on every axis, and what is still
|
||||
five providers that exist today (kubernetes, gcp, kvm, kvm2, proxmox) meet
|
||||
the contract, which profile each one chose on every axis, and what is still
|
||||
missing. It is reference material for whoever writes the next provider, not
|
||||
a user guide; the user guides are [kvm-provider.md](kvm-provider.md),
|
||||
[kvm2-provider.md](kvm2-provider.md),
|
||||
[proxmox-provider.md](proxmox-provider.md),
|
||||
[gcp-vm-validation.md](gcp-vm-validation.md) and
|
||||
[gcp-in-specific-project.md](gcp-in-specific-project.md). The README's
|
||||
"Implemented providers" table stays the short public summary. kvm2 is the
|
||||
one provider built from the contract itself (clean-room, from
|
||||
[kvm2-provider-spec.md](kvm2-provider-spec.md), without reading kvm's code),
|
||||
so its column is also a record of what the contract produced.
|
||||
"Implemented providers" table stays the short public summary. kvm2 and
|
||||
proxmox are the two providers built from the contract itself (kvm2
|
||||
clean-room from [kvm2-provider-spec.md](kvm2-provider-spec.md) without
|
||||
reading kvm's code, proxmox from
|
||||
[proxmox-provider-spec.md](proxmox-provider-spec.md)), so their columns are
|
||||
also a record of what the contract produced.
|
||||
|
||||
The per-provider call flows are drawn in
|
||||
[architecture.md §5](architecture.md#5-what-provider-calls-do-in-the-outside-world),
|
||||
@@ -27,59 +30,59 @@ Code references are a file path plus a function or symbol name.
|
||||
One row per axis of contract §4, in the same order, with the profile name
|
||||
and the symbol that shows it.
|
||||
|
||||
| Axis | kubernetes | gcp | kvm | kvm2 |
|
||||
|---|---|---|---|---|
|
||||
| `readiness` | immediate: a Pod is scheduled within seconds (`stateFromPod`) | provisioning-window: minutes of boot (`mapState`) | provisioning-window: boot plus, on first use, an image download (`imageCache.ensure`) | provisioning-window: boot plus, on first use, an image download (`ensureImage`) |
|
||||
| `create-return` | submit: returns after `client.Create` | submit: `Insert` is never awaited, `Operation.Wait` is not called | **blocking**: waits through `DomainCreate` (`ensureRunning`), and the first image fetch runs inside (`imageCache.fetch`) | **blocking**: waits through `DomainCreate` and a 15-second poll for running (`ensureRunning`, `waitRunning`), and the first image fetch runs inside (`fetchImage`) |
|
||||
| `delete-return` | returns-early: `client.Delete`, Pod terminates on its own | returns-early: `Delete` submits an operation each time | waits: destroy, undefine and volume removal complete before return (`removeDomain`, `removeVolumes`) | waits: destroy, undefine and volume removal complete before return (`removeDomain`, `removeVolumes`) |
|
||||
| `ip-source` | assigned-at-create: `PodIP` (`instanceFromPod`) | assigned-at-create: the external NAT IP of the first NIC (`toInstance`) | discovered: ARP table first, then the guest agent, first routable IPv4 (`address`, `firstIPv4`) | discovered: the guest agent first, then the ARP table, first routable IPv4 (`addressSources`, `address`, `firstIPv4`) |
|
||||
| `gc-mark` | native-labels: Pod labels (`buildPod`) | native-labels: instance labels (`buildInsertRequest`) | written-metadata: a namespaced `<metadata>` element written in `DomainDefineXML` (`metadata.go`) | written-metadata: an element in kvm2's own namespace written in `DomainDefineXML` (`metadataURI`, `newInstanceMetadata`) |
|
||||
| `created-at` | backend-timestamp: `CreationTimestamp` | backend-timestamp: `CreationTimestamp`, parse error gives zero | written-at-create: `createdAt` in the metadata element (`newInstanceMetadata`) | written-at-create: `createdAt` in the metadata element, parse error gives zero (`newInstanceMetadata`, `createdAt`) |
|
||||
| `list-partial` | n/a: one call, all namespaces | omit-unreachable: `ReturnPartialSuccess` on `AggregatedList` | fail-whole per host, one unreadable domain skipped (`ListByTag`, `managedInstance`) | fail-whole per host, one unreadable domain skipped unless ctx ended; `ID`, `UID` and `CreatedAt` from the name and the mark only, no state or address RPC (`ListByTag`) |
|
||||
| `quota` | absent: a quota 403 is Permanent (`classify`) | detected: 429, and 403 with reason `quotaExceeded` or `rateLimitExceeded` (`classify`, `hasReason`) | absent (`classify`) | detected: a pool-space precheck before any volume is created (`checkPoolSpace`, `neededBytes`) |
|
||||
| `not-found` | status-404: `apierrors.IsNotFound` | status-404: `isNotFound` | error-code: `ErrNoDomain`, `ErrNoStorageVol` (`hasCode`) | error-code: `ErrNoDomain`, `ErrNoStorageVol`, Transient inside Create (`classifyLibvirt`) |
|
||||
| `primary-key` | name-is-key: `namespace/name` | name-is-key: `zones/<zone>/instances/<name>` | name-is-key: `hosts/<host>/domains/<name>` | name-is-key: `hosts/<host>/domains/<name>`, with a UID guard on adoption (`create`) |
|
||||
| `provisioning-state` | native: `PodPending` | native: `PROVISIONING`, `STAGING`, `REPAIRING` | create-blocks: a domain is defined and started in one `Create`; `nostate` maps to Provisioning | create-blocks: defined, started and polled to running in one `Create`; `nostate` maps to Provisioning (`mapState`) |
|
||||
| `async-ops` | none | task-handle, but the handle is discarded: an insert that fails after acceptance is never read | none | none |
|
||||
| `secondary-resources` | none | cascade: `AutoDelete` on the boot disk (`buildInsertRequest`) | explicit: overlay and seed removed by name (`removeVolumes`); base image kept | explicit: overlay and seed removed by name (`removeVolumes`); base image kept |
|
||||
| `credentials` | workload-identity: in-cluster config or kubeconfig (`NewWithTransportWrapper`) | environment: Application Default Credentials | mounted-files: SSH key and known_hosts from the optional `kvm-ssh` Secret, read lazily (`newLazySSH`) | mounted-files: the same `kvm-ssh` Secret, read on every dial and never cached (`loadCredentials`) |
|
||||
| `client-kind` | vendor-sdk: controller-runtime `client.Client`, no seam interface | vendor-sdk: `cloud.google.com/go/compute` behind `instancesAPI` | rpc-over-ssh: `go-libvirt` over an SSH-forwarded unix socket, behind `libvirtAPI` (`conn.go`) | rpc-over-ssh: `go-libvirt` over an SSH-forwarded unix socket, behind the ctx-first `backend` seam (`backend.go`) |
|
||||
| `timeouts` | ctx-only: no `rest.Config.Timeout` is set | ctx-only: SDK defaults, with SDK retries on Get and List | per-call for the dial only (`defaultHandshakeTimeout` in `dialSSH`); RPCs are ctx-only and a cancelled `call` closes the connection | per-call: a budget around the dial, every RPC, the uploads and the download (`defaultBudgets`, `call`); a cancelled or overlong call kills the pooled connection |
|
||||
| `logging` | none | tiered: V(1) outcomes and `logAPIError`, V(2) detail, V(5) wire log (`wirelog.go`) | tiered: V(1) outcomes and one `libvirt call` line per RPC (`trace.go`), V(2) XML and addresses; no wire tier | tiered: V(1) outcomes and one `libvirt call` line per RPC (`observe.go`), V(2) XML and addresses; no wire tier |
|
||||
| `child-spans` | transport-wrapper: `tracing.RestConfigWrapper()` on its own `rest.Config`, parent-gated | sdk-instrumentation: the SDK adds `otelhttp` itself; not parent-gated, but every call site has a parent | **none**: `trace.go` is a log decorator, not spans | span-decorator: `kvm2.<Method>` per RPC (`observe`), plus `kvm2.dial` (`libvirtDialer`) and `kvm2.download` (`fetchImage`) |
|
||||
| `backend-test` | cluster-e2e: kind (`test/e2e`) | manual-runbook: [gcp-vm-validation.md](gcp-vm-validation.md) | integration-tag: `kvm_integration_test.go`, `make test-integration-kvm` | integration-tag: `kvm2_integration_test.go`, `make test-integration-kvm2` |
|
||||
| Axis | kubernetes | gcp | kvm | kvm2 | proxmox |
|
||||
|---|---|---|---|---|---|
|
||||
| `readiness` | immediate: a Pod is scheduled within seconds (`stateFromPod`) | provisioning-window: minutes of boot (`mapState`) | provisioning-window: boot plus, on first use, an image download (`imageCache.ensure`) | provisioning-window: boot plus, on first use, an image download (`ensureImage`) | provisioning-window: boot plus the guest agent's install at first boot, about 50 s to an address on the test host (`agentAddress`) |
|
||||
| `create-return` | submit: returns after `client.Create` | submit: `Insert` is never awaited, `Operation.Wait` is not called | **blocking**: waits through `DomainCreate` (`ensureRunning`), and the first image fetch runs inside (`imageCache.fetch`) | **blocking**: waits through `DomainCreate` and a 15-second poll for running (`ensureRunning`, `waitRunning`), and the first image fetch runs inside (`fetchImage`) | **blocking**: waits through the clone, upload, resize and start tasks and a 60-second poll for running (`provision`, `waitRunning`) |
|
||||
| `delete-return` | returns-early: `client.Delete`, Pod terminates on its own | returns-early: `Delete` submits an operation each time | waits: destroy, undefine and volume removal complete before return (`removeDomain`, `removeVolumes`) | waits: destroy, undefine and volume removal complete before return (`removeDomain`, `removeVolumes`) | waits: the stop, destroy and seed delete tasks complete before return (`destroy`, `deleteSeed`) |
|
||||
| `ip-source` | assigned-at-create: `PodIP` (`instanceFromPod`) | assigned-at-create: the external NAT IP of the first NIC (`toInstance`) | discovered: ARP table first, then the guest agent, first routable IPv4 (`address`, `firstIPv4`) | discovered: the guest agent first, then the ARP table, first routable IPv4 (`addressSources`, `address`, `firstIPv4`) | discovered: the guest agent only, first routable IPv4 in the agent's order (`agentAddress`, `firstIPv4`) |
|
||||
| `gc-mark` | native-labels: Pod labels (`buildPod`) | native-labels: instance labels (`buildInsertRequest`) | written-metadata: a namespaced `<metadata>` element written in `DomainDefineXML` (`metadata.go`) | written-metadata: an element in kvm2's own namespace written in `DomainDefineXML` (`metadataURI`, `newInstanceMetadata`) | written-metadata: three `key=value` lines in the VM's description, passed in the clone call itself (`formatDescription`, `parseDescription`) |
|
||||
| `created-at` | backend-timestamp: `CreationTimestamp` | backend-timestamp: `CreationTimestamp`, parse error gives zero | written-at-create: `createdAt` in the metadata element (`newInstanceMetadata`) | written-at-create: `createdAt` in the metadata element, parse error gives zero (`newInstanceMetadata`, `createdAt`) | written-at-create: `proxy-operator-created-at` in the description, parse error gives zero (`parseDescription`) |
|
||||
| `list-partial` | n/a: one call, all namespaces | omit-unreachable: `ReturnPartialSuccess` on `AggregatedList` | fail-whole per host, one unreadable domain skipped (`ListByTag`, `managedInstance`) | fail-whole per host, one unreadable domain skipped unless ctx ended; `ID`, `UID` and `CreatedAt` from the name and the mark only, no state or address RPC (`ListByTag`) | fail-whole for the one cluster-wide `/cluster/resources` call, one unreadable VM skipped unless ctx ended; `ID`, `UID` and `CreatedAt` from the description only, no status or agent call (`ListByTag`) |
|
||||
| `quota` | absent: a quota 403 is Permanent (`classify`) | detected: 429, and 403 with reason `quotaExceeded` or `rateLimitExceeded` (`classify`, `hasReason`) | absent (`classify`) | detected: a pool-space precheck before any volume is created (`checkPoolSpace`, `neededBytes`) | absent: thin storage overprovisions, so no out-of-space answer exists to match; free space goes to the Checker's details (`classify`) |
|
||||
| `not-found` | status-404: `apierrors.IsNotFound` | status-404: `isNotFound` | error-code: `ErrNoDomain`, `ErrNoStorageVol` (`hasCode`) | error-code: `ErrNoDomain`, `ErrNoStorageVol`, Transient inside Create (`classifyLibvirt`) | error-code: the 500 `Configuration file '…' does not exist`, plus list-and-miss on `/cluster/resources`; Transient inside Create (`isNotExist`, `classifyAPI`) |
|
||||
| `primary-key` | name-is-key: `namespace/name` | name-is-key: `zones/<zone>/instances/<name>` | name-is-key: `hosts/<host>/domains/<name>` | name-is-key: `hosts/<host>/domains/<name>`, with a UID guard on adoption (`create`) | **allocated-id**: `vms/<vmid>/<name>`, the VMID from an FNV-1a hash of the UID with forward probing, each taken VMID put through an ownership check with a UID guard (`vmidFor`, `allocate`, `inspect`) |
|
||||
| `provisioning-state` | native: `PodPending` | native: `PROVISIONING`, `STAGING`, `REPAIRING` | create-blocks: a domain is defined and started in one `Create`; `nostate` maps to Provisioning | create-blocks: defined, started and polled to running in one `Create`; `nostate` maps to Provisioning (`mapState`) | create-blocks: cloned, configured, started and polled to running in one `Create`; `qmpstatus: prelaunch` maps to Provisioning (`mapState`) |
|
||||
| `async-ops` | none | task-handle, but the handle is discarded: an insert that fails after acceptance is never read | none | none | task-handle and backend-lock: every clone, upload, resize, start, stop, destroy and volume delete returns a UPID polled to its end within the call (`waitTask`); a VM's lock is waited on in Create and Transient in Get and Delete (`waitUnlocked`) |
|
||||
| `secondary-resources` | none | cascade: `AutoDelete` on the boot disk (`buildInsertRequest`) | explicit: overlay and seed removed by name (`removeVolumes`); base image kept | explicit: overlay and seed removed by name (`removeVolumes`); base image kept | explicit: the seed ISO deleted by name (`deleteSeed`, `deleteSeedEverywhere`); the disks go with the destroy's `purge=1&destroy-unreferenced-disks=1` |
|
||||
| `credentials` | workload-identity: in-cluster config or kubeconfig (`NewWithTransportWrapper`) | environment: Application Default Credentials | mounted-files: SSH key and known_hosts from the optional `kvm-ssh` Secret, read lazily (`newLazySSH`) | mounted-files: the same `kvm-ssh` Secret, read on every dial and never cached (`loadCredentials`) | mounted-files: the token from the optional `proxmox-token` Secret, read on every request (`readToken`); the CA from the same Secret, loaded once and cached (`httpClient`) |
|
||||
| `client-kind` | vendor-sdk: controller-runtime `client.Client`, no seam interface | vendor-sdk: `cloud.google.com/go/compute` behind `instancesAPI` | rpc-over-ssh: `go-libvirt` over an SSH-forwarded unix socket, behind `libvirtAPI` (`conn.go`) | rpc-over-ssh: `go-libvirt` over an SSH-forwarded unix socket, behind the ctx-first `backend` seam (`backend.go`) | thin-client: `net/http` and `encoding/json` behind the ctx-first `api` seam, one method per endpoint (`api.go`, `httpAPI`) |
|
||||
| `timeouts` | ctx-only: no `rest.Config.Timeout` is set | ctx-only: SDK defaults, with SDK retries on Get and List | per-call for the dial only (`defaultHandshakeTimeout` in `dialSSH`); RPCs are ctx-only and a cancelled `call` closes the connection | per-call: a budget around the dial, every RPC, the uploads and the download (`defaultBudgets`, `call`); a cancelled or overlong call kills the pooled connection | per-call: dial and TLS handshake 10 s, each request `requestTimeoutSeconds`, a budget per task wait (`defaultBudgets`); a cancelled request ends only itself |
|
||||
| `logging` | none | tiered: V(1) outcomes and `logAPIError`, V(2) detail, V(5) wire log (`wirelog.go`) | tiered: V(1) outcomes and one `libvirt call` line per RPC (`trace.go`), V(2) XML and addresses; no wire tier | tiered: V(1) outcomes and one `libvirt call` line per RPC (`observe.go`), V(2) XML and addresses; no wire tier | tiered: V(1) outcomes and one `proxmox api call` line per request (`observe.go`), V(2) parameters, config summaries and addresses; no wire tier |
|
||||
| `child-spans` | transport-wrapper: `tracing.RestConfigWrapper()` on its own `rest.Config`, parent-gated | sdk-instrumentation: the SDK adds `otelhttp` itself; not parent-gated, but every call site has a parent | **none**: `trace.go` is a log decorator, not spans | span-decorator: `kvm2.<Method>` per RPC (`observe`), plus `kvm2.dial` (`libvirtDialer`) and `kvm2.download` (`fetchImage`) | transport-wrapper: `tracing.RestConfigWrapper()` on the client's transport, parent-gated, one `HTTP <METHOD>` span per request (`newHTTPAPI`) |
|
||||
| `backend-test` | cluster-e2e: kind (`test/e2e`) | manual-runbook: [gcp-vm-validation.md](gcp-vm-validation.md) | integration-tag: `kvm_integration_test.go`, `make test-integration-kvm` | integration-tag: `kvm2_integration_test.go`, `make test-integration-kvm2` | integration-tag: `proxmox_integration_test.go`, `make test-integration-proxmox`; compiled, not yet run against a host |
|
||||
|
||||
## 2. Other differences
|
||||
|
||||
Rows that are not profile axes but that a new provider author compares
|
||||
against.
|
||||
|
||||
| Feature | kubernetes | gcp | kvm | kvm2 |
|
||||
|---|---|---|---|---|
|
||||
| Create verifies the adopted instance's UID | no | no | no | **yes**: a domain without the kvm2 mark or with a different non-empty UID is Permanent; a marked one with a matching or empty UID is adopted (`create`) |
|
||||
| Placement used | none | Zone, MachineType, Image | Zone (host), MachineType, Image | Zone (host), MachineType, Image |
|
||||
| CloudInit delivered | **no**: ignored | yes: `user-data` metadata | yes: NoCloud seed ISO built in-process (`buildSeedISO`) | yes: NoCloud seed ISO built in-process (`nocloud.BuildSeedISO`) |
|
||||
| Port used | yes: Squid config and containerPort (`squidConf`) | no: firewall is manual | no | no |
|
||||
| States produced | Provisioning, Running, Terminated | all four | Provisioning, Running, Stopped | Provisioning, Running, Stopped |
|
||||
| Deleting instance still reports its state | yes: a Pod being deleted reports its phase | n/a | n/a | n/a: Delete waits, and a concurrent Get sees the domain until it is undefined |
|
||||
| Error.Provider is the configured name | **no**: hardcoded `"kubernetes"` | yes | yes | yes (`wrapErr`) |
|
||||
| Checker target | `apiserver` | the project | each host, concurrently | each host, concurrently, on a fresh connection |
|
||||
| Checker Details | none | none | `user`, `pool`, `hostname`, `libvirtVersion`, `poolState`, `poolCapacity`, `poolAvailable` | `user`, `pool`, `hostname`, `libvirtVersion`, `poolState`, `poolCapacityGiB`, `poolAvailableGiB`, filled in as far as the check got |
|
||||
| Credential rotation without restart | n/a | via ADC | no: cached after first successful load | yes: both files read on every dial (`loadCredentials`) |
|
||||
| Connection handling | client-go | SDK client, never closed | one pooled SSH plus libvirt connection per host, 15s keepalive (`hostPool`) | one pooled SSH plus libvirt connection per host, 15s keepalive (`hostTable`); dedicated connections for the image upload and the check (`dialFresh`) |
|
||||
| Unit-test double | controller-runtime fake client | `fakeAPI` behind `instancesAPI`, plus `httptest` for the real SDK client in `check_test.go` | `fakeAPI` behind `libvirtAPI`, plus an in-process `x/crypto/ssh` server in `conn_test.go` | `fakeBackend` behind `backend` (`fake_test.go`), plus an in-process `x/crypto/ssh` server (`sshserver_test.go`) and go-libvirt's own mock for the adapter |
|
||||
| Proxy software | generated Squid config (`pod.go`) | from `cloudInit`; the image must run cloud-init | from `cloudInit`, plus a vendor-data script that installs the guest agent (`seed.go`) | from `cloudInit`, plus a vendor-data script that installs the guest agent (`nocloud.GuestAgentScript`) |
|
||||
| IP reachable from the operator | Pod network, so in-cluster only (not from `make run-dev` on a laptop) | external IP, once a firewall rule allows the cluster's egress IP | LAN address on the host's bridge; the operator needs a route to that LAN | LAN address on the host's bridge; the operator needs a route to that LAN |
|
||||
| Distinct egress IP per proxy | no: the cluster's egress | yes: ephemeral external IP | whatever the host's LAN routes through; not per VM by itself | whatever the host's LAN routes through; not per VM by itself |
|
||||
| Feature | kubernetes | gcp | kvm | kvm2 | proxmox |
|
||||
|---|---|---|---|---|---|
|
||||
| Create verifies the adopted instance's UID | no | no | no | **yes**: a domain without the kvm2 mark or with a different non-empty UID is Permanent; a marked one with a matching or empty UID is adopted (`create`) | **yes**: a VM with the proxy's name and no mark, or with a different non-empty UID, is Permanent; a marked one with a matching or empty UID is adopted (`inspect`) |
|
||||
| Placement used | none | Zone, MachineType, Image | Zone (host), MachineType, Image | Zone (host), MachineType, Image | Zone (node), MachineType, Image (a template key) |
|
||||
| CloudInit delivered | **no**: ignored | yes: `user-data` metadata | yes: NoCloud seed ISO built in-process (`buildSeedISO`) | yes: NoCloud seed ISO built in-process (`nocloud.BuildSeedISO`) | yes: NoCloud seed ISO built in-process (`nocloud.BuildSeedISO`) and uploaded through the API (`uploadSeed`) |
|
||||
| Port used | yes: Squid config and containerPort (`squidConf`) | no: firewall is manual | no | no | no |
|
||||
| States produced | Provisioning, Running, Terminated | all four | Provisioning, Running, Stopped | Provisioning, Running, Stopped | Provisioning, Running, Stopped |
|
||||
| Deleting instance still reports its state | yes: a Pod being deleted reports its phase | n/a | n/a | n/a: Delete waits, and a concurrent Get sees the domain until it is undefined | n/a: Delete waits, and a concurrent Get sees the VM (Stopped once stopped) until the destroy removes its config |
|
||||
| Error.Provider is the configured name | **no**: hardcoded `"kubernetes"` | yes | yes | yes (`wrapErr`) | yes (`wrapErr`) |
|
||||
| Checker target | `apiserver` | the project | each host, concurrently | each host, concurrently, on a fresh connection | each node, concurrently, on the shared HTTP client |
|
||||
| Checker Details | none | none | `user`, `pool`, `hostname`, `libvirtVersion`, `poolState`, `poolCapacity`, `poolAvailable` | `user`, `pool`, `hostname`, `libvirtVersion`, `poolState`, `poolCapacityGiB`, `poolAvailableGiB`, filled in as far as the check got | `isoStorage`, `tokenID`, `pveVersion`, `kernel`, `isoStorageAvailGiB`, `templates`, filled in as far as the check got |
|
||||
| Credential rotation without restart | n/a | via ADC | no: cached after first successful load | yes: both files read on every dial (`loadCredentials`) | the token yes, read on every request (`readToken`); the CA no, cached after first use |
|
||||
| Connection handling | client-go | SDK client, never closed | one pooled SSH plus libvirt connection per host, 15s keepalive (`hostPool`) | one pooled SSH plus libvirt connection per host, 15s keepalive (`hostTable`); dedicated connections for the image upload and the check (`dialFresh`) | one `net/http` client, 4 idle connections per host, HTTP/1.1 only; a cancelled request ends only itself |
|
||||
| Unit-test double | controller-runtime fake client | `fakeAPI` behind `instancesAPI`, plus `httptest` for the real SDK client in `check_test.go` | `fakeAPI` behind `libvirtAPI`, plus an in-process `x/crypto/ssh` server in `conn_test.go` | `fakeBackend` behind `backend` (`fake_test.go`), plus an in-process `x/crypto/ssh` server (`sshserver_test.go`) and go-libvirt's own mock for the adapter | `fakeAPI` behind `api` (`fake_test.go`), plus `httptest.NewTLSServer` replaying the Phase 0 captures for the real client (`client_test.go`) |
|
||||
| Proxy software | generated Squid config (`pod.go`) | from `cloudInit`; the image must run cloud-init | from `cloudInit`, plus a vendor-data script that installs the guest agent (`seed.go`) | from `cloudInit`, plus a vendor-data script that installs the guest agent (`nocloud.GuestAgentScript`) | from `cloudInit`, plus a vendor-data script that installs the guest agent (`nocloud.GuestAgentScript`) |
|
||||
| IP reachable from the operator | Pod network, so in-cluster only (not from `make run-dev` on a laptop) | external IP, once a firewall rule allows the cluster's egress IP | LAN address on the host's bridge; the operator needs a route to that LAN | LAN address on the host's bridge; the operator needs a route to that LAN | address on the proxy network (bridge or VLAN from the config); the operator needs a route to it |
|
||||
| Distinct egress IP per proxy | no: the cluster's egress | yes: ephemeral external IP | whatever the host's LAN routes through; not per VM by itself | whatever the host's LAN routes through; not per VM by itself | whatever the proxy network routes through; not per VM by itself |
|
||||
|
||||
## 3. Observability status
|
||||
|
||||
| Layer | kubernetes | gcp | kvm | kvm2 |
|
||||
|---|---|---|---|---|
|
||||
| 1. Operation spans and metrics | yes: decorators | yes: decorators | yes: decorators | yes: decorators |
|
||||
| 2. Child spans per backend call | yes: `tracing.RestConfigWrapper()`, parent-gated (`cmd/main.go`, `NewWithTransportWrapper`) | yes: the SDK's own `otelhttp`; not parent-gated | **no** | yes: `kvm2.<Method>` from the `observe` decorator, `kvm2.dial`, `kvm2.download`; always under the caller's span, since every call carries the caller's ctx |
|
||||
| 3. Logs carry `traceID` and `spanID` | n/a: the package does not log | yes, including the V(5) wire log | yes: every `libvirt call` line at V(1) | yes: every `libvirt call` line at V(1), written inside its RPC's span |
|
||||
| Tested against a real collector | yes: the kind e2e checks `provider.create` and `provider.delete` in Tempo (`test/e2e/tracing_test.go`), operation spans only | no | no | no; the integration test asserts the parent-child topology in an in-memory recorder (`assertLifecycleSpans`) |
|
||||
| Layer | kubernetes | gcp | kvm | kvm2 | proxmox |
|
||||
|---|---|---|---|---|---|
|
||||
| 1. Operation spans and metrics | yes: decorators | yes: decorators | yes: decorators | yes: decorators | yes: decorators |
|
||||
| 2. Child spans per backend call | yes: `tracing.RestConfigWrapper()`, parent-gated (`cmd/main.go`, `NewWithTransportWrapper`) | yes: the SDK's own `otelhttp`; not parent-gated | **no** | yes: `kvm2.<Method>` from the `observe` decorator, `kvm2.dial`, `kvm2.download`; always under the caller's span, since every call carries the caller's ctx | yes: `tracing.RestConfigWrapper()` on the transport, parent-gated, one `HTTP <METHOD>` per request |
|
||||
| 3. Logs carry `traceID` and `spanID` | n/a: the package does not log | yes, including the V(5) wire log | yes: every `libvirt call` line at V(1) | yes: every `libvirt call` line at V(1), written inside its RPC's span | yes: every `proxmox api call` line at V(1), with the operation span's `spanID` |
|
||||
| Tested against a real collector | yes: the kind e2e checks `provider.create` and `provider.delete` in Tempo (`test/e2e/tracing_test.go`), operation spans only | no | no | no; the integration test asserts the parent-child topology in an in-memory recorder (`assertLifecycleSpans`) | no; `TestTracing_httpSpansUnderOperationSpan` asserts the parent-child topology in an in-memory recorder |
|
||||
|
||||
## 4. kubernetes (`internal/provider/kubernetes`)
|
||||
|
||||
@@ -528,20 +531,209 @@ same libvirt.
|
||||
its backing image or size.
|
||||
- The sample cloud-init installs Squid with `http_access allow all`.
|
||||
|
||||
## 8. Gaps shared by the existing providers
|
||||
## 8. proxmox (`internal/provider/proxmox`)
|
||||
|
||||
- **Adoption is not verified in three of the four.** kubernetes, gcp and
|
||||
The full user guide is [proxmox-provider.md](proxmox-provider.md) and the
|
||||
design it was built from is
|
||||
[proxmox-provider-spec.md](proxmox-provider-spec.md); this section is the
|
||||
author's view. It is the second provider built from the contract, and the
|
||||
first on a REST API of the hypervisor's own: no SSH, no libvirt. Every
|
||||
error text it matches was captured from a PVE 9.2.20 host before the code
|
||||
was written.
|
||||
|
||||
- **Package and seam.** `proxmox.go` (`Provider`, `New`, `withDefaults`,
|
||||
`node`, `wrapErr`, the per-name lock, the budgets), `create.go`,
|
||||
`get.go`, `delete.go`, `list.go` (the four methods), `api.go` (the
|
||||
seam), `client.go` (the production adapter `httpAPI`, the token and CA
|
||||
loading), `observe.go` (the logging decorator), `task.go` (task, lock
|
||||
and running waits), `seed.go`, `mark.go`, `id.go`, `state.go`,
|
||||
`errors.go`, `check.go`; the seed ISO comes from the shared
|
||||
`internal/provider/nocloud` and the name lock from
|
||||
`internal/provider/keyedlock`. The seam is `api`: one method per
|
||||
endpoint, sixteen in all, each with `ctx` first. `New` wraps `httpAPI` in
|
||||
`observe` once, so provider code never holds an undecorated seam;
|
||||
`newProvider` is the test constructor.
|
||||
- **Config.** `ProviderConfig.Proxmox`, the `ProxmoxConfig` type in
|
||||
`internal/provider/config_proxmox.go`, validated by `validateProxmox`
|
||||
(the URL shape, node names and storage ids by regexp because they end
|
||||
up in request paths, `vlanTag`, `vmidRange`, the two catalogues).
|
||||
`isoStorage` has no default on purpose: the token needs
|
||||
`Datastore.Allocate` there. Defaults (`vmidRange` 100000 to 199999,
|
||||
`requestTimeoutSeconds` 30, `installGuestAgent` true) are filled by
|
||||
`withDefaults`. `New` never makes a request; it warns when the token
|
||||
file does not exist, or when `insecureSkipTLSVerify` is on.
|
||||
- **ID.** `formatProviderID(vmid, name)` gives `vms/<vmid>/<name>`, with no
|
||||
node, because VMs migrate; the node is resolved from
|
||||
`/cluster/resources` on every call (`resolve`). `parseProviderID`
|
||||
accepts exactly three segments, the literal `vms`, a decimal VMID
|
||||
without leading zeros between 100 and 999999999 and a non-empty name,
|
||||
else `ErrPermanent`. The name is there so `Get` and `Delete` can tell
|
||||
our VM from another that later got the same VMID.
|
||||
- **Create.** `placement` checks `Zone` (`node`; optional with one node),
|
||||
`MachineType` and `Image` (both catalogue keys) before any request, and
|
||||
logs the resolved parameters with `cloudInitBytes` at V(2). Then, under
|
||||
the per-name lock, `create`: one `ClusterResources` read;
|
||||
`adoptByName` (two VMs with the name are Permanent, one goes to
|
||||
`inspect`); with none, `template` reads the template's config and
|
||||
refuses a non-template, a cloud-init drive, a missing `net0` when the
|
||||
config rewrites it, and an unsized boot disk when `diskGiB` is set, all
|
||||
before any clone; `allocate` probes `vmidFor(uid, attempt)` up to 32
|
||||
times, cloning onto a free VMID and running `inspect` on a taken one or
|
||||
on the clone's `config file already exists` answer. `inspect` waits out
|
||||
a lock (`waitUnlocked`; a clone in progress is also nameless), then
|
||||
adopts our name with our mark and a matching or empty UID, probes past
|
||||
another name, and is Permanent for our name without the mark, with
|
||||
another UID, or hibernated. An adopted VM gets one `VMStatus`: running
|
||||
returns at once. Otherwise `provision`: `uploadSeed` (rebuilt and
|
||||
overwritten every attempt), `SetVMConfig` (`configParams`: cores,
|
||||
memory, `agent=1`, the tag, `ide2`, and `net0` through `rewriteNet0`
|
||||
when a bridge or VLAN is configured), the resize task when
|
||||
`resizeTarget` says the boot disk is smaller than `diskGiB`, the start
|
||||
task, and `waitRunning` (every second up to 60 s, Transient on expiry
|
||||
or on a stop). `Create` returns only when the VM runs.
|
||||
- **Get and state.** `get` checks in order: absent from the resources is
|
||||
NotFound; a lock other than `suspended` is Transient (checked before the
|
||||
name, because a mid-clone entry has none); `status: unknown` is
|
||||
Transient; the config's `does not exist` is NotFound, a config lock
|
||||
Transient; another name or no mark is NotFound; hibernated is Stopped
|
||||
without further calls. Then `VMStatus` and `mapState(status,
|
||||
qmpstatus, ip)`:
|
||||
|
||||
| `status` | `qmpstatus` | agent | Provider state |
|
||||
|---|---|---|---|
|
||||
| running | running | an IPv4 | Running |
|
||||
| running | running | 200 without an IPv4, or `QEMU guest agent is not running` | Provisioning |
|
||||
| running | running | `No QEMU guest agent configured` | Stopped |
|
||||
| running | prelaunch | not asked | Provisioning |
|
||||
| running | paused, suspended, io-error, anything else | not asked | Stopped |
|
||||
| stopped, anything else | any | not asked | Stopped |
|
||||
|
||||
Any other agent answer, including `VM <vmid> is not running`, is
|
||||
Transient. `UID` and `CreatedAt` are read from the description on every
|
||||
call. A running `Get` is four calls: resources, config, status, agent.
|
||||
- **Delete.** Under the per-name lock, `destroy`: `resolve`; a lock other
|
||||
than `suspended` or `status: unknown` is Transient; the config's `does
|
||||
not exist`, another name or no mark leave the VM alone; our VM
|
||||
hibernated is Permanent, because the destroy refuses it and the lock
|
||||
never clears. Otherwise stop (unless stopped) and destroy with
|
||||
`purge=1&destroy-unreferenced-disks=1`, each a task waited on through
|
||||
`runTask`, where `does not exist` means gone. Then `deleteSeed` on the
|
||||
VM's node, where a failure fails the call (a 403 for
|
||||
`Datastore.Allocate` is Permanent), or, with no VM of ours destroyed,
|
||||
`deleteSeedEverywhere` over every configured node, where a node that
|
||||
fails is logged and skipped unless ctx ended. A missing volume still
|
||||
ends in a successful task, so no message is matched. Nothing is left
|
||||
when nil is returned.
|
||||
- **ListByTag.** One `ClusterResources` covers every node, unreachable
|
||||
ones included (their guests are listed with `status: unknown`).
|
||||
`isCandidate` keeps QEMU VMs that are not templates and carry the tag
|
||||
`proxy_operator` or a name starting `proxy-`; a nameless entry (a clone
|
||||
in progress) is not a candidate. One `VMConfig` per candidate; the mark
|
||||
in the description decides, and the `Instance` is `ID`, `UID` and
|
||||
`CreatedAt` only. A candidate whose config cannot be read is skipped at
|
||||
V(1), unless ctx ended, which fails the call; the resources call failing
|
||||
fails the whole call.
|
||||
- **Errors.** `classify(op, err)`: an error already carrying a class keeps
|
||||
it; then `apiError`s by `classifyAPI` (every 4xx Permanent; the 500
|
||||
`does not exist` NotFound, Transient when `op == "create"`; every other
|
||||
status Transient, including the lock, the taken VMID and pveproxy's 595
|
||||
and 596); TLS verification failures (`tls.CertificateVerificationError`
|
||||
and the three `x509` types) Permanent; everything else Transient. The
|
||||
500 matchers (`isNotExist`, `isLocked`, `isAlreadyExists`, the three
|
||||
agent ones) are anchored to the 500 status and each is backed by a
|
||||
captured response. `permanentf` and `transientf` classify at the
|
||||
source: placement, the template checks, the ownership check, a
|
||||
malformed token file or CA file (Permanent); a missing token or CA file,
|
||||
an expired wait, a failed task (Transient). Nothing is Quota.
|
||||
- **Check.** `Check` runs `checkNode` per node concurrently on the shared
|
||||
client. `probeNode` reads the token file for `tokenID` (no request when
|
||||
it is unusable), then `Version`, `NodeStatus`, `checkStorage` (a 500
|
||||
whatever its text, inactive, or no `iso` content is `StorageNotFound`)
|
||||
and `checkTemplates` (each catalogue VMID exists and is a template,
|
||||
else `TemplateNotFound`). `checkNode` answers `Timeout` whenever the
|
||||
round's ctx ended; otherwise `checkReason`: the two failures above,
|
||||
`errCredentials` `CredentialsUnavailable`, `errCAFile` or a TLS
|
||||
verification failure `CertificateInvalid`, 401 `Unauthorized`, 403
|
||||
`Forbidden`, 595 and 596 `Unreachable`, any other status `APIError`, a
|
||||
`net.OpError` `Unreachable` (before the deadline, so a dial out of its
|
||||
own budget is a node that did not answer), a deadline or cancellation
|
||||
`Timeout`, a cut connection `Unreachable`.
|
||||
- **Connections.** One `http.Client`, built on first use once the CA
|
||||
loads (`httpClient`), shared by every call and the Checker: dial and
|
||||
TLS handshake 10 s each, `Client.Timeout` `requestTimeoutSeconds`, 4
|
||||
idle connections per host kept 90 s, HTTP/2 disabled. A cancelled
|
||||
request ends only itself. The token file is read by `readToken` on
|
||||
every request.
|
||||
- **Logging.** Logger from the context, named `proxmox`, with `provider`,
|
||||
and `vmid`, `name` and `node` on lines about an instance. V(1): one line
|
||||
per operation outcome (`proxmox vm ensured running`, `proxmox vm
|
||||
fetched`, `proxmox vm deleted` with `destroyed`, `proxmox vms listed`,
|
||||
`proxmox node checked`), one `proxmox api call` line per seam call from
|
||||
`observe` (call, identifiers, UPID, duration, `httpStatus` and `error`),
|
||||
`proxmox task failed`, and the no-op lines. V(2): request parameters,
|
||||
config summaries, agent addresses, the resolved Create parameters.
|
||||
Cloud-init, the token, the CA and the upload body are never logged.
|
||||
- **Tracing.** No span of its own: `newHTTPAPI` wraps the transport in
|
||||
`tracing.RestConfigWrapper(opts...)`, so each request is an
|
||||
`HTTP <METHOD>` client span under the caller's span, and none without
|
||||
one. `observe` only logs, so nothing is counted twice.
|
||||
- **Tests.** Pure pieces (`id_test.go`, `mark_test.go`, `state_test.go`,
|
||||
`seed_test.go`, `errors_test.go` with the classify table over the
|
||||
captures, `api_test.go`); `client_test.go` (the real client over
|
||||
`httptest.NewTLSServer`: the token never in errors or logs, x509
|
||||
failures Permanent, cancellation, the child span); `create_test.go`,
|
||||
`get_test.go`, `delete_test.go`, `list_test.go`, `task_test.go`,
|
||||
`proxmox_test.go`, `check_test.go`, `observe_test.go` (every seam
|
||||
method logs exactly one line, by reflection) and `provider_test.go`
|
||||
(the contract §11 sweep, named test by test in the spec's entry 21).
|
||||
The double is `fakeAPI` in `fake_test.go`: VMs, tasks and ISOs in
|
||||
memory, a clone that leaves a nameless locked entry until its task
|
||||
ends, per-method errors and blocking, a call log; its error texts are
|
||||
checked against the captures. Real backend:
|
||||
`proxmox_integration_test.go` behind `//go:build integration`, driven
|
||||
by `PROXMOX_TEST_*` variables and `make test-integration-proxmox`,
|
||||
described in
|
||||
[testing.md](testing.md#the-proxmox-integration-test-real-proxmox-ve-host);
|
||||
it compiles but has not run against a host yet.
|
||||
- **Deployment.** The optional `proxmox-token` Secret (keys `token` and
|
||||
`ca.pem`) mounted at `/etc/proxy-operator/proxmox`
|
||||
(`config/manager/manager.yaml`), `make proxmox-token-secret`, a
|
||||
commented `proxmox-home` entry in both providers configs (loaded by
|
||||
`TestLoadConfig_proxmoxSampleEntry`), `config/samples/proxy_proxmox.yaml`,
|
||||
the `proxmox` type in the `status.type` comment of
|
||||
`api/v1alpha1/providerhealth_types.go`. No RBAC, and no
|
||||
`hack/providers-dev.yaml` entry, since it needs credentials.
|
||||
- **Known gaps** (see also [proxmox-provider.md § Known gaps](proxmox-provider.md#known-gaps)).
|
||||
- `Create` blocks until the VM runs.
|
||||
- A clone lock that never clears makes every Create for that Proxy wait
|
||||
out the 10-minute clone budget; `qm unlock` by hand.
|
||||
- GC blind spot: a seed ISO whose VM was destroyed outside the operator
|
||||
is invisible to GC, and a seed on node-local storage stays behind
|
||||
when its VM migrated.
|
||||
- The VMID range is shared with humans; a hand-made VM with a proxy's
|
||||
name is Permanent.
|
||||
- Cross-node clones (`target`) need shared storage and are untested.
|
||||
- A hibernated VM of ours latches its Proxy in Create and in Delete.
|
||||
- The CA is cached after first use; the token is not.
|
||||
- Never run against a real host yet; three answers were never captured
|
||||
(the storage status for a missing storage, a destroy on a hibernated
|
||||
VM, a stop on a paused VM).
|
||||
|
||||
## 9. Gaps shared by the existing providers
|
||||
|
||||
- **Adoption is not verified in three of the five.** kubernetes, gcp and
|
||||
kvm adopt any existing instance with the right name without checking
|
||||
`LabelUID`; kvm2 returns `ErrPermanent` when the UID differs.
|
||||
- **The Create comment does not match kvm and kvm2.** The interface comment
|
||||
in `internal/provider/provider.go` says `Create` returns as soon as the
|
||||
request is submitted; both libvirt providers block until the domain has
|
||||
started.
|
||||
`LabelUID`; kvm2 and proxmox return `ErrPermanent` when the UID differs.
|
||||
- **The Create comment does not match kvm, kvm2 and proxmox.** The
|
||||
interface comment in `internal/provider/provider.go` says `Create`
|
||||
returns as soon as the request is submitted; the three VM providers on
|
||||
your own hypervisors block until the VM has started.
|
||||
- **`Stopped` and `Terminated` are produced inconsistently.** kubernetes
|
||||
never produces Stopped, kvm and kvm2 never produce Terminated, and the
|
||||
reconciler treats them the same.
|
||||
never produces Stopped, kvm, kvm2 and proxmox never produce Terminated,
|
||||
and the reconciler treats them the same.
|
||||
- **Different `ListByTag` partial-failure policies.** gcp leaves failed
|
||||
parts out, kvm and kvm2 fail the whole list. Both are safe.
|
||||
parts out, kvm and kvm2 fail the whole list, and proxmox fails it when
|
||||
its one cluster-wide listing fails. Both are safe.
|
||||
- **Stale `mock` comments** in `internal/provider/provider.go`,
|
||||
`internal/provider/registry/registry.go` and `api/v1alpha1/proxy_types.go`
|
||||
(and so the generated CRD's `spec.provider` description).
|
||||
@@ -555,10 +747,10 @@ same libvirt.
|
||||
log lines. kvm2 has them, but only an in-memory recorder checks their
|
||||
topology. Only kubernetes is tested against a real collector, and that
|
||||
test checks only the operation spans; nothing checks that gcp's SDK spans,
|
||||
the kubernetes API spans or kvm2's spans nest under `provider.*` in a
|
||||
real collector.
|
||||
the kubernetes API spans, kvm2's spans or proxmox's HTTP spans nest under
|
||||
`provider.*` in a real collector.
|
||||
|
||||
## 9. Proposed features
|
||||
## 10. Proposed features
|
||||
|
||||
Listed only. Nothing below is designed or implemented, except where a
|
||||
bullet says kvm2 already does it.
|
||||
@@ -576,7 +768,7 @@ bullet says kvm2 already does it.
|
||||
backend, so a new provider gets the contract tests for free.
|
||||
- **Verify the UID when `Create` adopts an instance.** Return `ErrPermanent`
|
||||
on a mismatch instead of adopting an instance that belongs to someone else.
|
||||
kvm2 does this; the other three do not.
|
||||
kvm2 and proxmox do this; the other three do not.
|
||||
- **Report asynchronous create failures:** gcp keeps the operation name, and
|
||||
`Get` reports the failure instead of NotFound.
|
||||
- **Capacity or quota signals** for kvm, such as pool free space and host
|
||||
|
||||
+2
-1
@@ -14,12 +14,13 @@ the existing ones happen to work.
|
||||
open. Copy it to `docs/<type>-provider-spec.md`, fill it in, and hand it to
|
||||
the implementer with the other two files.
|
||||
- **[providers-existing.md](providers-existing.md)**: how kubernetes, gcp,
|
||||
kvm and kvm2 meet the contract, a table mapping each of them to every
|
||||
kvm, kvm2 and proxmox meet the contract, a table mapping each of them to every
|
||||
profile, their per-provider reference notes, shared gaps and proposed
|
||||
features.
|
||||
|
||||
Per-provider user guides: [kvm-provider.md](kvm-provider.md),
|
||||
[kvm2-provider.md](kvm2-provider.md),
|
||||
[proxmox-provider.md](proxmox-provider.md),
|
||||
[gcp-vm-validation.md](gcp-vm-validation.md) and
|
||||
[gcp-in-specific-project.md](gcp-in-specific-project.md).
|
||||
|
||||
|
||||
@@ -0,0 +1,646 @@
|
||||
# The proxmox provider
|
||||
|
||||
Runs Managed proxies as VMs on a Proxmox VE cluster. It is the third
|
||||
VM-style provider and the second built from
|
||||
[provider-contract.md](provider-contract.md), from the design in
|
||||
[proxmox-provider-spec.md](proxmox-provider-spec.md). The operator talks to
|
||||
the cluster's REST API only (`https://<host>:8006/api2/json`) with an API
|
||||
token. It needs no SSH, no shell on the nodes, and no library beyond Go's
|
||||
standard `net/http`.
|
||||
|
||||
Each VM is a **full clone of a template** you prepare once. The operator
|
||||
builds the Proxy's cloud-init into a NoCloud seed ISO in-process, uploads
|
||||
it to a storage on the VM's node, attaches it as a CD-ROM, sets the size,
|
||||
starts the VM and learns its address from the QEMU guest agent. Everything
|
||||
here was proven against **Proxmox VE 9.2.20** (QEMU 11.0) on the single
|
||||
node `pv1-1`; every error text the provider matches was captured from that
|
||||
host.
|
||||
|
||||
## Proxmox prerequisites
|
||||
|
||||
Once per cluster, as root on a node:
|
||||
|
||||
1. **Proxmox VE 9.x with the API reachable** from the operator's pods on
|
||||
port 8006. The operator's pods and the crawler clients must also route to
|
||||
the network the proxies land on (see step 5).
|
||||
2. **A template without a Proxmox cloud-init drive.** The provider brings
|
||||
its own seed on `ide2`; a Proxmox cloud-init drive would compete with it
|
||||
and the Proxy's user-data would be ignored. Debian 13's generic cloud
|
||||
image boots UEFI (under BIOS it hangs in GRUB on a headless VM), so the
|
||||
template does too:
|
||||
|
||||
```sh
|
||||
wget https://cloud.debian.org/images/cloud/trixie/latest/debian-13-generic-amd64.qcow2
|
||||
qm create 9001 --name debian-13-template --memory 2048 --cores 2 --ostype l26 \
|
||||
--machine q35 --bios ovmf --efidisk0 local-lvm:1,efitype=4m,pre-enrolled-keys=0 \
|
||||
--scsihw virtio-scsi-pci --net0 virtio,bridge=vmbr0 \
|
||||
--serial0 socket --vga serial0 --agent 1
|
||||
qm set 9001 --scsi0 local-lvm:0,import-from=/root/debian-13-generic-amd64.qcow2
|
||||
qm set 9001 --boot order=scsi0
|
||||
qm template 9001
|
||||
```
|
||||
|
||||
The template must boot from its first disk, run cloud-init with the
|
||||
NoCloud datasource (the Debian and Ubuntu cloud images do), and either
|
||||
ship `qemu-guest-agent` or be able to install it at first boot. Before
|
||||
the first clone for a Proxy, `Create` reads the template's config and
|
||||
refuses it with a permanent error when:
|
||||
|
||||
- it is not a template (`qm template` was not run);
|
||||
- any disk slot holds a Proxmox cloud-init drive (a value containing
|
||||
`cloudinit`), such as the one `qm set <vmid> --ide2 local-lvm:cloudinit`
|
||||
adds for Proxmox's own cloud-init support;
|
||||
- it has no `net0` while the config sets `bridge` or `vlanTag`;
|
||||
- the machine type sets `diskGiB` and the template has no boot disk, or
|
||||
one without a readable `size=`.
|
||||
|
||||
These are checked on the template because the clone copies its config:
|
||||
a VM cloned from a bad template could not be fixed by fixing the
|
||||
template. The template is checked only when a VM is about to be cloned;
|
||||
a VM that already exists no longer depends on it. The health check sees
|
||||
only whether each template exists and is a template.
|
||||
3. **A storage for the seed ISOs**, holding `iso` content. A dedicated
|
||||
directory storage is best, because the token needs `Datastore.Allocate`
|
||||
on it to delete seeds, and on a dedicated storage that privilege cannot
|
||||
touch backups or other ISOs:
|
||||
|
||||
```sh
|
||||
pvesm add dir proxy-seeds --path /var/lib/proxy-seeds --content iso
|
||||
```
|
||||
|
||||
A seed is tens of KiB. Proxmox stages uploads under `/var/tmp`, not the
|
||||
node's tmpfs `/tmp`. On a cluster, a storage that is not shared keeps
|
||||
each seed on the node the VM was created on (see *Known gaps*).
|
||||
4. **A user, two roles and a token.** `VM.Monitor` no longer exists in PVE
|
||||
9 and `pveum` rejects it; the agent read needs `VM.GuestAgent.Audit`
|
||||
alone. `VM.Config.Cloudinit` is not needed for the seed on `ide2`.
|
||||
`SDN.Use` on `/` is what lets the config write put a VLAN tag on
|
||||
`vmbr0`:
|
||||
|
||||
```sh
|
||||
pveum user add proxy-operator@pve --comment "egress-proxies-operator"
|
||||
pveum role add ProxyOperator -privs "VM.Allocate VM.Clone VM.Audit VM.Config.CDROM VM.Config.CPU VM.Config.Memory VM.Config.Disk VM.Config.Network VM.Config.Options VM.PowerMgmt VM.GuestAgent.Audit Datastore.Audit Datastore.AllocateSpace Datastore.AllocateTemplate Sys.Audit SDN.Use"
|
||||
pveum aclmod / -user proxy-operator@pve -role ProxyOperator
|
||||
pveum role add ProxyOperatorSeeds -privs "Datastore.Allocate Datastore.AllocateTemplate Datastore.AllocateSpace Datastore.Audit"
|
||||
pveum aclmod /storage/proxy-seeds -user proxy-operator@pve -role ProxyOperatorSeeds
|
||||
pveum user token add proxy-operator@pve operator --privsep 0
|
||||
```
|
||||
|
||||
The second role repeats three privileges of the first on purpose: an
|
||||
ACL on a more specific path **replaces** what the user inherits from
|
||||
`/` for that path, so a role with `Datastore.Allocate` alone would take
|
||||
the upload's `Datastore.AllocateTemplate` away. The role on `/` covers
|
||||
clone, config, resize, start, stop, the agent read, the upload and the
|
||||
destroy; what it lacks is `Datastore.Allocate`, without which deleting
|
||||
a seed is a 403 (`Permission check failed (/storage/<id>,
|
||||
Datastore.Allocate)`). With `--privsep 0` the token carries the user's
|
||||
privileges.
|
||||
|
||||
The token's secret is printed once. Save it on your workstation as one
|
||||
line, `proxy-operator@pve!operator=<secret>`, in
|
||||
`~/.config/proxmox-token` with mode 0600. Copy the node's CA,
|
||||
`/etc/pve/pve-root-ca.pem`, to `~/.config/proxmox-ca.pem`.
|
||||
5. **A proxy network with DHCP and a route to the internet.** Each VM
|
||||
keeps the template's one NIC, optionally moved to another bridge or
|
||||
VLAN by the config, and takes its address from that network's DHCP. The
|
||||
network needs internet twice over: the vendor-data script installs
|
||||
`qemu-guest-agent` from the distro mirrors at first boot, and the agent
|
||||
is the only way the operator learns the VM's address; and an egress
|
||||
proxy needs egress anyway. Baking `qemu-guest-agent` into the template
|
||||
and setting `installGuestAgent: false` removes the mirror from IP
|
||||
discovery. The operator's pods and the clients must route to that
|
||||
network: the health engine probes `http://<ip>:<port>` through the
|
||||
proxy, and clients get the same address from the discovery API. The
|
||||
provider does not use `spec.port` and configures no Proxmox firewall.
|
||||
6. **A reserved VMID range.** VMIDs are allocated from `vmidRange`
|
||||
(default 100000 to 199999). Keep hand-made VMs out of it: one there
|
||||
costs a probe, and one named like a proxy is refused (see *What lands
|
||||
on Proxmox*).
|
||||
|
||||
Check from the operator's point of view before wiring it in. This sends
|
||||
the token through a file descriptor, so it never reaches the command line:
|
||||
|
||||
```sh
|
||||
pve() {
|
||||
curl -s --cacert ~/.config/proxmox-ca.pem \
|
||||
-H @<(printf 'Authorization: PVEAPIToken=%s\n' "$(tr -d '[:space:]' < ~/.config/proxmox-token)") \
|
||||
"https://192.168.0.8:8006/api2/json/$1"; echo
|
||||
}
|
||||
pve version # the token works, TLS verifies
|
||||
pve nodes/pv1-1/storage/proxy-seeds/status # active, content includes iso
|
||||
pve 'cluster/resources?type=vm' # the template is listed with "template":1
|
||||
```
|
||||
|
||||
## Cluster-side setup
|
||||
|
||||
1. Create the Secret. It holds two keys, `token` (the one-line token file)
|
||||
and `ca.pem` (the node's CA):
|
||||
|
||||
```sh
|
||||
make proxmox-token-secret
|
||||
# or by hand:
|
||||
kubectl -n egress-proxies-operator-system create secret generic proxmox-token \
|
||||
--from-file=token=$HOME/.config/proxmox-token \
|
||||
--from-file=ca.pem=$HOME/.config/proxmox-ca.pem
|
||||
```
|
||||
|
||||
The make target runs that command as an apply, so it also updates an
|
||||
existing Secret. Override the paths with
|
||||
`PROXMOX_TOKEN_FILE=... PROXMOX_CA_FILE=...`. Before it applies, it
|
||||
checks that both files exist, that the token file is exactly one line
|
||||
of `<user>@<realm>!<tokenid>=<secret>` (without printing it), and that
|
||||
the CA file holds a PEM certificate. With `insecureSkipTLSVerify`
|
||||
instead of a CA, create the Secret by hand with the `token` key only.
|
||||
|
||||
`config/manager/manager.yaml` mounts the Secret (optional, read-only)
|
||||
at `/etc/proxy-operator/proxmox`, so the files are
|
||||
`/etc/proxy-operator/proxmox/token` and
|
||||
`/etc/proxy-operator/proxmox/ca.pem`. The deployment starts fine
|
||||
without it, even with a proxmox provider configured: `New` makes no
|
||||
request, only checks that the token file exists, and logs a `WARNING:`
|
||||
line when it does not. The provider's proxies then fail transiently and
|
||||
recover once the Secret lands; the health check reports
|
||||
`CredentialsUnavailable` meanwhile.
|
||||
|
||||
**Rotating the token needs no restart.** The token file is read on
|
||||
every request, so the next request after the kubelet refreshes the
|
||||
mount uses the new token. **Rotating the CA does**: it is loaded on
|
||||
first use and cached for the life of the process. A missing CA file is
|
||||
retried until it lands; one without a certificate is a permanent error.
|
||||
|
||||
2. Add a provider entry to the providers config
|
||||
(`config/manager/providers_config.yaml` in-cluster; the full sample,
|
||||
commented out, is in `config/samples/providers-config.yaml`):
|
||||
|
||||
```yaml
|
||||
- name: proxmox-home
|
||||
type: proxmox
|
||||
proxmox:
|
||||
url: https://192.168.0.8:8006
|
||||
tokenFile: /etc/proxy-operator/proxmox/token
|
||||
caFile: /etc/proxy-operator/proxmox/ca.pem
|
||||
nodes: [pv1-1]
|
||||
isoStorage: proxy-seeds
|
||||
vlanTag: 4
|
||||
templates:
|
||||
debian-13: {vmid: 9001}
|
||||
machineTypes:
|
||||
small: {cores: 1, memoryMiB: 1024}
|
||||
```
|
||||
|
||||
**The URL's host must be in the node certificate.** The node's
|
||||
self-signed certificate lists its own addresses and names, and TLS
|
||||
verification checks the URL's host against them. On `pv1-1` it lists
|
||||
`192.168.0.8`, `pv1-1` and `pv1-1.home.lab`, so
|
||||
`https://192.168.2.8:8006` reaches the same node and still fails
|
||||
verification (`CertificateInvalid` in the health check, a permanent
|
||||
error on every call). See what yours lists with:
|
||||
|
||||
```sh
|
||||
openssl s_client -connect 192.168.0.8:8006 </dev/null 2>/dev/null | openssl x509 -noout -ext subjectAltName
|
||||
```
|
||||
|
||||
If the address the operator must use is not there, either give the
|
||||
node a certificate that names it (a custom certificate in the node's
|
||||
Certificates panel) and put that certificate's CA in `caFile`, or set
|
||||
`insecureSkipTLSVerify: true`, which `New` logs as a `WARNING:` line.
|
||||
|
||||
`nodes` holds node names as Proxmox knows them, not addresses: the one
|
||||
URL reaches the whole cluster. The providers config is read once per
|
||||
process, so adding a node, a template or a machine type needs a manager
|
||||
restart.
|
||||
|
||||
3. Apply a Proxy (`config/samples/proxy_proxmox.yaml`) and watch it:
|
||||
|
||||
```sh
|
||||
kubectl get px -w
|
||||
# NAME MODE PROVIDER PHASE IP HEALTHY
|
||||
# proxy-proxmox-sample Managed proxmox-home Provisioning
|
||||
# proxy-proxmox-sample Managed proxmox-home Running 192.168.4.147
|
||||
# proxy-proxmox-sample Managed proxmox-home Ready 192.168.4.147 True
|
||||
```
|
||||
|
||||
(Illustrative.) `Create` returns once the VM runs, a few seconds after
|
||||
the clone; the address follows when the guest agent is up, about 50
|
||||
seconds after the start on `pv1-1` with the agent installed at first
|
||||
boot.
|
||||
|
||||
## Config reference
|
||||
|
||||
The block is `proxmox:` and its Go type is `ProxmoxConfig` in
|
||||
`internal/provider/config_proxmox.go`, validated at load by
|
||||
`validateProxmox`; unknown keys are rejected. Defaults are applied by
|
||||
`withDefaults` in `internal/provider/proxmox/proxmox.go`.
|
||||
|
||||
| Field | Required | Default | Meaning |
|
||||
|---|---|---|---|
|
||||
| `url` | yes | | `https://<host>:8006`, with no path (the client appends `/api2/json`). The host must be in the node certificate |
|
||||
| `tokenFile` | yes | | file holding one line, `<user>@<realm>!<tokenid>=<secret>`, read on every request and sent as `Authorization: PVEAPIToken=<line>` |
|
||||
| `caFile` | | system roots | PEM bundle trusted for the API's certificate, typically the node's `/etc/pve/pve-root-ca.pem`. Loaded on first use and cached |
|
||||
| `insecureSkipTLSVerify` | | `false` | skip certificate verification; `New` logs a `WARNING:` line when it is on |
|
||||
| `nodes` | yes, at least one | | node names `placement.zone` selects from; unique, lowercase letters, digits and dashes. They end up in request paths and as the clone's `target` |
|
||||
| `isoStorage` | yes | | storage id the seeds are uploaded to and deleted from. Must hold `iso` content; the token needs `Datastore.Allocate` on it. No default on purpose |
|
||||
| `storage` | | the template's disk storage | storage id the full clone's disks go to |
|
||||
| `bridge` | | as cloned | replaces `bridge=` in the cloned `net0` |
|
||||
| `vlanTag` | | as cloned | 1 to 4094; replaces or appends `tag=` in the cloned `net0` |
|
||||
| `vmidRange.min`, `vmidRange.max` | | `100000`, `199999` | where VMIDs are allocated; at least 100, at most 999999999, min below max |
|
||||
| `templates.<key>.vmid` | at least one template | | the key names the template in `placement.image`; the VMID is at least 100 |
|
||||
| `machineTypes.<key>` | at least one | | `{cores, memoryMiB, diskGiB}` size profile. `cores` and `memoryMiB` above zero. `diskGiB` is optional: when larger than the cloned boot disk the disk is grown to it; it is never shrunk |
|
||||
| `requestTimeoutSeconds` | | `30` | bound on one API request, response included. The seed upload is one request, so a multi-megabyte `vendorData` needs more |
|
||||
| `vendorData` | | | extra cloud-init vendor-data placed on every seed next to the Proxy's user-data; a MIME multipart part beside the agent script when `installGuestAgent` is on. Cloud-init drops any top-level key of a vendor-data cloud-config that the user-data also defines |
|
||||
| `installGuestAgent` | | `true` | add the vendor-data script that installs and starts `qemu-guest-agent`. Turn it off for templates that ship the agent |
|
||||
|
||||
## Placement
|
||||
|
||||
All three catalogue fields are checked before any request. An unknown
|
||||
value is `ErrPermanent`, which the reconciler latches as `Failed` with the
|
||||
message below until the spec changes.
|
||||
|
||||
| Proxy field | Meaning for proxmox | Error when unknown |
|
||||
|---|---|---|
|
||||
| `placement.zone` | a node from `nodes`. May be omitted when exactly one node is configured; otherwise required | `placement.zone "x" is not in proxmox.nodes`; with several nodes and no zone, `placement.zone is required when proxmox.nodes lists more than one node` |
|
||||
| `placement.machineType` | a key in `machineTypes` | `placement.machineType "x" is not in proxmox.machineTypes` |
|
||||
| `placement.image` | a key in `templates` | `placement.image "x" is not in proxmox.templates` |
|
||||
| `placement.region` | ignored. It is still part of the spec hash, so changing it replaces the VM and does nothing else | |
|
||||
|
||||
The messages come from `placement` in `internal/provider/proxmox/create.go`
|
||||
and `node` in `proxmox.go`. A key that exists but names a VMID that is
|
||||
missing or not a template fails later, before any clone:
|
||||
`placement.image "x": template VM 9001 does not exist` or `... is not a
|
||||
template`.
|
||||
|
||||
**Catalogue edits affect only new VMs.** The spec hash covers the
|
||||
placement values, not what the config maps them to, so repointing
|
||||
`templates.debian-13.vmid` at another template changes the VMs created
|
||||
afterwards and leaves the running ones alone. Add a new key rather than
|
||||
repointing one in use, so the Proxies that name a key keep a meaning you
|
||||
can reason about. A template may be edited, replaced or deleted without
|
||||
affecting existing VMs: they are full clones and share nothing with it.
|
||||
|
||||
## What lands on Proxmox
|
||||
|
||||
For a Proxy whose deterministic name is `proxy-abc…`:
|
||||
|
||||
| Object | Name | Lifetime |
|
||||
|---|---|---|
|
||||
| VM | name `proxy-abc…`, a VMID from `vmidRange` | with the Proxy |
|
||||
| its disks | the template's disks, fully copied onto `storage` (or the template's storage) | destroyed with the VM (`purge=1&destroy-unreferenced-disks=1`) |
|
||||
| seed ISO | `<isoStorage>:iso/proxy-abc…-seed.iso` on the VM's node | with the Proxy |
|
||||
|
||||
**The VMID** is derived from the Proxy's UID, so a retried Create lands on
|
||||
the same number: an FNV-1a hash of the UID placed in `vmidRange`, moving
|
||||
forward one at a time on collision, up to 32 probes (`vmidFor` in
|
||||
`id.go`). Before allocating, `Create` looks the name up in the cluster's
|
||||
resources. A VMID that is taken, or a VM that already has the name, goes
|
||||
through an ownership check (`inspect` in `create.go`): a VM with a lock
|
||||
(a clone still running, which also has no name yet) is waited on; then
|
||||
|
||||
- another name: someone else's VM, so the next VMID is probed;
|
||||
- our name and our mark with our UID or an empty one: **adopted**, and
|
||||
Create carries on from where the previous attempt stopped;
|
||||
- our name without the mark, or with another UID: **permanent error**
|
||||
naming the VMID and both UIDs. The operator never takes over a VM it did
|
||||
not create;
|
||||
- our name, our mark, hibernated (`lock: suspended`): **permanent error**,
|
||||
`resume or destroy it in Proxmox` (see *Known gaps*).
|
||||
|
||||
Two VMs with the proxy's name are a permanent error naming every VMID;
|
||||
remove all but one.
|
||||
|
||||
**The providerID** is `vms/<vmid>/<name>`. It names no node, because VMs
|
||||
migrate: the node is looked up in `GET /cluster/resources` on every call.
|
||||
`Get` and `Delete` check that the VM at that VMID still has that name and
|
||||
the mark. Another name means ours is gone and the VMID was reused: `Get`
|
||||
answers NotFound, `Delete` succeeds without touching it.
|
||||
|
||||
**The mark** is three lines in the VM's description (Notes in the UI),
|
||||
passed in the clone call itself, so a VM is never unmarked:
|
||||
|
||||
```
|
||||
proxy-operator-managed=true
|
||||
proxy-operator-uid=0f3a…
|
||||
proxy-operator-created-at=2026-09-23T17:42:11Z
|
||||
```
|
||||
|
||||
That is how orphan GC recognises its own VMs and how a repeat `Create`
|
||||
knows whether a VM of the same name belongs to this Proxy. Lines an admin
|
||||
adds to the description are kept and ignored. The clone's own
|
||||
`meta: ctime=` is useless for this: Proxmox copies it from the template.
|
||||
|
||||
**The config write.** After the clone and the seed upload, one
|
||||
`PUT /nodes/{node}/qemu/{vmid}/config` sets:
|
||||
|
||||
| Key | Value |
|
||||
|---|---|
|
||||
| `cores`, `memory` | from the machine type |
|
||||
| `agent` | `1` |
|
||||
| `tags` | `proxy_operator` |
|
||||
| `ide2` | `<isoStorage>:iso/<name>-seed.iso,media=cdrom` |
|
||||
| `net0` | only when `bridge` or `vlanTag` is set: the cloned value with `bridge=` and `tag=` replaced or appended, the model, MAC and every other option kept |
|
||||
|
||||
The write **replaces the template's `agent` option and tags**: a
|
||||
template's `agent: 1,fstrim_cloned_disks=1` becomes `agent: 1`, and its
|
||||
tags give way to `proxy_operator`. The tag is for filtering in the
|
||||
Proxmox UI and narrows GC's candidates; the mark decides. Proxmox appends
|
||||
`ide2` to the boot order on its own (`order=scsi0;ide2`), which is
|
||||
harmless because the boot disk stays first. When the machine type's
|
||||
`diskGiB` is larger than the boot disk (the first disk in `boot: order=`
|
||||
that is not a CD-ROM), a resize task grows it; cloud-init's growpart
|
||||
expands the root filesystem at first boot.
|
||||
|
||||
**The seed** (`nocloud.BuildSeedISO`, shared with kvm2 in
|
||||
`internal/provider/nocloud`) is an ISO 9660 image labelled `cidata`.
|
||||
`user-data` is the Proxy's `cloudInit` **byte for byte**; the operator
|
||||
never parses or re-renders it. `meta-data` holds `instance-id: <name>` and
|
||||
`local-hostname: <name>`. `vendor-data` carries the guest-agent script
|
||||
(a shell script, not a cloud-config, so a Proxy's own `packages:` cannot
|
||||
shadow it), joined with `vendorData` as a MIME multipart when both are
|
||||
set. It is staged under `/tmp` (the manager's emptyDir), built in memory
|
||||
and uploaded with `POST /nodes/{node}/storage/{isoStorage}/upload`. An
|
||||
upload overwrites a file of the same name, so it is rebuilt and
|
||||
re-uploaded on every Create attempt with no delete first: a previous
|
||||
attempt may have been cut off. Only the API is used; the storage upload
|
||||
endpoint does not accept `snippets`, so Proxmox's own `cicustom` would
|
||||
need shell access to the node.
|
||||
|
||||
**What a retried Create finds.** Every step is safe to repeat. After a
|
||||
crash mid-clone, the retry lands on the same VMID, waits for the clone's
|
||||
lock, and adopts the VM. A stopped VM of ours gets the seed, the config
|
||||
and the resize again (the upload overwrites, the config is a replace, the
|
||||
resize is skipped once large enough), then the start. A running one is
|
||||
returned at once.
|
||||
|
||||
**A clone lock that never clears.** A node that crashes mid-clone can
|
||||
leave the VM locked with `lock: clone` for good. Every Create for that
|
||||
Proxy then waits out the 10 minute clone budget and returns a transient
|
||||
error, and the reconciler keeps retrying rather than latching. Clear it
|
||||
on the node:
|
||||
|
||||
```sh
|
||||
qm unlock <vmid> # then let the next Create adopt and finish it
|
||||
qm destroy <vmid> --purge # or remove it and let Create clone again
|
||||
```
|
||||
|
||||
**Seeds without a VM.** A VM destroyed outside the operator leaves its
|
||||
seed ISO, which GC cannot see (it lists VMs). Seeds are named
|
||||
`proxy-*-seed.iso` on `isoStorage`; any without a matching VM may be
|
||||
removed by hand:
|
||||
|
||||
```sh
|
||||
pvesm list proxy-seeds --content iso
|
||||
pvesm free proxy-seeds:iso/proxy-abc…-seed.iso
|
||||
```
|
||||
|
||||
**Delete** runs under a per-name lock shared with Create: stop (a paused
|
||||
VM too), destroy with `purge=1&destroy-unreferenced-disks=1`, then delete
|
||||
the seed on the VM's node. Each is a task that is waited on. A VM that is
|
||||
already gone, not ours, or unmarked is left alone, and the seed is then
|
||||
looked for on every configured node, where a node that fails is logged
|
||||
and skipped. On the VM's own node a failed seed delete fails the call:
|
||||
**without `Datastore.Allocate` on `isoStorage` the 403 makes Delete
|
||||
permanent**, and the Proxy's deletion stays stuck with that message until
|
||||
the privilege is granted. When Delete returns nil, nothing of the Proxy
|
||||
is left.
|
||||
|
||||
## Logging
|
||||
|
||||
The provider logs through controller-runtime's logger, named `proxmox`,
|
||||
with `provider` (the configured name) on every line, and `vmid`, `name`
|
||||
and `node` on every line about an instance. Every line inside a traced
|
||||
operation also carries `traceID` and `spanID`.
|
||||
|
||||
- At startup (always on): `WARNING: token file is not readable; every
|
||||
call fails until it is` with `path` and `reason`, and `WARNING:
|
||||
insecureSkipTLSVerify is set`.
|
||||
- `--zap-log-level=debug` (logr V(1)):
|
||||
- one line per operation outcome: `proxmox vm ensured running`,
|
||||
`proxmox vm adopted` with `status` and `qmpstatus`,
|
||||
`proxmox vm fetched` with `state`, `ip`, `uid`, `status` and
|
||||
`qmpstatus`, `proxmox vm not found` with `reason`,
|
||||
`proxmox vm deleted` with `destroyed` (false when no VM of ours was
|
||||
left to destroy), `proxmox vms listed` with `vms`, `candidates`,
|
||||
`managed` and `skipped` counts, and `proxmox node checked` with
|
||||
`ready`, `reason` and `latency`;
|
||||
- **one `proxmox api call` line per API request**, written by the
|
||||
decorator in `internal/provider/proxmox/observe.go`: `call` (the seam
|
||||
method, for example `Clone`), the identifiers it was given (`node`,
|
||||
`vmid`, `storage`), `upid` for a task, `newid` and `name` on a clone,
|
||||
`filename` on an upload, `volid` on a volume delete, `duration`, and
|
||||
on failure `httpStatus` and `error`. A running `Get` is four calls; a
|
||||
fresh Create is a dozen or more, depending on how many polls each
|
||||
task takes;
|
||||
- `proxmox task failed` with `upid` and `exitstatus`, and
|
||||
`proxmox task finished with warnings`;
|
||||
- the no-op lines: `proxmox vm already gone`,
|
||||
`proxmox vm at this vmid is not ours, left alone` with `vmName`,
|
||||
`proxmox vm carries no proxy-operator mark, left alone`,
|
||||
`proxmox seed delete failed on a node, skipped`,
|
||||
`skipping unreadable proxmox vm`, and the probe's
|
||||
`proxmox vmid taken by another vm, probing on`.
|
||||
- `--zap-log-level=2`: `proxmox create resolved` with the resolved
|
||||
parameters (node, machine type, cores, memory, disk, template VMID,
|
||||
storages, bridge, VLAN tag) and `cloudInitBytes`; the `proxmox api
|
||||
call` lines also carry the config write's parameters, a summary of each
|
||||
config read, the clone's description, storage and target, the resize
|
||||
size, VM and task status, the agent's addresses and the upload's byte
|
||||
count; `proxmox guest agent answered`; `proxmox listed instance` per
|
||||
listed VM.
|
||||
- Never logged at any level: the cloud-init content, the token, the CA
|
||||
content and the upload body. There is no raw wire tier.
|
||||
|
||||
## Tracing
|
||||
|
||||
With an OTLP exporter configured ([architecture.md
|
||||
§10](architecture.md#10-tracing-internaltracing)), every API request is a
|
||||
child span of the operation span the decorators open (`provider.create`,
|
||||
`provider.get`, `provider.delete`, `provider.list`) or of the monitor's
|
||||
`providerhealth.check`:
|
||||
|
||||
| Span | Around | Attributes |
|
||||
|---|---|---|
|
||||
| `HTTP GET`, `HTTP POST`, `HTTP PUT`, `HTTP DELETE` | one API request, from `otelhttp` on the client's transport | otelhttp's standard HTTP client attributes: method, URL, server address and port, response status |
|
||||
|
||||
The transport is `tracing.RestConfigWrapper()`, the same parent-gated
|
||||
wrapper the kubernetes provider uses, so a request without a parent span
|
||||
starts none. The logging decorator starts no span of its own, so nothing
|
||||
is counted twice; its `proxmox api call` line carries the operation
|
||||
span's `spanID`, not the HTTP span's. The token travels in a header,
|
||||
which otelhttp does not record, and no attribute carries cloud-init.
|
||||
|
||||
## Timeouts
|
||||
|
||||
No caller puts a deadline on the context, so every request and every wait
|
||||
carries its own budget (`defaultBudgets` in
|
||||
`internal/provider/proxmox/proxmox.go`, and the transport in `client.go`):
|
||||
|
||||
| Bound | Budget |
|
||||
|---|---|
|
||||
| TCP connect | 10 s |
|
||||
| TLS handshake | 10 s |
|
||||
| one request, response included | `requestTimeoutSeconds`, default 30 s |
|
||||
| clone task, and the wait for a VM's lock in the ownership check | 10 m |
|
||||
| upload task, resize task, start task, stop task | 2 m each |
|
||||
| destroy task | 5 m |
|
||||
| seed delete task | 1 m |
|
||||
| wait for `running` after the start | 60 s |
|
||||
| task and status poll interval | 1 s |
|
||||
|
||||
A task wait that runs out is a transient error naming the UPID; the task
|
||||
keeps running on Proxmox, and the next attempt finds its result through
|
||||
the ownership check or the lock. A task that ends with any `exitstatus`
|
||||
other than `OK` or `WARNINGS: <n>` is transient too, with Proxmox's text.
|
||||
|
||||
Requests share one HTTP client and its idle connections (4 per host,
|
||||
closed after 90 s idle). A cancelled or overlong request ends only
|
||||
itself: nothing else in flight is disturbed, and there is no connection
|
||||
to kill as there is for kvm2. HTTP/2 is disabled, because pveproxy speaks
|
||||
HTTP/1.1 only. Create and Delete for the same name are serialised on a
|
||||
per-name lock, so the reconciler's Delete and GC's Delete never race on
|
||||
stop and destroy; Get and ListByTag take no lock.
|
||||
|
||||
## Health check
|
||||
|
||||
The operator checks every configured node every
|
||||
`--provider-check-interval` (default 1m) and publishes the result in the
|
||||
`ProviderHealth` object named after the provider
|
||||
(`kubectl get ph proxmox-home -o yaml`) and in the
|
||||
`proxy_operator_provider_target_up{provider,target}` gauge. One target per
|
||||
`nodes` entry, named after the node with the URL as its address, in
|
||||
config order, all checked concurrently.
|
||||
|
||||
Each check (`checkNode` in `internal/provider/proxmox/check.go`) first
|
||||
reads the token file for its id, then makes four requests on the shared
|
||||
client: `GET /version`, `GET /nodes/{node}/status`,
|
||||
`GET /nodes/{node}/storage/{isoStorage}/status` (the storage must exist,
|
||||
be active and hold `iso` content), and `GET /cluster/resources?type=vm`
|
||||
(every configured template must exist and be a template). A check that
|
||||
runs out of time cancels only its own requests.
|
||||
|
||||
Every target's `details` carry `isoStorage`. As facts arrive they gain
|
||||
`tokenID` (the part of the token before `=`, never the secret),
|
||||
`pveVersion`, `kernel`, `isoStorageAvailGiB` and `templates`
|
||||
(`<found>/<configured>`), so a target that fails late still says what the
|
||||
node answered before. There is no quota signal; the storage's free space
|
||||
is here instead.
|
||||
|
||||
| Reason | Meaning | Look at |
|
||||
|---|---|---|
|
||||
| `Reachable` | ready | |
|
||||
| `CredentialsUnavailable` | the token file is missing or not `<user>@<realm>!<tokenid>=<secret>`; no request was made | the `proxmox-token` Secret; fixed in the cluster, not in Proxmox |
|
||||
| `CertificateInvalid` | TLS verification failed, or `caFile` is missing or holds no certificate | `url` (its host must be in the node certificate) or `caFile`; not the node |
|
||||
| `Unreachable` | the TCP connect or TLS handshake failed for a network reason, the connection was cut mid-response, or pveproxy answered 595 or 596 (it cannot reach the node behind it) | routing and firewalls between the cluster and `url`; `pveproxy` running; the node up |
|
||||
| `Unauthorized` | 401: the token id or secret is wrong | the token file; `pveum user token list proxy-operator@pve` |
|
||||
| `Forbidden` | 403; the message names the path and the privilege | the roles in *Proxmox prerequisites* |
|
||||
| `Timeout` | the monitor's `--provider-check-timeout` (default 15s) ran out, whichever request was running, or a request ran out of `requestTimeoutSeconds` while the round still had time | a slow or hung pveproxy, a saturated link |
|
||||
| `StorageNotFound` | `isoStorage` answered with a 500 on that node (it does not exist, is not enabled there, or is offline), is not active, or holds no `iso` content; the message carries Proxmox's text | `pvesm status`; the storage's nodes and content types |
|
||||
| `TemplateNotFound` | a configured template VMID does not exist or is not a template | `qm list`; the `templates` catalogue |
|
||||
| `APIError` | any other failure | the message carries Proxmox's own text |
|
||||
|
||||
The check does not look inside the templates: a cloud-init drive, a
|
||||
missing `net0` or an unreadable disk size surface only when a Create is
|
||||
about to clone.
|
||||
|
||||
## Error classes
|
||||
|
||||
Every error is classified once, in `classify` in
|
||||
`internal/provider/proxmox/errors.go`, with errors that carry a class from
|
||||
their source (validation, the token and CA files, the ownership check)
|
||||
keeping it. The reconciler latches `ErrPermanent` as `Failed` until the
|
||||
spec changes, recreates on `ErrNotFound` from `Get`, and retries
|
||||
everything else with backoff. The provider never returns
|
||||
`ErrQuotaExceeded`: the thin `local-lvm` storage overprovisions, so no
|
||||
out-of-space answer could be produced to match, and capacity failures are
|
||||
transient.
|
||||
|
||||
| Class | What lands there |
|
||||
|---|---|
|
||||
| `ErrPermanent` | placement values not in the catalogue; a template VMID that is missing or not a template, has a cloud-init drive, or lacks `net0` or a sized boot disk when the config needs them; a malformed providerID; any 4xx (401 `Authentication failed!` or `No ticket`: the token; 403 `Permission check failed (<path>, <privilege>)`: a missing privilege, including `Datastore.Allocate` for the seed delete; 400 `Parameter verification failed.` with the field errors); a TLS verification failure; a token file that exists but is not `<user>@<realm>!<tokenid>=<secret>`; a CA file with no certificate; a VM with the proxy's name that carries no mark or another UID; two VMs with the proxy's name; our VM hibernated (`lock: suspended`), in both Create and Delete |
|
||||
| `ErrNotFound` | from `Get`: the VMID absent from the cluster's resources, the 500 `Configuration file 'nodes/<node>/qemu-server/<vmid>.conf' does not exist`, the VM at the VMID now named differently or unmarked. Inside `Create` the `does not exist` answer is `ErrTransient`, because the contract forbids NotFound from Create; `Delete` treats all of these as done |
|
||||
| `ErrTransient` | everything else: a missing token or CA file (the Secret has not landed yet); network errors and cancellation; 500 `VM is locked (<lock>)` and any lock seen in the resources or config; a VM whose node does not answer (`status: unknown`); the agent's answers that `Get` cannot place, such as `VM <vmid> is not running`; any other 5xx, including pveproxy's 595 and 596; a task that fails or runs out of its budget; no free VMID after 32 probes; the wait for running expiring, or a VM that stops right after its start |
|
||||
|
||||
`Get` turns two agent answers into states rather than errors:
|
||||
`QEMU guest agent is not running` is Provisioning (booting, or the agent
|
||||
install has not finished), and `No QEMU guest agent configured` is
|
||||
Stopped (someone removed `agent: 1`; the reconciler replaces the VM).
|
||||
|
||||
Error strings name VMIDs, nodes, storages, UPIDs and Proxmox's own text.
|
||||
They never contain the token, the CA content or cloud-init.
|
||||
|
||||
## Known gaps
|
||||
|
||||
- **Create blocks** until the VM runs, holding one of the three reconcile
|
||||
workers for its duration. On `pv1-1` that is about five seconds plus the
|
||||
clone; large templates on slow storage take longer. Other Proxies keep
|
||||
reconciling on the remaining workers.
|
||||
- **A clone lock that never clears** (a node crash mid-clone) makes every
|
||||
Create for that Proxy wait out the 10 minute clone budget and return a
|
||||
transient error. Clear it with `qm unlock <vmid>` or destroy the VM (see
|
||||
*What lands on Proxmox*).
|
||||
- **Seed ISOs of VMs destroyed outside the operator are invisible to
|
||||
GC.** Remove `proxy-*-seed.iso` files without a matching VM by hand.
|
||||
On a storage that is not shared, a VM migrated to another node leaves
|
||||
its seed on the original node; Delete then looks on the VM's current
|
||||
node, finds nothing, and the seed stays. Shared ISO storage avoids it.
|
||||
- **The VMID range is shared with humans.** A VM created by hand inside
|
||||
it costs a probe; a hand-made VM named like a proxy is refused as
|
||||
`ErrPermanent`. Reserve the range.
|
||||
- **Cross-node clones need shared storage and are untested.** When the
|
||||
zone's node differs from the template's, the clone is sent with
|
||||
`target`, which Proxmox allows only when the template's disks are on
|
||||
shared storage. The host this was built on has one node.
|
||||
- **A VM whose agent was switched off is replaced**, not repaired: with
|
||||
`agent: 1` removed it can never report an address, so `Get` reports it
|
||||
Stopped.
|
||||
- **The proxy network needs internet** for the agent install, unless the
|
||||
template ships the agent and `installGuestAgent` is `false`. Without it
|
||||
the VM boots, gets an address, and stays `Provisioning` forever,
|
||||
because the agent is the only address source. Check
|
||||
`systemctl status qemu-guest-agent` and
|
||||
`/var/log/cloud-init-output.log` on the guest (`qm terminal <vmid>` on
|
||||
the node, or the console in the UI).
|
||||
- **The CA is cached** after first use; rotating it needs a manager
|
||||
restart. The token rotates freely.
|
||||
- **A node that stops answering** makes `Get` and `Delete` for its VMs
|
||||
transient until it returns, so deleting such a Proxy waits. Proxmox
|
||||
lists an unreachable node's guests with `status: unknown` rather than
|
||||
dropping them, so they are never mistaken for gone. That is Proxmox's
|
||||
documented behaviour; a single-node host could not show it.
|
||||
- **Contract limits that apply unchanged** ([provider-contract.md
|
||||
§13](provider-contract.md)): the providers config is read once per
|
||||
process; GC has no installation identity, so two operators sharing a
|
||||
Proxmox cluster and a VMID range delete each other's VMs; and a Proxy
|
||||
naming an unconfigured provider latches until its spec changes.
|
||||
|
||||
Beyond the ten limits accepted in the design:
|
||||
|
||||
- **A hibernated VM latches its Proxy.** A VM of ours with
|
||||
`lock: suspended` (hibernated to disk) makes both Create and Delete fail
|
||||
permanently with `resume or destroy it in Proxmox`: the lock never
|
||||
clears on its own, and the destroy refuses a locked VM. `Get` reports it
|
||||
Stopped. Resume it by starting it (`qm start <vmid>` restores the
|
||||
hibernated state) or destroy it on the node, then touch the Proxy's
|
||||
spec, or let the pending Delete retry.
|
||||
- **The seed upload is one request**, bounded by `requestTimeoutSeconds`.
|
||||
A seed is tens of KiB, but a multi-megabyte `vendorData` would need a
|
||||
larger timeout.
|
||||
- **The sample cloud-init is an open proxy.**
|
||||
`config/samples/proxy_proxmox.yaml` installs Squid with
|
||||
`http_access allow all`; restrict it before exposing the proxy network
|
||||
to anything but the cluster.
|
||||
- **Not yet proven end to end.** The unit tests run over captured
|
||||
responses from `pv1-1` and an in-memory fake; the integration tests
|
||||
below compile but have not run against a host yet. Three answers were
|
||||
never captured: the storage status for a missing storage, a destroy on
|
||||
a hibernated VM, and a stop on a paused VM.
|
||||
|
||||
## Testing
|
||||
|
||||
Unit tests run over an in-memory fake of the API seam, and the HTTP
|
||||
client over a TLS test server replaying the responses captured from
|
||||
`pv1-1` (`internal/provider/proxmox/testdata/`). The optional integration
|
||||
test against a real host is described in
|
||||
[testing.md](testing.md#the-proxmox-integration-test-real-proxmox-ve-host),
|
||||
which also lists the variables it needs; run it with
|
||||
`make test-integration-proxmox`. `TestProxmoxCheck_realHost` runs the
|
||||
health check against the same host and creates nothing:
|
||||
`go test -tags=integration -count=1 -v -run TestProxmoxCheck ./internal/provider/proxmox/`.
|
||||
@@ -1,8 +1,12 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"sigs.k8s.io/yaml"
|
||||
)
|
||||
|
||||
const validProxmoxYAML = `
|
||||
@@ -130,3 +134,117 @@ machineTypes: {small: {cores: 1, memoryMiB: 1024}}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// commentedEntry cuts the commented-out providers entry named name out of a
|
||||
// config file and strips its "# " at the entry's own indent, so a sample
|
||||
// that is shipped commented out is still loaded by a test.
|
||||
func commentedEntry(t *testing.T, path, name string) string {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("reading %s: %v", path, err)
|
||||
}
|
||||
var out []string
|
||||
var prefix string
|
||||
for line := range strings.SplitSeq(string(data), "\n") {
|
||||
if prefix == "" {
|
||||
if strings.TrimSpace(line) == "# - name: "+name {
|
||||
prefix = line[:strings.Index(line, "#")]
|
||||
out = append(out, prefix+"- name: "+name)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !strings.HasPrefix(line, prefix+"# ") {
|
||||
break
|
||||
}
|
||||
out = append(out, prefix+strings.TrimPrefix(line, prefix+"# "))
|
||||
}
|
||||
if prefix == "" {
|
||||
t.Fatalf("%s has no commented entry %q", path, name)
|
||||
}
|
||||
return strings.Join(out, "\n") + "\n"
|
||||
}
|
||||
|
||||
func TestLoadConfig_proxmoxSampleEntry(t *testing.T) {
|
||||
t.Parallel()
|
||||
for name, path := range map[string]string{
|
||||
"sample": "../../config/samples/providers-config.yaml",
|
||||
"configmap": "../../config/manager/providers_config.yaml",
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
cfg, err := LoadConfig([]byte("providers:\n" + commentedEntry(t, path, "proxmox-home")))
|
||||
if err != nil {
|
||||
t.Fatalf("LoadConfig(proxmox-home from %s) error = %v, want nil", path, err)
|
||||
}
|
||||
p := cfg.Providers[0]
|
||||
// LoadConfig already refused a proxmox block under another type.
|
||||
if p.Name != "proxmox-home" || p.Proxmox == nil {
|
||||
t.Fatalf("provider %q has proxmox block %v, want proxmox-home with one", p.Name, p.Proxmox)
|
||||
}
|
||||
c := p.Proxmox
|
||||
if c.URL != "https://192.168.0.8:8006" {
|
||||
t.Errorf("url = %q, want https://192.168.0.8:8006 (the address in the node certificate)", c.URL)
|
||||
}
|
||||
if c.TokenFile != "/etc/proxy-operator/proxmox/token" || c.CAFile != "/etc/proxy-operator/proxmox/ca.pem" {
|
||||
t.Errorf("tokenFile, caFile = %q, %q, want the proxmox-token Secret's mount paths", c.TokenFile, c.CAFile)
|
||||
}
|
||||
if c.InsecureSkipTLSVerify {
|
||||
t.Error("insecureSkipTLSVerify = true, want it left commented out")
|
||||
}
|
||||
if len(c.Nodes) != 1 || c.Nodes[0] != "pv1-1" {
|
||||
t.Errorf("nodes = %v, want [pv1-1]", c.Nodes)
|
||||
}
|
||||
if c.ISOStorage != "proxy-seeds" || c.Storage != "" || c.Bridge != "" || c.VLANTag != 4 {
|
||||
t.Errorf("isoStorage, storage, bridge, vlanTag = %q, %q, %q, %d, want proxy-seeds, \"\", \"\", 4", c.ISOStorage, c.Storage, c.Bridge, c.VLANTag)
|
||||
}
|
||||
if len(c.Templates) != 1 || c.Templates["debian-13"].VMID != 9001 {
|
||||
t.Errorf("templates = %v, want {debian-13: {vmid: 9001}}", c.Templates)
|
||||
}
|
||||
if mt, ok := c.MachineTypes["small"]; !ok || mt.Cores != 1 || mt.MemoryMiB != 1024 || mt.DiskGiB != 0 {
|
||||
t.Errorf("machineTypes[small] = %+v (present %v), want {1 1024 0}", mt, ok)
|
||||
}
|
||||
if c.VMIDRange != nil || c.RequestTimeoutSeconds != 0 || c.InstallGuestAgent != nil || c.VendorData != "" {
|
||||
t.Errorf("optional fields set: %+v, want vmidRange, requestTimeoutSeconds, installGuestAgent and vendorData left commented out", c)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfig_proxmoxSampleProxyMatchesSampleEntry(t *testing.T) {
|
||||
t.Parallel()
|
||||
cfg, err := LoadConfig([]byte("providers:\n" + commentedEntry(t, "../../config/samples/providers-config.yaml", "proxmox-home")))
|
||||
if err != nil {
|
||||
t.Fatalf("LoadConfig() error = %v, want nil", err)
|
||||
}
|
||||
data, err := os.ReadFile("../../config/samples/proxy_proxmox.yaml")
|
||||
if err != nil {
|
||||
t.Fatalf("reading the sample Proxy: %v", err)
|
||||
}
|
||||
var px struct {
|
||||
Spec struct {
|
||||
Provider string `json:"provider"`
|
||||
Placement struct {
|
||||
Zone string `json:"zone"`
|
||||
MachineType string `json:"machineType"`
|
||||
Image string `json:"image"`
|
||||
} `json:"placement"`
|
||||
} `json:"spec"`
|
||||
}
|
||||
if err := yaml.Unmarshal(data, &px); err != nil {
|
||||
t.Fatalf("parsing the sample Proxy: %v", err)
|
||||
}
|
||||
c := cfg.Providers[0].Proxmox
|
||||
if px.Spec.Provider != cfg.Providers[0].Name {
|
||||
t.Errorf("spec.provider = %q, want %q", px.Spec.Provider, cfg.Providers[0].Name)
|
||||
}
|
||||
if !slices.Contains(c.Nodes, px.Spec.Placement.Zone) {
|
||||
t.Errorf("placement.zone = %q, not in nodes %v", px.Spec.Placement.Zone, c.Nodes)
|
||||
}
|
||||
if _, ok := c.MachineTypes[px.Spec.Placement.MachineType]; !ok {
|
||||
t.Errorf("placement.machineType = %q, not in machineTypes", px.Spec.Placement.MachineType)
|
||||
}
|
||||
if _, ok := c.Templates[px.Spec.Placement.Image]; !ok {
|
||||
t.Errorf("placement.image = %q, not in templates", px.Spec.Placement.Image)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user