kacerr/egress-proxies-operator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge the qemu-guest-agent bootstrap into each Proxy's cloud-config instead of vendor-data

Browse Source 
In the kind cluster two KVM Proxies with real cloud-configs booted fine
but never left Provisioning: the guest agent was not installed, so the
provider could not learn the IP. cloud-init merges vendor-data under
user-data with its default mergers, which keep the user-data's value
for any key both define and never append lists, so a cloud-config with
its own `packages:` silently dropped the vendor-data's packages and
runcmd. The integration test passed only because its user-data was
empty.

withGuestAgent now appends qemu-guest-agent to `packages` and the
enable command to `runcmd` of the Proxy's own cloud-config (skipping
entries already present) and re-renders it. Vendor-data is no longer
defaulted; `vendorData` is plain extra data. User-data that is not a
cloud-config document (script, #include, MIME) still gets the bootstrap
as vendor-data. A cloud-config that is not a YAML mapping, or whose
packages/runcmd is not a list, fails the Proxy as ErrPermanent.

Validated in-cluster: both Proxies Ready and healthy in two minutes, 204
through squid from the LAN. Docs, sample comment, CHANGELOG and the
plan execution log (Run 7) updated.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
kacerr
2026-09-10 21:06:54 +00:00
co-authored by Claude Fable 5.1
parent 220967660d
commit 366736f53f
9 changed files with 382 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/provider/config_kvm.go
+4 -3
View File
@@ -22,9 +22,10 @@ type KVMConfig struct {
// DiskSizeGB is the capacity of each VM's overlay disk. Default 10. The
// guest's cloud-init growpart expands the root filesystem into it.
DiskSizeGB int64 `json:"diskSizeGb,omitempty"`
// VendorData is cloud-init vendor-data placed on every seed next to the
// Proxy's own user-data. Default: installs and starts qemu-guest-agent so
// the provider can read the VM's IP even before it shows up in ARP.
// VendorData is optional cloud-init vendor-data placed on every seed next
// to the Proxy's own user-data. cloud-init drops any top-level key here
// that the user-data also defines. The qemu-guest-agent bootstrap does
// not live here: the provider merges it into each Proxy's cloud-config.
VendorData string `json:"vendorData,omitempty"`
// Images are named base images; placement.image names a key here (or is
// an http(s) URL directly). The provider downloads each image into a
internal/provider/kvm/kvm.go
+5 -4
View File
@@ -42,9 +42,6 @@ func withDefaults(cfg provider.KVMConfig) provider.KVMConfig {
if cfg.DiskSizeGB == 0 {
cfg.DiskSizeGB = defaultDiskSizeGB
}
if cfg.VendorData == "" {
cfg.VendorData = defaultVendorData
}
images := make(map[string]provider.KVMImage, len(cfg.Images))
for k, img := range cfg.Images {
if img.Format == "" {
@@ -249,7 +246,11 @@ func (p *Provider) writeSeed(log logr.Logger, api libvirtAPI, pool libvirt.Stora
if err := deleteIfExists(api, pool, seedName); err != nil {
return libvirt.StorageVol{}, err
}
iso, err := buildSeedISO(name, userData, p.cfg.VendorData)
userData, vendorData, err := withGuestAgent(userData, p.cfg.VendorData)
if err != nil {
return libvirt.StorageVol{}, fmt.Errorf("%w: %v", provider.ErrPermanent, err)
}
iso, err := buildSeedISO(name, userData, vendorData)
if err != nil {
return libvirt.StorageVol{}, fmt.Errorf("%w: %v", provider.ErrPermanent, err)
}
internal/provider/kvm/kvm_test.go
+7 -3
View File
@@ -30,8 +30,8 @@ func TestWithDefaults(t *testing.T) {
if cfg.DiskSizeGB != defaultDiskSizeGB {
t.Errorf("diskSizeGb = %d, want %d", cfg.DiskSizeGB, defaultDiskSizeGB)
}
if cfg.VendorData != defaultVendorData {
t.Errorf("vendorData = %q, want the default", cfg.VendorData)
if cfg.VendorData != "" {
t.Errorf("vendorData = %q, want none: the guest-agent bootstrap is merged into the user-data, not defaulted here", cfg.VendorData)
}
if cfg.Images["debian"].Format != "qcow2" || cfg.Images["raw"].Format != "raw" {
t.Errorf("image formats = %q/%q, want qcow2 default and raw kept", cfg.Images["debian"].Format, cfg.Images["raw"].Format)
@@ -187,6 +187,10 @@ func assertCreated(t *testing.T, api *fakeAPI) {
if seed.format != formatRaw || len(seed.data) == 0 || int64(len(seed.data)) != seed.capacity {
t.Errorf("seed = format %s, %d bytes, capacity %d; want raw with capacity == uploaded size", seed.format, len(seed.data), seed.capacity)
}
_, files := readSeed(t, seed.data)
if ud := files["user-data"]; !strings.Contains(ud, "squid") || !strings.Contains(ud, guestAgentPackage) {
t.Errorf("seed user-data = %q, want the Proxy's cloud-config with the guest agent merged in", ud)
}
}
func TestCreate_freshHost(t *testing.T) {
@@ -956,7 +960,7 @@ func TestLogging_verbosityTiers(t *testing.T) {
ctx, snapshot := captureContext(tc.verbosity)
req := testRequest()
const sentinel = "SENTINEL-cloud-init-must-never-be-logged"
req.CloudInit = "#cloud-config\n" + sentinel + "\n"
req.CloudInit = "#cloud-config\nhostname: " + sentinel + "\n"
runAllOps(t, ctx, req)
internal/provider/kvm/seed.go
+94 -8
View File
@@ -3,28 +3,114 @@ package kvm
import (
"bytes"
"fmt"
"reflect"
"strings"
"github.com/kdomanski/iso9660"
"sigs.k8s.io/yaml"
)
// seedLabel is the volume label cloud-init's NoCloud datasource looks for.
const seedLabel = "cidata"
// defaultVendorData is the cloud-init vendor-data shipped on every seed
// unless the config overrides it. cloud-init merges it under the Proxy's own
// user-data, so stock images grow a guest agent (which lets the provider
// read the VM's IP before ARP sees traffic) without the user's cloud-init
// having to know about it.
const defaultVendorData = `#cloud-config
const (
cloudConfigHeader = "#cloud-config"
guestAgentPackage = "qemu-guest-agent"
)
// guestAgentEnable is the runcmd entry that starts the agent once it is
// installed, so the provider can read the VM's IP before ARP sees traffic.
var guestAgentEnable = []any{"systemctl", "enable", "--now", guestAgentPackage}
// guestAgentVendorData is the same bootstrap as a standalone cloud-config.
// It is only used as vendor-data when the Proxy's user-data is not a
// cloud-config document (a shell script, a MIME archive, an #include), i.e.
// when there are no top-level keys for it to collide with.
const guestAgentVendorData = cloudConfigHeader + `
packages:
- qemu-guest-agent
runcmd:
- [systemctl, enable, --now, qemu-guest-agent]
`
// buildSeedISO renders a NoCloud seed: user-data verbatim, vendor-data as
// given, and a meta-data whose instance-id is the deterministic VM name, so
// withGuestAgent decides where the guest-agent bootstrap goes and returns
// the user-data and vendor-data to put on the seed.
//
// It cannot simply ride along as vendor-data: cloud-init merges vendor-data
// under user-data with its default mergers, which keep the user-data's
// value for any key both define and never append lists. A Proxy whose
// cloud-config has its own `packages:` (most do) would silently lose the
// agent and sit in Provisioning forever. So for a cloud-config user-data
// the bootstrap is merged into the document itself: `qemu-guest-agent`
// appended to `packages`, the enable command to `runcmd`, both skipped
// when already present. Anything the operator cannot parse as a
// cloud-config mapping is an error — cloud-init would reject it too, and a
// VM that boots without its config is harder to diagnose than a Proxy that
// fails fast.
func withGuestAgent(userData, vendorData string) (string, string, error) {
if !isCloudConfig(userData) {
if vendorData == "" {
vendorData = guestAgentVendorData
}
return userData, vendorData, nil
}
var cfg map[string]any
if err := yaml.Unmarshal([]byte(userData), &cfg); err != nil {
return "", "", fmt.Errorf("parsing cloud-config user-data: %w", err)
}
if cfg == nil {
cfg = map[string]any{}
}
pkgs, err := appendUnique(cfg, "packages", guestAgentPackage)
if err != nil {
return "", "", err
}
cfg["packages"] = pkgs
cmds, err := appendUnique(cfg, "runcmd", guestAgentEnable)
if err != nil {
return "", "", err
}
cfg["runcmd"] = cmds
out, err := yaml.Marshal(cfg)
if err != nil {
return "", "", fmt.Errorf("rendering cloud-config user-data: %w", err)
}
return cloudConfigHeader + "\n" + string(out), vendorData, nil
}
// isCloudConfig reports whether userData is a cloud-config document by
// cloud-init's own rule: the first line is the #cloud-config header. Empty
// user-data counts, so a Proxy without cloud-init still gets the agent.
func isCloudConfig(userData string) bool {
if strings.TrimSpace(userData) == "" {
return true
}
first, _, _ := strings.Cut(userData, "\n")
return strings.TrimSpace(first) == cloudConfigHeader
}
// appendUnique returns cfg[key] with item appended unless an equal entry
// is already there. A missing key is an empty list; anything but a list is
// an error, since cloud-init expects one there as well.
func appendUnique(cfg map[string]any, key string, item any) ([]any, error) {
var items []any
switch v := cfg[key].(type) {
case nil:
case []any:
items = v
default:
return nil, fmt.Errorf("cloud-config %s: want a list, got %T", key, v)
}
for _, it := range items {
if reflect.DeepEqual(it, item) {
return items, nil
}
}
return append(items, item), nil
}
// buildSeedISO renders a NoCloud seed: user-data and vendor-data verbatim,
// and a meta-data whose instance-id is the deterministic VM name, so
// cloud-init never mistakes a reboot for a new instance. The ISO's volume
// descriptor carries the build time, so two builds differ in a few
// timestamp bytes; the files and label are identical.
internal/provider/kvm/seed_test.go
+152 -3
View File
@@ -4,10 +4,12 @@ import (
"bytes"
"io"
"maps"
"reflect"
"strings"
"testing"
"github.com/kdomanski/iso9660"
"sigs.k8s.io/yaml"
)
func readSeed(t *testing.T, iso []byte) (label string, files map[string]string) {
@@ -42,7 +44,7 @@ func readSeed(t *testing.T, iso []byte) (label string, files map[string]string)
func TestBuildSeedISO_contents(t *testing.T) {
t.Parallel()
userData := "#cloud-config\npackages: [squid]\n"
iso, err := buildSeedISO("proxy-abc123def456ghij", userData, defaultVendorData)
iso, err := buildSeedISO("proxy-abc123def456ghij", userData, guestAgentVendorData)
if err != nil {
t.Fatalf("buildSeedISO() error = %v", err)
}
@@ -56,7 +58,7 @@ func TestBuildSeedISO_contents(t *testing.T) {
if files["user-data"] != userData {
t.Errorf("user-data = %q, want verbatim cloud-init", files["user-data"])
}
if files["vendor-data"] != defaultVendorData {
if files["vendor-data"] != guestAgentVendorData {
t.Errorf("vendor-data = %q", files["vendor-data"])
}
if got, want := files["meta-data"], "instance-id: proxy-abc123def456ghij\nlocal-hostname: proxy-abc123def456ghij\n"; got != want {
@@ -89,7 +91,7 @@ func TestBuildSeedISO_deterministicContent(t *testing.T) {
func TestBuildSeedISO_emptyUserData(t *testing.T) {
t.Parallel()
iso, err := buildSeedISO("proxy-x", "", defaultVendorData)
iso, err := buildSeedISO("proxy-x", "", guestAgentVendorData)
if err != nil {
t.Fatalf("buildSeedISO() error = %v", err)
}
@@ -98,3 +100,150 @@ func TestBuildSeedISO_emptyUserData(t *testing.T) {
t.Errorf("user-data = %q (present=%v), want an empty file to still exist", body, ok)
}
}
// parseCloudConfig is what cloud-init sees: the document under the header.
func parseCloudConfig(t *testing.T, doc string) map[string]any {
t.Helper()
if first, _, _ := strings.Cut(doc, "\n"); first != cloudConfigHeader {
t.Fatalf("first line = %q, want %q", first, cloudConfigHeader)
}
var cfg map[string]any
if err := yaml.Unmarshal([]byte(doc), &cfg); err != nil {
t.Fatalf("merged user-data is not YAML: %v\n%s", err, doc)
}
return cfg
}
func TestWithGuestAgent_mergesIntoCloudConfig(t *testing.T) {
t.Parallel()
agentCmd := []any{"systemctl", "enable", "--now", "qemu-guest-agent"}
tests := []struct {
name string
userData string
wantPackages []any
wantRuncmd []any
}{
{
name: "empty document",
userData: "#cloud-config\n",
wantPackages: []any{"qemu-guest-agent"},
wantRuncmd: []any{agentCmd},
},
{
name: "no user-data at all",
userData: "",
wantPackages: []any{"qemu-guest-agent"},
wantRuncmd: []any{agentCmd},
},
{
// The case that motivated this: cloud-init's default merger keeps
// the user-data's lists and drops vendor-data's, so the agent must
// be appended to the user's own packages and runcmd.
name: "user packages and runcmd are kept and extended",
userData: "#cloud-config\npackages:\n - squid\nruncmd:\n - systemctl restart squid\n" +
"write_files:\n - path: /etc/squid/conf.d/x.conf\n content: |\n http_port 3128\n via off\n",
wantPackages: []any{"squid", "qemu-guest-agent"},
wantRuncmd: []any{"systemctl restart squid", agentCmd},
},
{
name: "agent already present is not duplicated",
userData: "#cloud-config\npackages: [qemu-guest-agent]\nruncmd:\n - [systemctl, enable, --now, qemu-guest-agent]\n",
wantPackages: []any{"qemu-guest-agent"},
wantRuncmd: []any{agentCmd},
},
{
name: "header with trailing whitespace",
userData: "#cloud-config \r\npackages: [curl]\n",
wantPackages: []any{"curl", "qemu-guest-agent"},
wantRuncmd: []any{agentCmd},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
user, vendor, err := withGuestAgent(tc.userData, "")
if err != nil {
t.Fatalf("withGuestAgent() error = %v", err)
}
if vendor != "" {
t.Errorf("vendor-data = %q, want none when the agent is merged into the user-data", vendor)
}
cfg := parseCloudConfig(t, user)
if !reflect.DeepEqual(cfg["packages"], tc.wantPackages) {
t.Errorf("packages = %#v, want %#v", cfg["packages"], tc.wantPackages)
}
if !reflect.DeepEqual(cfg["runcmd"], tc.wantRuncmd) {
t.Errorf("runcmd = %#v, want %#v", cfg["runcmd"], tc.wantRuncmd)
}
})
}
}
func TestWithGuestAgent_keepsOtherKeysAndVendorData(t *testing.T) {
t.Parallel()
userData := "#cloud-config\nhostname: px\nwrite_files:\n - path: /etc/x\n content: |\n line one\n line two\n permissions: '0644'\nbootcmd: [echo hi]\n"
user, vendor, err := withGuestAgent(userData, "#cloud-config\nntp: {enabled: true}\n")
if err != nil {
t.Fatalf("withGuestAgent() error = %v", err)
}
if vendor != "#cloud-config\nntp: {enabled: true}\n" {
t.Errorf("vendor-data = %q, want the configured one verbatim", vendor)
}
got := parseCloudConfig(t, user)
delete(got, "packages")
delete(got, "runcmd")
want := parseCloudConfig(t, userData)
if !reflect.DeepEqual(got, want) {
t.Errorf("merged user-data changed the user's keys:\n got %#v\nwant %#v", got, want)
}
}
func TestWithGuestAgent_nonCloudConfigFallsBackToVendorData(t *testing.T) {
t.Parallel()
tests := []struct {
name string
userData string
vendorData string
wantVendor string
}{
{"shell script, no vendor-data", "#!/bin/sh\napt-get install -y squid\n", "", guestAgentVendorData},
{"include, no vendor-data", "#include\nhttps://example/x\n", "", guestAgentVendorData},
{"shell script, own vendor-data wins", "#!/bin/sh\n", "#cloud-config\nntp: {}\n", "#cloud-config\nntp: {}\n"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
user, vendor, err := withGuestAgent(tc.userData, tc.vendorData)
if err != nil {
t.Fatalf("withGuestAgent() error = %v", err)
}
if user != tc.userData {
t.Errorf("user-data = %q, want it untouched", user)
}
if vendor != tc.wantVendor {
t.Errorf("vendor-data = %q, want %q", vendor, tc.wantVendor)
}
})
}
}
func TestWithGuestAgent_rejectsBrokenCloudConfig(t *testing.T) {
t.Parallel()
tests := []struct {
name string
userData string
}{
{"not yaml", "#cloud-config\npackages: [squid\n"},
{"not a mapping", "#cloud-config\n- squid\n"},
{"packages is not a list", "#cloud-config\npackages: squid\n"},
{"runcmd is not a list", "#cloud-config\nruncmd:\n a: b\n"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
if _, _, err := withGuestAgent(tc.userData, ""); err == nil {
t.Errorf("withGuestAgent() = nil error, want one for %q", tc.userData)
}
})
}
}