Install the guest agent from a vendor-data script instead of editing the user's cloud-config

Merging the bootstrap into each Proxy's cloud-config was the wrong mechanism twice over. Detection only matched a literal #cloud-config first line, so a jinja template or a MIME archive still fell back to vendor-data, where cloud-init's default mergers drop it — the exact failure the merge existed to prevent. And the YAML round-trip re-typed unquoted scalars: version: 1.10 reached the guest as 1.1, comments were dropped and keys reordered. Neither is worth patching, because vendor-data delivered as a shell script bypasses the merger entirely: cloud-init hands vendor-data to its own handler chain, writes scripts to a separate directory and runs them from cc_scripts_vendor, after the user's packages and before the user's runcmd. The merge could not have been rescued anyway — merge_how in user-data governs only the first source, and merge_how in vendor-data is discarded before the document is stored. The Proxy's cloud-init now reaches the seed byte for byte in any format. installGuestAgent (default true) turns the script off; a configured vendorData travels beside it as a MIME multipart, declared text/plain so cloud-init re-sniffs it and this code never classifies user content. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-10 22:42:55 +00:00
parent 10ca71331e
commit c23def0051
8 changed files with 323 additions and 261 deletions
@@ -250,7 +250,7 @@ func (p *Provider) writeSeed(log logr.Logger, api libvirtAPI, pool libvirt.Stora
 	if err := deleteIfExists(api, pool, seedName); err != nil {
 		return libvirt.StorageVol{}, err
 	}
-	userData, vendorData, err := withGuestAgent(userData, p.cfg.VendorData)
+	vendorData, err := buildVendorData(p.cfg.VendorData, p.cfg.InstallGuestAgent == nil || *p.cfg.InstallGuestAgent)
 	if err != nil {
 		return libvirt.StorageVol{}, fmt.Errorf("%w: %v", provider.ErrPermanent, err)
 	}
@@ -188,8 +188,11 @@ func assertCreated(t *testing.T, api *fakeAPI) {
 		t.Errorf("seed = format %s, %d bytes, capacity %d; want raw with capacity == uploaded size", seed.format, len(seed.data), seed.capacity)
 	}
 	_, files := readSeed(t, seed.data)
-	if ud := files["user-data"]; !strings.Contains(ud, "squid") || !strings.Contains(ud, guestAgentPackage) {
-		t.Errorf("seed user-data = %q, want the Proxy's cloud-config with the guest agent merged in", ud)
+	if files["user-data"] != testCloudInit {
+		t.Errorf("seed user-data = %q, want the Proxy's cloud-init verbatim", files["user-data"])
+	}
+	if !strings.Contains(files["vendor-data"], "qemu-guest-agent") {
+		t.Errorf("seed vendor-data = %q, want the guest-agent script", files["vendor-data"])
 	}
 }

@@ -3,110 +3,104 @@ package kvm
 import (
 	"bytes"
 	"fmt"
-	"reflect"
+	"io"
+	"mime/multipart"
+	"net/textproto"
 	"strings"

 	"github.com/kdomanski/iso9660"
-	"sigs.k8s.io/yaml"
 )

 // seedLabel is the volume label cloud-init's NoCloud datasource looks for.
 const seedLabel = "cidata"

-const (
-	cloudConfigHeader = "#cloud-config"
-	guestAgentPackage = "qemu-guest-agent"
-)
-
-// guestAgentEnable is the runcmd entry that starts the agent once it is
-// installed, so the provider can read the VM's IP before ARP sees traffic.
-var guestAgentEnable = []any{"systemctl", "enable", "--now", guestAgentPackage}
-
-// guestAgentVendorData is the same bootstrap as a standalone cloud-config.
-// It is only used as vendor-data when the Proxy's user-data is not a
-// cloud-config document (a shell script, a MIME archive, an #include), i.e.
-// when there are no top-level keys for it to collide with.
-const guestAgentVendorData = cloudConfigHeader + `
-packages:
-  - qemu-guest-agent
-runcmd:
-  - [systemctl, enable, --now, qemu-guest-agent]
+// guestAgentScript installs and starts the QEMU guest agent, which is how
+// the provider learns a VM's address before the host's ARP table has seen
+// any traffic from it.
+//
+// It ships as vendor-data, and as a *script* rather than a cloud-config,
+// because that is the only shape cloud-init cannot drop. Vendor-data
+// cloud-config is merged under the Proxy's own user-data with mergers that
+// keep the user-data's value for every key both documents define and never
+// append lists, so a Proxy carrying its own packages: silently lost the
+// agent. The strategy cannot be overridden from either document: merge_how
+// in user-data governs only the first source, and merge_how in vendor-data
+// is consumed and discarded before the document is stored. A vendor-data
+// script bypasses all of it — cloud-init gives vendor-data its own handler
+// chain, writes scripts to a separate directory and runs them from
+// cc_scripts_vendor, after the user's packages are installed and before the
+// user's runcmd.
+const guestAgentScript = `#!/bin/sh
+# Added by egress-proxies-operator so the hypervisor can report this VM's
+# address. Set installGuestAgent: false on the provider to leave it out.
+set -e
+if ! command -v qemu-ga >/dev/null 2>&1 && [ ! -x /usr/sbin/qemu-ga ]; then
+	if command -v apt-get >/dev/null 2>&1; then
+		apt-get update
+		DEBIAN_FRONTEND=noninteractive apt-get install -y qemu-guest-agent
+	elif command -v dnf >/dev/null 2>&1; then
+		dnf install -y qemu-guest-agent
+	elif command -v yum >/dev/null 2>&1; then
+		yum install -y qemu-guest-agent
+	elif command -v zypper >/dev/null 2>&1; then
+		zypper --non-interactive install qemu-guest-agent
+	elif command -v apk >/dev/null 2>&1; then
+		apk add qemu-guest-agent
+	else
+		echo "no known package manager; install qemu-guest-agent in the image" >&2
+		exit 1
+	fi
+fi
+if command -v systemctl >/dev/null 2>&1; then
+	systemctl enable --now qemu-guest-agent
+elif command -v rc-update >/dev/null 2>&1; then
+	rc-update add qemu-guest-agent default
+	rc-service qemu-guest-agent start
+fi
 `

-// withGuestAgent decides where the guest-agent bootstrap goes and returns
-// the user-data and vendor-data to put on the seed.
-//
-// It cannot simply ride along as vendor-data: cloud-init merges vendor-data
-// under user-data with its default mergers, which keep the user-data's
-// value for any key both define and never append lists. A Proxy whose
-// cloud-config has its own `packages:` (most do) would silently lose the
-// agent and sit in Provisioning forever. So for a cloud-config user-data
-// the bootstrap is merged into the document itself: `qemu-guest-agent`
-// appended to `packages`, the enable command to `runcmd`, both skipped
-// when already present. Anything the operator cannot parse as a
-// cloud-config mapping is an error — cloud-init would reject it too, and a
-// VM that boots without its config is harder to diagnose than a Proxy that
-// fails fast.
-func withGuestAgent(userData, vendorData string) (string, string, error) {
-	if !isCloudConfig(userData) {
-		if vendorData == "" {
-			vendorData = guestAgentVendorData
-		}
-		return userData, vendorData, nil
-	}
-	var cfg map[string]any
-	if err := yaml.Unmarshal([]byte(userData), &cfg); err != nil {
-		return "", "", fmt.Errorf("parsing cloud-config user-data: %w", err)
-	}
-	if cfg == nil {
-		cfg = map[string]any{}
-	}
-	pkgs, err := appendUnique(cfg, "packages", guestAgentPackage)
-	if err != nil {
-		return "", "", err
-	}
-	cfg["packages"] = pkgs
-	cmds, err := appendUnique(cfg, "runcmd", guestAgentEnable)
-	if err != nil {
-		return "", "", err
-	}
-	cfg["runcmd"] = cmds
-	out, err := yaml.Marshal(cfg)
-	if err != nil {
-		return "", "", fmt.Errorf("rendering cloud-config user-data: %w", err)
-	}
-	return cloudConfigHeader + "\n" + string(out), vendorData, nil
-}
-
-// isCloudConfig reports whether userData is a cloud-config document by
-// cloud-init's own rule: the first line is the #cloud-config header. Empty
-// user-data counts, so a Proxy without cloud-init still gets the agent.
-func isCloudConfig(userData string) bool {
-	if strings.TrimSpace(userData) == "" {
-		return true
-	}
-	first, _, _ := strings.Cut(userData, "\n")
-	return strings.TrimSpace(first) == cloudConfigHeader
-}
-
-// appendUnique returns cfg[key] with item appended unless an equal entry
-// is already there. A missing key is an empty list; anything but a list is
-// an error, since cloud-init expects one there as well.
-func appendUnique(cfg map[string]any, key string, item any) ([]any, error) {
-	var items []any
-	switch v := cfg[key].(type) {
-	case nil:
-	case []any:
-		items = v
+// buildVendorData renders the seed's vendor-data. With nothing configured
+// it is the guest-agent script alone; with both, a MIME multipart carrying
+// the configured document and the script as separate parts, since
+// vendor-data is a single file and the two must not be concatenated.
+func buildVendorData(configured string, installAgent bool) (string, error) {
+	switch {
+	case !installAgent:
+		return configured, nil
+	case strings.TrimSpace(configured) == "":
+		return guestAgentScript, nil
 	default:
-		return nil, fmt.Errorf("cloud-config %s: want a list, got %T", key, v)
+		return multipartVendorData(configured, guestAgentScript)
 	}
-	for _, it := range items {
-		if reflect.DeepEqual(it, item) {
-			return items, nil
+}
+
+// multipartVendorData packs several documents into one vendor-data blob.
+// cloud-init decides a payload is multipart by finding "mime-version:"
+// within its first 4096 bytes, so those headers lead. The configured
+// document goes in as text/plain on purpose: cloud-init re-sniffs that type
+// from the content's own header line, which keeps this code out of the
+// business of classifying what the user wrote.
+func multipartVendorData(configured, script string) (string, error) {
+	var body bytes.Buffer
+	w := multipart.NewWriter(&body)
+	parts := []struct{ contentType, content string }{
+		{"text/plain", configured},
+		{"text/x-shellscript", script},
+	}
+	for _, p := range parts {
+		part, err := w.CreatePart(textproto.MIMEHeader{"Content-Type": {p.contentType}})
+		if err != nil {
+			return "", fmt.Errorf("building vendor-data part: %w", err)
+		}
+		if _, err := io.WriteString(part, p.content); err != nil {
+			return "", fmt.Errorf("writing vendor-data part: %w", err)
 		}
 	}
-	return append(items, item), nil
+	if err := w.Close(); err != nil {
+		return "", fmt.Errorf("closing vendor-data: %w", err)
+	}
+	return fmt.Sprintf("MIME-Version: 1.0\r\nContent-Type: multipart/mixed; boundary=%q\r\n\r\n%s",
+		w.Boundary(), body.String()), nil
 }

 // buildSeedISO renders a NoCloud seed: user-data and vendor-data verbatim,
@@ -4,12 +4,12 @@ import (
 	"bytes"
 	"io"
 	"maps"
-	"reflect"
+	"mime"
+	"mime/multipart"
 	"strings"
 	"testing"

 	"github.com/kdomanski/iso9660"
-	"sigs.k8s.io/yaml"
 )

 func readSeed(t *testing.T, iso []byte) (label string, files map[string]string) {
@@ -44,7 +44,7 @@ func readSeed(t *testing.T, iso []byte) (label string, files map[string]string)
 func TestBuildSeedISO_contents(t *testing.T) {
 	t.Parallel()
 	userData := "#cloud-config\npackages: [squid]\n"
-	iso, err := buildSeedISO("proxy-abc123def456ghij", userData, guestAgentVendorData)
+	iso, err := buildSeedISO("proxy-abc123def456ghij", userData, guestAgentScript)
 	if err != nil {
 		t.Fatalf("buildSeedISO() error = %v", err)
 	}
@@ -58,7 +58,7 @@ func TestBuildSeedISO_contents(t *testing.T) {
 	if files["user-data"] != userData {
 		t.Errorf("user-data = %q, want verbatim cloud-init", files["user-data"])
 	}
-	if files["vendor-data"] != guestAgentVendorData {
+	if files["vendor-data"] != guestAgentScript {
 		t.Errorf("vendor-data = %q", files["vendor-data"])
 	}
 	if got, want := files["meta-data"], "instance-id: proxy-abc123def456ghij\nlocal-hostname: proxy-abc123def456ghij\n"; got != want {
@@ -91,7 +91,7 @@ func TestBuildSeedISO_deterministicContent(t *testing.T) {

 func TestBuildSeedISO_emptyUserData(t *testing.T) {
 	t.Parallel()
-	iso, err := buildSeedISO("proxy-x", "", guestAgentVendorData)
+	iso, err := buildSeedISO("proxy-x", "", guestAgentScript)
 	if err != nil {
 		t.Fatalf("buildSeedISO() error = %v", err)
 	}
@@ -101,149 +101,133 @@ func TestBuildSeedISO_emptyUserData(t *testing.T) {
 	}
 }

-// parseCloudConfig is what cloud-init sees: the document under the header.
-func parseCloudConfig(t *testing.T, doc string) map[string]any {
-	t.Helper()
-	if first, _, _ := strings.Cut(doc, "\n"); first != cloudConfigHeader {
-		t.Fatalf("first line = %q, want %q", first, cloudConfigHeader)
-	}
-	var cfg map[string]any
-	if err := yaml.Unmarshal([]byte(doc), &cfg); err != nil {
-		t.Fatalf("merged user-data is not YAML: %v\n%s", err, doc)
-	}
-	return cfg
-}
-
-func TestWithGuestAgent_mergesIntoCloudConfig(t *testing.T) {
-	t.Parallel()
-	agentCmd := []any{"systemctl", "enable", "--now", "qemu-guest-agent"}
-	tests := []struct {
-		name         string
-		userData     string
-		wantPackages []any
-		wantRuncmd   []any
-	}{
-		{
-			name:         "empty document",
-			userData:     "#cloud-config\n",
-			wantPackages: []any{"qemu-guest-agent"},
-			wantRuncmd:   []any{agentCmd},
-		},
-		{
-			name:         "no user-data at all",
-			userData:     "",
-			wantPackages: []any{"qemu-guest-agent"},
-			wantRuncmd:   []any{agentCmd},
-		},
-		{
-			// The case that motivated this: cloud-init's default merger keeps
-			// the user-data's lists and drops vendor-data's, so the agent must
-			// be appended to the user's own packages and runcmd.
-			name: "user packages and runcmd are kept and extended",
-			userData: "#cloud-config\npackages:\n  - squid\nruncmd:\n  - systemctl restart squid\n" +
-				"write_files:\n  - path: /etc/squid/conf.d/x.conf\n    content: |\n      http_port 3128\n      via off\n",
-			wantPackages: []any{"squid", "qemu-guest-agent"},
-			wantRuncmd:   []any{"systemctl restart squid", agentCmd},
-		},
-		{
-			name:         "agent already present is not duplicated",
-			userData:     "#cloud-config\npackages: [qemu-guest-agent]\nruncmd:\n  - [systemctl, enable, --now, qemu-guest-agent]\n",
-			wantPackages: []any{"qemu-guest-agent"},
-			wantRuncmd:   []any{agentCmd},
-		},
-		{
-			name:         "header with trailing whitespace",
-			userData:     "#cloud-config   \r\npackages: [curl]\n",
-			wantPackages: []any{"curl", "qemu-guest-agent"},
-			wantRuncmd:   []any{agentCmd},
-		},
-	}
-	for _, tc := range tests {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-			user, vendor, err := withGuestAgent(tc.userData, "")
-			if err != nil {
-				t.Fatalf("withGuestAgent() error = %v", err)
-			}
-			if vendor != "" {
-				t.Errorf("vendor-data = %q, want none when the agent is merged into the user-data", vendor)
-			}
-			cfg := parseCloudConfig(t, user)
-			if !reflect.DeepEqual(cfg["packages"], tc.wantPackages) {
-				t.Errorf("packages = %#v, want %#v", cfg["packages"], tc.wantPackages)
-			}
-			if !reflect.DeepEqual(cfg["runcmd"], tc.wantRuncmd) {
-				t.Errorf("runcmd = %#v, want %#v", cfg["runcmd"], tc.wantRuncmd)
-			}
-		})
-	}
-}
-
-func TestWithGuestAgent_keepsOtherKeysAndVendorData(t *testing.T) {
-	t.Parallel()
-	userData := "#cloud-config\nhostname: px\nwrite_files:\n  - path: /etc/x\n    content: |\n      line one\n      line two\n    permissions: '0644'\nbootcmd: [echo hi]\n"
-	user, vendor, err := withGuestAgent(userData, "#cloud-config\nntp: {enabled: true}\n")
-	if err != nil {
-		t.Fatalf("withGuestAgent() error = %v", err)
-	}
-	if vendor != "#cloud-config\nntp: {enabled: true}\n" {
-		t.Errorf("vendor-data = %q, want the configured one verbatim", vendor)
-	}
-	got := parseCloudConfig(t, user)
-	delete(got, "packages")
-	delete(got, "runcmd")
-	want := parseCloudConfig(t, userData)
-	if !reflect.DeepEqual(got, want) {
-		t.Errorf("merged user-data changed the user's keys:\n got %#v\nwant %#v", got, want)
-	}
-}
-
-func TestWithGuestAgent_nonCloudConfigFallsBackToVendorData(t *testing.T) {
-	t.Parallel()
-	tests := []struct {
-		name       string
-		userData   string
-		vendorData string
-		wantVendor string
-	}{
-		{"shell script, no vendor-data", "#!/bin/sh\napt-get install -y squid\n", "", guestAgentVendorData},
-		{"include, no vendor-data", "#include\nhttps://example/x\n", "", guestAgentVendorData},
-		{"shell script, own vendor-data wins", "#!/bin/sh\n", "#cloud-config\nntp: {}\n", "#cloud-config\nntp: {}\n"},
-	}
-	for _, tc := range tests {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-			user, vendor, err := withGuestAgent(tc.userData, tc.vendorData)
-			if err != nil {
-				t.Fatalf("withGuestAgent() error = %v", err)
-			}
-			if user != tc.userData {
-				t.Errorf("user-data = %q, want it untouched", user)
-			}
-			if vendor != tc.wantVendor {
-				t.Errorf("vendor-data = %q, want %q", vendor, tc.wantVendor)
-			}
-		})
-	}
-}
-
-func TestWithGuestAgent_rejectsBrokenCloudConfig(t *testing.T) {
+// The whole point of shipping the agent as vendor-data: whatever the Proxy
+// wrote reaches the guest untouched, in any of cloud-init's formats. The
+// previous implementation parsed and re-rendered cloud-config user-data,
+// which dropped comments, reordered keys and re-typed unquoted scalars
+// (version: 1.10 arrived as 1.1).
+func TestBuildSeedISO_userDataIsNeverRewritten(t *testing.T) {
 	t.Parallel()
 	tests := []struct {
 		name     string
 		userData string
 	}{
-		{"not yaml", "#cloud-config\npackages: [squid\n"},
-		{"not a mapping", "#cloud-config\n- squid\n"},
-		{"packages is not a list", "#cloud-config\npackages: squid\n"},
-		{"runcmd is not a list", "#cloud-config\nruncmd:\n  a: b\n"},
+		{"cloud-config with its own packages", "#cloud-config\n# keep me\npackages:\n  - squid\nversion: 1.10\nchpasswd: {list: 'u:0123456789'}\n"},
+		{"jinja template", "## template: jinja\n#cloud-config\nhostname: {{ v1.local_hostname }}\n"},
+		{"shell script", "#!/bin/sh\napt-get install -y squid\n"},
+		{"include", "#include\nhttps://example.invalid/x\n"},
+		{"mime multipart", "MIME-Version: 1.0\r\nContent-Type: multipart/mixed; boundary=\"x\"\r\n\r\n--x\r\nContent-Type: text/cloud-config\r\n\r\n#cloud-config\npackages: [squid]\n\r\n--x--\r\n"},
+		{"gzip payload", "\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03binary\x00bytes"},
+		{"leading blank line", "\n\n#cloud-config\npackages: [squid]\n"},
 	}
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
-			if _, _, err := withGuestAgent(tc.userData, ""); err == nil {
-				t.Errorf("withGuestAgent() = nil error, want one for %q", tc.userData)
+			vendor, err := buildVendorData("", true)
+			if err != nil {
+				t.Fatalf("buildVendorData() error = %v", err)
+			}
+			iso, err := buildSeedISO("proxy-x", tc.userData, vendor)
+			if err != nil {
+				t.Fatalf("buildSeedISO() error = %v", err)
+			}
+			_, files := readSeed(t, iso)
+			if files["user-data"] != tc.userData {
+				t.Errorf("user-data was rewritten:\n got %q\nwant %q", files["user-data"], tc.userData)
+			}
+			if !strings.Contains(files["vendor-data"], "qemu-guest-agent") {
+				t.Errorf("vendor-data = %q, want the agent script", files["vendor-data"])
 			}
 		})
 	}
 }
+
+func TestBuildVendorData_agentScriptAlone(t *testing.T) {
+	t.Parallel()
+	got, err := buildVendorData("", true)
+	if err != nil {
+		t.Fatalf("buildVendorData() error = %v", err)
+	}
+	if got != guestAgentScript {
+		t.Errorf("vendor-data = %q, want the bare script", got)
+	}
+	if !strings.HasPrefix(got, "#!") {
+		t.Error("vendor-data must start with a shebang or cloud-init treats it as something other than a script")
+	}
+}
+
+func TestBuildVendorData_optOutKeepsConfiguredOnly(t *testing.T) {
+	t.Parallel()
+	configured := "#cloud-config\nntp: {enabled: true}\n"
+	tests := []struct {
+		name       string
+		configured string
+		want       string
+	}{
+		{"nothing configured", "", ""},
+		{"configured document passes through", configured, configured},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			got, err := buildVendorData(tc.configured, false)
+			if err != nil {
+				t.Fatalf("buildVendorData() error = %v", err)
+			}
+			if got != tc.want {
+				t.Errorf("vendor-data = %q, want %q", got, tc.want)
+			}
+			if strings.Contains(got, "qemu-guest-agent") {
+				t.Error("agent script present although installGuestAgent is off")
+			}
+		})
+	}
+}
+
+// Both documents have to survive in one file, and cloud-init only looks for
+// the multipart marker in the first 4096 bytes.
+func TestBuildVendorData_multipartCarriesBoth(t *testing.T) {
+	t.Parallel()
+	configured := "#cloud-config\nntp:\n  enabled: true\n"
+	got, err := buildVendorData(configured, true)
+	if err != nil {
+		t.Fatalf("buildVendorData() error = %v", err)
+	}
+	head := got
+	if len(head) > 4096 {
+		head = head[:4096]
+	}
+	if !strings.Contains(strings.ToLower(head), "mime-version:") {
+		t.Fatalf("no mime-version: in the first 4096 bytes; cloud-init would treat this as a single document:\n%s", head)
+	}
+	_, params, err := mime.ParseMediaType(strings.SplitN(got, "\r\n\r\n", 2)[0][strings.Index(got, "Content-Type:")+len("Content-Type: "):])
+	if err != nil {
+		t.Fatalf("parsing the multipart content type: %v", err)
+	}
+	body := strings.SplitN(got, "\r\n\r\n", 2)[1]
+	r := multipart.NewReader(strings.NewReader(body), params["boundary"])
+	var bodies, types []string
+	for {
+		part, err := r.NextPart()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Fatalf("reading part: %v", err)
+		}
+		content, err := io.ReadAll(part)
+		if err != nil {
+			t.Fatalf("reading part body: %v", err)
+		}
+		bodies = append(bodies, string(content))
+		types = append(types, part.Header.Get("Content-Type"))
+	}
+	if len(bodies) != 2 || bodies[0] != configured || bodies[1] != guestAgentScript {
+		t.Fatalf("parts = %q, want the configured document then the script", bodies)
+	}
+	// text/plain makes cloud-init re-sniff the user's document from its own
+	// header line, so this code never has to classify it.
+	if types[0] != "text/plain" || types[1] != "text/x-shellscript" {
+		t.Errorf("part types = %v, want [text/plain text/x-shellscript]", types)
+	}
+}