Install the guest agent from a vendor-data script instead of editing the user's cloud-config
Merging the bootstrap into each Proxy's cloud-config was the wrong mechanism twice over. Detection only matched a literal #cloud-config first line, so a jinja template or a MIME archive still fell back to vendor-data, where cloud-init's default mergers drop it — the exact failure the merge existed to prevent. And the YAML round-trip re-typed unquoted scalars: version: 1.10 reached the guest as 1.1, comments were dropped and keys reordered. Neither is worth patching, because vendor-data delivered as a shell script bypasses the merger entirely: cloud-init hands vendor-data to its own handler chain, writes scripts to a separate directory and runs them from cc_scripts_vendor, after the user's packages and before the user's runcmd. The merge could not have been rescued anyway — merge_how in user-data governs only the first source, and merge_how in vendor-data is discarded before the document is stored. The Proxy's cloud-init now reaches the seed byte for byte in any format. installGuestAgent (default true) turns the script off; a configured vendorData travels beside it as a MIME multipart, declared text/plain so cloud-init re-sniffs it and this code never classifies user content. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -3,110 +3,104 @@ package kvm
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/textproto"
|
||||
"strings"
|
||||
|
||||
"github.com/kdomanski/iso9660"
|
||||
"sigs.k8s.io/yaml"
|
||||
)
|
||||
|
||||
// seedLabel is the volume label cloud-init's NoCloud datasource looks for.
|
||||
const seedLabel = "cidata"
|
||||
|
||||
const (
|
||||
cloudConfigHeader = "#cloud-config"
|
||||
guestAgentPackage = "qemu-guest-agent"
|
||||
)
|
||||
|
||||
// guestAgentEnable is the runcmd entry that starts the agent once it is
|
||||
// installed, so the provider can read the VM's IP before ARP sees traffic.
|
||||
var guestAgentEnable = []any{"systemctl", "enable", "--now", guestAgentPackage}
|
||||
|
||||
// guestAgentVendorData is the same bootstrap as a standalone cloud-config.
|
||||
// It is only used as vendor-data when the Proxy's user-data is not a
|
||||
// cloud-config document (a shell script, a MIME archive, an #include), i.e.
|
||||
// when there are no top-level keys for it to collide with.
|
||||
const guestAgentVendorData = cloudConfigHeader + `
|
||||
packages:
|
||||
- qemu-guest-agent
|
||||
runcmd:
|
||||
- [systemctl, enable, --now, qemu-guest-agent]
|
||||
// guestAgentScript installs and starts the QEMU guest agent, which is how
|
||||
// the provider learns a VM's address before the host's ARP table has seen
|
||||
// any traffic from it.
|
||||
//
|
||||
// It ships as vendor-data, and as a *script* rather than a cloud-config,
|
||||
// because that is the only shape cloud-init cannot drop. Vendor-data
|
||||
// cloud-config is merged under the Proxy's own user-data with mergers that
|
||||
// keep the user-data's value for every key both documents define and never
|
||||
// append lists, so a Proxy carrying its own packages: silently lost the
|
||||
// agent. The strategy cannot be overridden from either document: merge_how
|
||||
// in user-data governs only the first source, and merge_how in vendor-data
|
||||
// is consumed and discarded before the document is stored. A vendor-data
|
||||
// script bypasses all of it — cloud-init gives vendor-data its own handler
|
||||
// chain, writes scripts to a separate directory and runs them from
|
||||
// cc_scripts_vendor, after the user's packages are installed and before the
|
||||
// user's runcmd.
|
||||
const guestAgentScript = `#!/bin/sh
|
||||
# Added by egress-proxies-operator so the hypervisor can report this VM's
|
||||
# address. Set installGuestAgent: false on the provider to leave it out.
|
||||
set -e
|
||||
if ! command -v qemu-ga >/dev/null 2>&1 && [ ! -x /usr/sbin/qemu-ga ]; then
|
||||
if command -v apt-get >/dev/null 2>&1; then
|
||||
apt-get update
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y qemu-guest-agent
|
||||
elif command -v dnf >/dev/null 2>&1; then
|
||||
dnf install -y qemu-guest-agent
|
||||
elif command -v yum >/dev/null 2>&1; then
|
||||
yum install -y qemu-guest-agent
|
||||
elif command -v zypper >/dev/null 2>&1; then
|
||||
zypper --non-interactive install qemu-guest-agent
|
||||
elif command -v apk >/dev/null 2>&1; then
|
||||
apk add qemu-guest-agent
|
||||
else
|
||||
echo "no known package manager; install qemu-guest-agent in the image" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
if command -v systemctl >/dev/null 2>&1; then
|
||||
systemctl enable --now qemu-guest-agent
|
||||
elif command -v rc-update >/dev/null 2>&1; then
|
||||
rc-update add qemu-guest-agent default
|
||||
rc-service qemu-guest-agent start
|
||||
fi
|
||||
`
|
||||
|
||||
// withGuestAgent decides where the guest-agent bootstrap goes and returns
|
||||
// the user-data and vendor-data to put on the seed.
|
||||
//
|
||||
// It cannot simply ride along as vendor-data: cloud-init merges vendor-data
|
||||
// under user-data with its default mergers, which keep the user-data's
|
||||
// value for any key both define and never append lists. A Proxy whose
|
||||
// cloud-config has its own `packages:` (most do) would silently lose the
|
||||
// agent and sit in Provisioning forever. So for a cloud-config user-data
|
||||
// the bootstrap is merged into the document itself: `qemu-guest-agent`
|
||||
// appended to `packages`, the enable command to `runcmd`, both skipped
|
||||
// when already present. Anything the operator cannot parse as a
|
||||
// cloud-config mapping is an error — cloud-init would reject it too, and a
|
||||
// VM that boots without its config is harder to diagnose than a Proxy that
|
||||
// fails fast.
|
||||
func withGuestAgent(userData, vendorData string) (string, string, error) {
|
||||
if !isCloudConfig(userData) {
|
||||
if vendorData == "" {
|
||||
vendorData = guestAgentVendorData
|
||||
}
|
||||
return userData, vendorData, nil
|
||||
}
|
||||
var cfg map[string]any
|
||||
if err := yaml.Unmarshal([]byte(userData), &cfg); err != nil {
|
||||
return "", "", fmt.Errorf("parsing cloud-config user-data: %w", err)
|
||||
}
|
||||
if cfg == nil {
|
||||
cfg = map[string]any{}
|
||||
}
|
||||
pkgs, err := appendUnique(cfg, "packages", guestAgentPackage)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
cfg["packages"] = pkgs
|
||||
cmds, err := appendUnique(cfg, "runcmd", guestAgentEnable)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
cfg["runcmd"] = cmds
|
||||
out, err := yaml.Marshal(cfg)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("rendering cloud-config user-data: %w", err)
|
||||
}
|
||||
return cloudConfigHeader + "\n" + string(out), vendorData, nil
|
||||
}
|
||||
|
||||
// isCloudConfig reports whether userData is a cloud-config document by
|
||||
// cloud-init's own rule: the first line is the #cloud-config header. Empty
|
||||
// user-data counts, so a Proxy without cloud-init still gets the agent.
|
||||
func isCloudConfig(userData string) bool {
|
||||
if strings.TrimSpace(userData) == "" {
|
||||
return true
|
||||
}
|
||||
first, _, _ := strings.Cut(userData, "\n")
|
||||
return strings.TrimSpace(first) == cloudConfigHeader
|
||||
}
|
||||
|
||||
// appendUnique returns cfg[key] with item appended unless an equal entry
|
||||
// is already there. A missing key is an empty list; anything but a list is
|
||||
// an error, since cloud-init expects one there as well.
|
||||
func appendUnique(cfg map[string]any, key string, item any) ([]any, error) {
|
||||
var items []any
|
||||
switch v := cfg[key].(type) {
|
||||
case nil:
|
||||
case []any:
|
||||
items = v
|
||||
// buildVendorData renders the seed's vendor-data. With nothing configured
|
||||
// it is the guest-agent script alone; with both, a MIME multipart carrying
|
||||
// the configured document and the script as separate parts, since
|
||||
// vendor-data is a single file and the two must not be concatenated.
|
||||
func buildVendorData(configured string, installAgent bool) (string, error) {
|
||||
switch {
|
||||
case !installAgent:
|
||||
return configured, nil
|
||||
case strings.TrimSpace(configured) == "":
|
||||
return guestAgentScript, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("cloud-config %s: want a list, got %T", key, v)
|
||||
return multipartVendorData(configured, guestAgentScript)
|
||||
}
|
||||
for _, it := range items {
|
||||
if reflect.DeepEqual(it, item) {
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// multipartVendorData packs several documents into one vendor-data blob.
|
||||
// cloud-init decides a payload is multipart by finding "mime-version:"
|
||||
// within its first 4096 bytes, so those headers lead. The configured
|
||||
// document goes in as text/plain on purpose: cloud-init re-sniffs that type
|
||||
// from the content's own header line, which keeps this code out of the
|
||||
// business of classifying what the user wrote.
|
||||
func multipartVendorData(configured, script string) (string, error) {
|
||||
var body bytes.Buffer
|
||||
w := multipart.NewWriter(&body)
|
||||
parts := []struct{ contentType, content string }{
|
||||
{"text/plain", configured},
|
||||
{"text/x-shellscript", script},
|
||||
}
|
||||
for _, p := range parts {
|
||||
part, err := w.CreatePart(textproto.MIMEHeader{"Content-Type": {p.contentType}})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("building vendor-data part: %w", err)
|
||||
}
|
||||
if _, err := io.WriteString(part, p.content); err != nil {
|
||||
return "", fmt.Errorf("writing vendor-data part: %w", err)
|
||||
}
|
||||
}
|
||||
return append(items, item), nil
|
||||
if err := w.Close(); err != nil {
|
||||
return "", fmt.Errorf("closing vendor-data: %w", err)
|
||||
}
|
||||
return fmt.Sprintf("MIME-Version: 1.0\r\nContent-Type: multipart/mixed; boundary=%q\r\n\r\n%s",
|
||||
w.Boundary(), body.String()), nil
|
||||
}
|
||||
|
||||
// buildSeedISO renders a NoCloud seed: user-data and vendor-data verbatim,
|
||||
|
||||
Reference in New Issue
Block a user