provider: extract nocloud seed builder and keyedlock from kvm2

kvm2's NoCloud seed builder moves to internal/provider/nocloud (exported,
with an ErrEmptyName sentinel instead of permanentf) and its keyed locks
to internal/provider/keyedlock, tests moved under their existing names.
kvm2 behaviour is unchanged. proxmox gains the per-name lock and the pure
seed helpers (seedISOName, seedVolID, buildSeed); unlike kvm2 (review
finding S5.1), a seed staging failure there is Transient, not Permanent.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-09-23 14:01:27 +00:00
co-authored by Claude Opus 5.5
parent 6e996ae2c7
commit 0a6c8624f3
18 changed files with 504 additions and 185 deletions
+3 -2
View File
@@ -11,6 +11,7 @@ import (
"github.com/go-logr/logr"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/nocloud"
)
// The wait after DomainCreate (spec entry 18): Create returns only once the
@@ -191,12 +192,12 @@ func (p *Provider) writeSeed(ctx context.Context, log logr.Logger, be backend, p
return libvirt.StorageVol{}, err
}
installAgent := p.cfg.InstallGuestAgent == nil || *p.cfg.InstallGuestAgent
vendorData, err := buildVendorData(p.cfg.VendorData, installAgent)
vendorData, err := nocloud.BuildVendorData(p.cfg.VendorData, installAgent)
if err != nil {
return libvirt.StorageVol{}, permanentf("building vendor-data for %s: %v", req.Name, err)
}
var iso bytes.Buffer
if err := buildSeedISO(req.Name, req.CloudInit, vendorData, &iso); err != nil {
if err := nocloud.BuildSeedISO(req.Name, req.CloudInit, vendorData, &iso); err != nil {
return libvirt.StorageVol{}, permanentf("building seed ISO for %s: %v", req.Name, err)
}
xmlDoc, err := buildVolumeXML(volumeSpec{Name: name, CapacityBytes: int64(iso.Len()), Format: provider.KVMFormatRaw})
+3 -2
View File
@@ -12,6 +12,7 @@ import (
"github.com/digitalocean/go-libvirt"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/nocloud"
)
func TestCreate_repeatReturnsSameIDAndCreatesNothing(t *testing.T) {
@@ -95,7 +96,7 @@ func TestCreate_resumesAfterEachPartialStep(t *testing.T) {
},
check: func(t *testing.T, fx *fixture) {
got := fx.fake.vol(seed)
if got == nil || len(got.content) <= 3 || !bytes.Contains(got.content, []byte(seedLabel)) {
if got == nil || len(got.content) <= 3 || !bytes.Contains(got.content, []byte(nocloud.SeedLabel)) {
t.Errorf("seed volume not rebuilt")
}
},
@@ -461,7 +462,7 @@ func TestCreate_seedAlwaysRebuilt(t *testing.T) {
if got := fx.fake.callCount("StorageVolUpload"); got != uploads+1 {
t.Errorf("StorageVolUpload calls = %d, want %d", got, uploads+1)
}
if len(second.content) != len(first.content) || !bytes.Contains(second.content, []byte(seedLabel)) {
if len(second.content) != len(first.content) || !bytes.Contains(second.content, []byte(nocloud.SeedLabel)) {
t.Errorf("rebuilt seed is %d bytes, first was %d; want a complete seed of the same size", len(second.content), len(first.content))
}
}
+1 -1
View File
@@ -73,7 +73,7 @@ func resolveImage(entry string, cfg provider.KVMConfig, image string) (imageSour
// returns promptly (contract §1.5) instead of waiting out a 20-minute
// transfer.
func (p *Provider) lockImage(ctx context.Context, hostName, volume string) (func(), error) {
release, err := p.imageLocks.acquire(ctx, hostName+"/"+volume)
release, err := p.imageLocks.Acquire(ctx, hostName+"/"+volume)
if err != nil {
return nil, fmt.Errorf("%w: waiting for image %s on host %s: %w", provider.ErrTransient, volume, hostName, err)
}
+6 -57
View File
@@ -9,13 +9,13 @@ import (
"fmt"
"net/http"
"os"
"sync"
"time"
"github.com/go-logr/logr"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider/keyedlock"
"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/tracing"
)
@@ -41,12 +41,12 @@ type Provider struct {
runningPoll time.Duration
// imageLocks serialises image fetches per (host, volume), spec entry 19.
imageLocks *keyedLocks
imageLocks *keyedlock.Locks
// nameLocks serialises Create and Delete per instance name, spec entry
// 19, so two Deletes never race on destroy and undefine and a Delete
// cannot interleave with a Create's steps.
nameLocks *keyedLocks
nameLocks *keyedlock.Locks
}
// providerOption adjusts what newProvider builds; tests inject an HTTP
@@ -111,8 +111,8 @@ func newProvider(name string, cfg provider.KVMConfig, d dialer, opts ...provider
http: http.DefaultClient,
runningWait: defaultRunningWait,
runningPoll: defaultRunningPoll,
imageLocks: newKeyedLocks(),
nameLocks: newKeyedLocks(),
imageLocks: keyedlock.New(),
nameLocks: keyedlock.New(),
}
for _, opt := range opts {
opt(p)
@@ -187,60 +187,9 @@ func (p *Provider) wrapErr(op, id string, err error) error {
// so a caller whose ctx ends while another caller's Create is downloading
// an image returns promptly.
func (p *Provider) lockName(ctx context.Context, name string) (func(), error) {
release, err := p.nameLocks.acquire(ctx, name)
release, err := p.nameLocks.Acquire(ctx, name)
if err != nil {
return nil, fmt.Errorf("%w: waiting for the lock on %s: %w", provider.ErrTransient, name, err)
}
return release, nil
}
// keyedLocks is a set of one-slot semaphores by key (spec entry 19): a
// semaphore rather than a mutex so a waiter can give up with its ctx. Each
// entry counts the callers holding or waiting for it and is dropped from
// the map at zero, so the map does not grow with every name ever seen while
// two waiters on one key still always share one semaphore.
type keyedLocks struct {
mu sync.Mutex
locks map[string]*keyedLock
}
type keyedLock struct {
sem chan struct{}
refs int
}
func newKeyedLocks() *keyedLocks {
return &keyedLocks{locks: make(map[string]*keyedLock)}
}
// acquire takes the lock for key, or gives up with ctx's error. The
// returned func releases it and must be called exactly once.
func (k *keyedLocks) acquire(ctx context.Context, key string) (func(), error) {
k.mu.Lock()
l, ok := k.locks[key]
if !ok {
l = &keyedLock{sem: make(chan struct{}, 1)}
k.locks[key] = l
}
l.refs++
k.mu.Unlock()
select {
case l.sem <- struct{}{}:
return func() {
<-l.sem
k.unref(key, l)
}, nil
case <-ctx.Done():
k.unref(key, l)
return nil, ctx.Err()
}
}
func (k *keyedLocks) unref(key string, l *keyedLock) {
k.mu.Lock()
defer k.mu.Unlock()
l.refs--
if l.refs == 0 {
delete(k.locks, key)
}
}
-81
View File
@@ -5,7 +5,6 @@ import (
"errors"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
@@ -267,86 +266,6 @@ func TestWithDefaults_doesNotAliasInput(t *testing.T) {
}
}
func lockCount(k *keyedLocks) int {
k.mu.Lock()
defer k.mu.Unlock()
return len(k.locks)
}
func TestKeyedLocks_entryRemovedWhenUncontended(t *testing.T) {
t.Parallel()
k := newKeyedLocks()
release, err := k.acquire(context.Background(), "a")
if err != nil {
t.Fatalf("acquire error = %v", err)
}
if lockCount(k) != 1 {
t.Errorf("entries while held = %d, want 1", lockCount(k))
}
release()
if lockCount(k) != 0 {
t.Errorf("entries after release = %d, want 0", lockCount(k))
}
if _, err := k.acquire(context.Background(), "a"); err != nil {
t.Fatalf("acquire after release error = %v", err)
}
}
func TestKeyedLocks_waitersShareOneLock(t *testing.T) {
t.Parallel()
k := newKeyedLocks()
const n = 20
var inside, overlaps atomic.Int32
var wg sync.WaitGroup
for range n {
wg.Go(func() {
release, err := k.acquire(context.Background(), "shared")
if err != nil {
t.Errorf("acquire error = %v", err)
return
}
if inside.Add(1) > 1 {
overlaps.Add(1)
}
time.Sleep(time.Millisecond)
inside.Add(-1)
release()
})
}
wg.Wait()
if overlaps.Load() != 0 {
t.Errorf("%d holders overlapped on one key", overlaps.Load())
}
if lockCount(k) != 0 {
t.Errorf("entries after every holder released = %d, want 0", lockCount(k))
}
}
func TestKeyedLocks_waiterGivesUpWithContext(t *testing.T) {
t.Parallel()
k := newKeyedLocks()
release, err := k.acquire(context.Background(), "a")
if err != nil {
t.Fatalf("acquire error = %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
defer cancel()
start := time.Now()
if _, err := k.acquire(ctx, "a"); !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("waiter error = %v, want DeadlineExceeded", err)
}
if time.Since(start) > time.Second {
t.Errorf("waiter took %v, want to give up with its ctx", time.Since(start))
}
if lockCount(k) != 1 {
t.Errorf("entries with the holder still in = %d, want 1: the waiter's exit must not drop the holder's entry", lockCount(k))
}
release()
if lockCount(k) != 0 {
t.Errorf("entries after the holder released = %d, want 0", lockCount(k))
}
}
func TestWrapErr_carriesProviderNameOpAndID(t *testing.T) {
t.Parallel()
p := newProvider("kvm2-home", twoHosts(), &stubDialer{})
-124
View File
@@ -1,124 +0,0 @@
package kvm2
import (
"bytes"
"errors"
"fmt"
"io"
"mime/multipart"
"net/textproto"
"strings"
"github.com/kdomanski/iso9660"
)
// seedLabel is the volume label cloud-init's NoCloud datasource looks for.
const seedLabel = "cidata"
// guestAgentScript is the vendor-data payload that gets qemu-guest-agent
// running so the provider can read the VM's address through libvirt. It is
// a shell script and not a cloud-config on purpose: cloud-init merges a
// vendor-data cloud-config under the user-data with mergers that keep the
// user-data's value for every top-level key and never append lists, so a
// Proxy with its own packages: would silently lose the agent. Scripts
// bypass the merge.
const guestAgentScript = `#!/bin/sh
# Added by egress-proxies-operator: installs and starts qemu-guest-agent so
# the operator can learn this VM's IP address through libvirt.
# To leave this out, set installGuestAgent: false in the provider's config.
set -e
if command -v qemu-ga >/dev/null 2>&1 || [ -x /usr/sbin/qemu-ga ]; then
echo "egress-proxies-operator: qemu-guest-agent already installed"
elif command -v apt-get >/dev/null 2>&1; then
export DEBIAN_FRONTEND=noninteractive
apt-get update
apt-get install -y qemu-guest-agent
elif command -v dnf >/dev/null 2>&1; then
dnf install -y qemu-guest-agent
elif command -v yum >/dev/null 2>&1; then
yum install -y qemu-guest-agent
elif command -v zypper >/dev/null 2>&1; then
zypper --non-interactive install qemu-guest-agent
elif command -v apk >/dev/null 2>&1; then
apk add qemu-guest-agent
else
echo "egress-proxies-operator: no supported package manager found; install qemu-guest-agent by hand" >&2
exit 1
fi
if command -v systemctl >/dev/null 2>&1; then
systemctl enable --now qemu-guest-agent
else
rc-update add qemu-guest-agent default
rc-service qemu-guest-agent start
fi
`
// buildSeedISO writes a NoCloud seed to w: user-data byte for byte,
// meta-data naming the instance, and vendor-data (an empty file when there
// is none, so cloud-init does not look elsewhere). iso9660.NewWriter stages
// the files in os.MkdirTemp("", ""), that is under os.TempDir(): $TMPDIR
// when set, else /tmp, the manager's 64Mi emptyDir. The staging dir is
// removed before returning. Nothing about the payload is logged here.
func buildSeedISO(name, userData, vendorData string, w io.Writer) (err error) {
if name == "" {
return permanentf("seed ISO: instance name is empty")
}
iw, err := iso9660.NewWriter()
if err != nil {
return fmt.Errorf("creating seed ISO staging dir: %w", err)
}
defer func() {
if cerr := iw.Cleanup(); cerr != nil {
err = errors.Join(err, fmt.Errorf("removing seed ISO staging dir: %w", cerr))
}
}()
metaData := "instance-id: " + name + "\nlocal-hostname: " + name + "\n"
files := []struct{ name, content string }{
{"user-data", userData},
{"meta-data", metaData},
{"vendor-data", vendorData},
}
for _, f := range files {
if err := iw.AddFile(strings.NewReader(f.content), f.name); err != nil {
return fmt.Errorf("adding %s to seed ISO: %w", f.name, err)
}
}
if err := iw.WriteTo(w, seedLabel); err != nil {
return fmt.Errorf("writing seed ISO: %w", err)
}
return nil
}
// buildVendorData composes the seed's vendor-data from the configured
// document and the guest agent script. With both, they travel as separate
// parts of a MIME multipart, so cloud-init handles each by its own type; the
// MIME-Version header comes first because cloud-init detects multipart by
// finding "mime-version:" within the first 4096 bytes.
func buildVendorData(configured string, installAgent bool) (string, error) {
switch {
case !installAgent:
return configured, nil
case strings.TrimSpace(configured) == "":
return guestAgentScript, nil
}
var buf bytes.Buffer
mw := multipart.NewWriter(&buf)
fmt.Fprintf(&buf, "MIME-Version: 1.0\r\nContent-Type: multipart/mixed; boundary=%q\r\n\r\n", mw.Boundary())
parts := []struct{ contentType, body string }{
{"text/plain", configured},
{"text/x-shellscript", guestAgentScript},
}
for _, p := range parts {
pw, err := mw.CreatePart(textproto.MIMEHeader{"Content-Type": {p.contentType}})
if err != nil {
return "", fmt.Errorf("creating vendor-data part %s: %w", p.contentType, err)
}
if _, err := io.WriteString(pw, p.body); err != nil {
return "", fmt.Errorf("writing vendor-data part %s: %w", p.contentType, err)
}
}
if err := mw.Close(); err != nil {
return "", fmt.Errorf("closing vendor-data multipart: %w", err)
}
return buf.String(), nil
}
-264
View File
@@ -1,264 +0,0 @@
package kvm2
import (
"bufio"
"bytes"
"io"
"mime"
"mime/multipart"
"net/textproto"
"os"
"strings"
"testing"
"github.com/kdomanski/iso9660"
)
// readSeed builds a seed and returns its files by name plus the label.
func readSeed(t *testing.T, name, userData, vendorData string) (map[string]string, string) {
t.Helper()
var buf bytes.Buffer
if err := buildSeedISO(name, userData, vendorData, &buf); err != nil {
t.Fatalf("buildSeedISO: %v", err)
}
img, err := iso9660.OpenImage(bytes.NewReader(buf.Bytes()))
if err != nil {
t.Fatalf("OpenImage: %v", err)
}
root, err := img.RootDir()
if err != nil {
t.Fatalf("RootDir: %v", err)
}
children, err := root.GetChildren()
if err != nil {
t.Fatalf("GetChildren: %v", err)
}
files := make(map[string]string, len(children))
for _, c := range children {
if c.IsDir() {
t.Errorf("unexpected directory %q in seed", c.Name())
continue
}
data, err := io.ReadAll(c.Reader())
if err != nil {
t.Fatalf("reading %s: %v", c.Name(), err)
}
files[c.Name()] = string(data)
}
label, err := img.Label()
if err != nil {
t.Fatalf("Label: %v", err)
}
return files, label
}
func TestSeedISO_filesAndLabel(t *testing.T) {
t.Parallel()
userData := "#cloud-config\npackages: [squid]\n"
vendorData := "#!/bin/sh\necho hi\n"
files, label := readSeed(t, "proxy-abc", userData, vendorData)
if label != seedLabel {
t.Errorf("label = %q, want %q", label, seedLabel)
}
want := map[string]string{
"user-data": userData,
"meta-data": "instance-id: proxy-abc\nlocal-hostname: proxy-abc\n",
"vendor-data": vendorData,
}
if len(files) != len(want) {
t.Errorf("seed has %d files %v, want exactly %d", len(files), keys(files), len(want))
}
for name, content := range want {
if got, ok := files[name]; !ok {
t.Errorf("seed lacks %s", name)
} else if got != content {
t.Errorf("%s = %q, want %q", name, got, content)
}
}
}
func keys(m map[string]string) []string {
out := make([]string, 0, len(m))
for k := range m {
out = append(out, k)
}
return out
}
func TestSeedISO_userDataByteForByte(t *testing.T) {
t.Parallel()
tests := []struct {
name string
userData string
}{
{name: "mime multipart with CRLF", userData: "MIME-Version: 1.0\r\nContent-Type: multipart/mixed; boundary=\"b\"\r\n\r\n--b\r\nContent-Type: text/cloud-config\r\n\r\n#cloud-config\r\npackages: [squid]\r\n--b--\r\n"},
{name: "jinja header", userData: "## template: jinja\n#cloud-config\nhostname: {{ v1.local_hostname }}\n"},
{name: "no trailing newline", userData: "#cloud-config\nruncmd:\n - echo done"},
{name: "bytes above 0x7f", userData: "#cloud-config\nwrite_files:\n - content: \"caf\xc3\xa9 \xff\xfe\x80\"\n"},
{name: "empty", userData: ""},
{name: "trailing whitespace preserved", userData: "#!/bin/sh\necho x \n\n\n"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
files, _ := readSeed(t, "proxy-abc", tc.userData, "")
if got := files["user-data"]; got != tc.userData {
t.Errorf("user-data changed:\n got %q\nwant %q", got, tc.userData)
}
})
}
}
// Not parallel: it changes TMPDIR for the process.
func TestSeedISO_tempDirRemoved(t *testing.T) {
tmp := t.TempDir()
t.Setenv("TMPDIR", tmp)
if got := os.TempDir(); got != tmp {
t.Fatalf("os.TempDir() = %q, want %q", got, tmp)
}
var buf bytes.Buffer
if err := buildSeedISO("proxy-abc", "#cloud-config\n", guestAgentScript, &buf); err != nil {
t.Fatalf("buildSeedISO: %v", err)
}
if buf.Len() == 0 {
t.Fatal("seed ISO is empty")
}
entries, err := os.ReadDir(tmp)
if err != nil {
t.Fatal(err)
}
if len(entries) != 0 {
names := make([]string, 0, len(entries))
for _, e := range entries {
names = append(names, e.Name())
}
t.Errorf("staging left behind in TMPDIR: %v", names)
}
}
func TestSeedISO_emptyVendorDataStillHasFile(t *testing.T) {
t.Parallel()
files, _ := readSeed(t, "proxy-abc", "#cloud-config\n", "")
got, ok := files["vendor-data"]
if !ok {
t.Fatalf("seed lacks vendor-data; files: %v", keys(files))
}
if got != "" {
t.Errorf("vendor-data = %q, want empty", got)
}
}
func TestSeedISO_rejectsEmptyName(t *testing.T) {
t.Parallel()
var buf bytes.Buffer
if err := buildSeedISO("", "#cloud-config\n", "", &buf); err == nil {
t.Fatal("buildSeedISO with empty name succeeded")
}
if buf.Len() != 0 {
t.Errorf("wrote %d bytes alongside the error", buf.Len())
}
}
func TestVendorData_isShellScriptNotCloudConfig(t *testing.T) {
t.Parallel()
got, err := buildVendorData("", true)
if err != nil {
t.Fatal(err)
}
if !strings.HasPrefix(got, "#!/bin/sh\n") {
t.Errorf("vendor-data starts with %q, want #!/bin/sh", firstLine(got))
}
if strings.HasPrefix(got, "#cloud-config") {
t.Error("vendor-data is a cloud-config; cloud-init would merge it away")
}
for _, want := range []string{"qemu-guest-agent", "set -e", "apt-get", "dnf", "yum", "zypper", "apk", "systemctl enable --now qemu-guest-agent", "rc-update", "installGuestAgent: false", "DEBIAN_FRONTEND=noninteractive"} {
if !strings.Contains(got, want) {
t.Errorf("script lacks %q", want)
}
}
if strings.Contains(got, "MIME-Version") {
t.Error("script alone should not be wrapped in a multipart")
}
}
func firstLine(s string) string {
line, _, _ := strings.Cut(s, "\n")
return line
}
func bufioReader(s string) *bufio.Reader {
return bufio.NewReader(strings.NewReader(s))
}
func TestVendorData_multipartWhenExtraProvided(t *testing.T) {
t.Parallel()
configured := "#cloud-config\nntp:\n enabled: true\n"
got, err := buildVendorData(configured, true)
if err != nil {
t.Fatal(err)
}
if !strings.HasPrefix(got, "MIME-Version: 1.0\r\n") {
t.Errorf("vendor-data starts with %q, want MIME-Version: 1.0", firstLine(got))
}
r := textproto.NewReader(bufioReader(got))
hdr, err := r.ReadMIMEHeader()
if err != nil {
t.Fatalf("reading top-level headers: %v", err)
}
mediaType, params, err := mime.ParseMediaType(hdr.Get("Content-Type"))
if err != nil {
t.Fatalf("parsing Content-Type %q: %v", hdr.Get("Content-Type"), err)
}
if mediaType != "multipart/mixed" {
t.Errorf("media type = %q, want multipart/mixed", mediaType)
}
mr := multipart.NewReader(r.R, params["boundary"])
type part struct{ contentType, body string }
var parts []part
for {
p, err := mr.NextPart()
if err == io.EOF {
break
}
if err != nil {
t.Fatalf("NextPart: %v", err)
}
body, err := io.ReadAll(p)
if err != nil {
t.Fatalf("reading part: %v", err)
}
parts = append(parts, part{p.Header.Get("Content-Type"), string(body)})
}
want := []part{{"text/plain", configured}, {"text/x-shellscript", guestAgentScript}}
if len(parts) != len(want) {
t.Fatalf("got %d parts, want %d", len(parts), len(want))
}
for i := range want {
if parts[i] != want[i] {
t.Errorf("part %d = {%q, %q}, want {%q, %q}", i, parts[i].contentType, firstLine(parts[i].body), want[i].contentType, firstLine(want[i].body))
}
}
}
func TestVendorData_offWithoutConfiguredIsEmpty(t *testing.T) {
t.Parallel()
got, err := buildVendorData("", false)
if err != nil {
t.Fatal(err)
}
if got != "" {
t.Errorf("vendor-data = %q, want empty", got)
}
}
func TestVendorData_offWithConfiguredPassesThrough(t *testing.T) {
t.Parallel()
configured := "#cloud-config\nntp:\n enabled: true"
got, err := buildVendorData(configured, false)
if err != nil {
t.Fatal(err)
}
if got != configured {
t.Errorf("vendor-data = %q, want the configured document unchanged", got)
}
}