kacerr/firecracker-orchestrator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: multi-distro support and tagged golden snapshots

Browse Source 
Add Alpine, Debian, and Ubuntu rootfs support to `init [distro]`.
Golden snapshots are now namespaced under `golden/<tag>/` so multiple
baselines can coexist. `spawn [tag] [N]` selects which snapshot to
clone from. Systemd-based distros (Debian, Ubuntu) get a fc-net-init
systemd unit; Alpine keeps its inittab-based init.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Honza Novak
2026-04-14 20:48:43 +00:00
parent bfc1f47287
commit fb1db7c9ea
10 changed files with 576 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

269
orchestrator/orchestrator.go
View File

@@ -43,13 +43,13 @@ func New(cfg Config) *Orchestrator {
}
}
func (o *Orchestrator) goldenDir() string { return filepath.Join(o.cfg.BaseDir, "golden") }
func (o *Orchestrator) goldenDir(tag string) string { return filepath.Join(o.cfg.BaseDir, "golden", tag) }
func (o *Orchestrator) clonesDir() string { return filepath.Join(o.cfg.BaseDir, "clones") }
func (o *Orchestrator) pidsDir() string { return filepath.Join(o.cfg.BaseDir, "pids") }
// ——— Init ————————————————————————————————————————————————————————————————
func (o *Orchestrator) Init() error {
func (o *Orchestrator) Init(distro string) error {
if err := os.MkdirAll(o.cfg.BaseDir, 0o755); err != nil {
return err
}
@@ -65,54 +65,94 @@ func (o *Orchestrator) Init() error {
}
// Build rootfs if missing
if _, err := os.Stat(o.cfg.Rootfs); os.IsNotExist(err) {
o.log.Info("building minimal Alpine rootfs ...")
if err := o.buildRootfs(); err != nil {
rootfsPath := o.cfg.RootfsPath(distro)
if _, err := os.Stat(rootfsPath); os.IsNotExist(err) {
o.log.Infof("building minimal %s rootfs ...", distro)
if err := o.buildRootfs(distro, rootfsPath); err != nil {
return fmt.Errorf("build rootfs: %w", err)
}
o.log.Infof("rootfs saved to %s", o.cfg.Rootfs)
o.log.Infof("rootfs saved to %s", rootfsPath)
}
o.log.Info("init complete")
return nil
}
func (o *Orchestrator) buildRootfs() error {
func (o *Orchestrator) buildRootfs(distro, rootfsPath string) error {
sizeMB := 512
if distro == "debian" {
sizeMB = 2048
}
mnt := filepath.Join(o.cfg.BaseDir, "mnt")
// create empty ext4 image
o.log.Infof("running: dd if=/dev/zero of=%s bs=1M count=%d status=none", o.cfg.Rootfs, sizeMB)
if err := run("dd", "if=/dev/zero", "of="+o.cfg.Rootfs,
o.log.Infof("running: dd if=/dev/zero of=%s bs=1M count=%d status=none", rootfsPath, sizeMB)
if err := run("dd", "if=/dev/zero", "of="+rootfsPath,
"bs=1M", fmt.Sprintf("count=%d", sizeMB), "status=none"); err != nil {
return err
}
o.log.Infof("running: mkfs.ext4 -qF %s", o.cfg.Rootfs)
if err := run("mkfs.ext4", "-qF", o.cfg.Rootfs); err != nil {
o.log.Infof("running: mkfs.ext4 -qF %s", rootfsPath)
if err := run("mkfs.ext4", "-qF", rootfsPath); err != nil {
return err
}
os.MkdirAll(mnt, 0o755)
o.log.Infof("running: mount -o loop %s %s", o.cfg.Rootfs, mnt)
if err := run("mount", "-o", "loop", o.cfg.Rootfs, mnt); err != nil {
o.log.Infof("running: mount -o loop %s %s", rootfsPath, mnt)
if err := run("mount", "-o", "loop", rootfsPath, mnt); err != nil {
return err
}
defer run("umount", mnt)
defer func() {
o.log.Infof("running: umount %s", mnt)
run("umount", mnt)
}()
// download and extract Alpine minirootfs
alpineVer := "3.20"
arch := "x86_64"
tarball := fmt.Sprintf("alpine-minirootfs-%s.0-%s.tar.gz", alpineVer, arch)
url := fmt.Sprintf("https://dl-cdn.alpinelinux.org/alpine/v%s/releases/%s/%s",
alpineVer, arch, tarball)
tarPath := filepath.Join(o.cfg.BaseDir, tarball)
if err := downloadFile(url, tarPath); err != nil {
return fmt.Errorf("download alpine: %w", err)
}
o.log.Infof("running: tar xzf %s -C %s", tarPath, mnt)
if err := run("tar", "xzf", tarPath, "-C", mnt); err != nil {
return err
// download and extract minirootfs
switch distro {
case "alpine":
alpineVer := "3.20"
arch := "x86_64"
tarball := fmt.Sprintf("alpine-minirootfs-%s.0-%s.tar.gz", alpineVer, arch)
url := fmt.Sprintf("https://dl-cdn.alpinelinux.org/alpine/v%s/releases/%s/%s",
alpineVer, arch, tarball)
tarPath := filepath.Join(o.cfg.BaseDir, tarball)
o.log.Infof("downloading http request: GET %s to %s", url, tarPath)
if err := downloadFile(url, tarPath); err != nil {
return fmt.Errorf("download alpine: %w", err)
}
o.log.Infof("running: tar xzf %s -C %s", tarPath, mnt)
if err := run("tar", "xzf", tarPath, "-C", mnt); err != nil {
return err
}
case "debian":
tarball := "debian-12-nocloud-amd64.tar.xz"
url := "https://cloud.debian.org/images/cloud/bookworm/latest/" + tarball
tarPath := filepath.Join(o.cfg.BaseDir, tarball)
o.log.Infof("downloading http request: GET %s to %s", url, tarPath)
if err := downloadFile(url, tarPath); err != nil {
return fmt.Errorf("download debian: %w", err)
}
o.log.Infof("running: tar xJf %s -C %s", tarPath, mnt)
if err := run("tar", "xJf", tarPath, "-C", mnt); err != nil {
return err
}
case "ubuntu":
tarball := "ubuntu-base-24.04.4-base-amd64.tar.gz"
url := "https://cdimage.ubuntu.com/ubuntu-base/releases/24.04/release/" + tarball
tarPath := filepath.Join(o.cfg.BaseDir, tarball)
o.log.Infof("downloading http request: GET %s to %s", url, tarPath)
if err := downloadFile(url, tarPath); err != nil {
return fmt.Errorf("download ubuntu: %w", err)
}
o.log.Infof("running: tar xzf %s -C %s", tarPath, mnt)
if err := run("tar", "xzf", tarPath, "-C", mnt); err != nil {
return err
}
o.log.Info("installing essential packages in ubuntu chroot ...")
if err := installUbuntuPackages(mnt, o.log); err != nil {
return fmt.Errorf("install ubuntu packages: %w", err)
}
default:
return fmt.Errorf("unsupported distro: %s", distro)
}
// write fc-net-init daemon: polls MMDS for IP config and applies it.
@@ -139,8 +179,15 @@ done
return err
}
// write init script
initScript := `#!/bin/sh
// write fc-net-init
os.MkdirAll(filepath.Join(mnt, "sbin"), 0o755)
if err := os.WriteFile(filepath.Join(mnt, "sbin", "fc-net-init"), []byte(netInitScript), 0o755); err != nil {
return err
}
if distro == "alpine" {
// write init script
initScript := `#!/bin/sh
mount -t proc proc /proc
mount -t sysfs sys /sys
mount -t devtmpfs devtmpfs /dev
@@ -148,35 +195,83 @@ ip link set eth0 up 2>/dev/null
ip route add 169.254.169.254 dev eth0 2>/dev/null
/sbin/fc-net-init &
`
initPath := filepath.Join(mnt, "etc", "init.d", "rcS")
os.MkdirAll(filepath.Dir(initPath), 0o755)
if err := os.WriteFile(initPath, []byte(initScript), 0o755); err != nil {
return err
}
initPath := filepath.Join(mnt, "etc", "init.d", "rcS")
os.MkdirAll(filepath.Dir(initPath), 0o755)
if err := os.WriteFile(initPath, []byte(initScript), 0o755); err != nil {
return err
}
// write inittab
inittab := "::sysinit:/etc/init.d/rcS\nttyS0::respawn:/bin/sh\n"
return os.WriteFile(filepath.Join(mnt, "etc", "inittab"), []byte(inittab), 0o644)
// write inittab
inittab := "::sysinit:/etc/init.d/rcS\nttyS0::respawn:/bin/sh\n"
return os.WriteFile(filepath.Join(mnt, "etc", "inittab"), []byte(inittab), 0o644)
} else {
// systemd-based distributions (Debian, Ubuntu)
svc := `[Unit]
Description=Firecracker Network Init
After=network.target
[Service]
Type=oneshot
ExecStart=/sbin/fc-net-init
RemainAfterExit=yes
[Install]
WantedBy=multi-user.target
`
svcPath := filepath.Join(mnt, "etc", "systemd", "system", "fc-net-init.service")
os.MkdirAll(filepath.Dir(svcPath), 0o755)
if err := os.WriteFile(svcPath, []byte(svc), 0o644); err != nil {
return err
}
// Enable service dynamically
wantsDir := filepath.Join(mnt, "etc", "systemd", "system", "multi-user.target.wants")
os.MkdirAll(wantsDir, 0o755)
os.Symlink("/etc/systemd/system/fc-net-init.service", filepath.Join(wantsDir, "fc-net-init.service")) //nolint:errcheck
// Ensure serial console is active
gettyWantsDir := filepath.Join(mnt, "etc", "systemd", "system", "getty.target.wants")
os.MkdirAll(gettyWantsDir, 0o755)
os.Symlink("/lib/systemd/system/serial-getty@.service", filepath.Join(gettyWantsDir, "serial-getty@ttyS0.service")) //nolint:errcheck
// Clear root password for auto-login on console
shadowPath := filepath.Join(mnt, "etc", "shadow")
if shadowBytes, err := os.ReadFile(shadowPath); err == nil {
lines := strings.Split(string(shadowBytes), "\n")
for i, line := range lines {
if strings.HasPrefix(line, "root:") {
parts := strings.Split(line, ":")
if len(parts) > 1 {
parts[1] = ""
lines[i] = strings.Join(parts, ":")
}
}
}
os.WriteFile(shadowPath, []byte(strings.Join(lines, "\n")), 0o640) //nolint:errcheck
}
}
return nil
}
// ——— Golden VM ——————————————————————————————————————————————————————————
func (o *Orchestrator) Golden() error {
func (o *Orchestrator) Golden(tag string, distro string) error {
if _, err := os.Stat(o.cfg.Kernel); err != nil {
return fmt.Errorf("kernel not found — run init first: %w", err)
}
if _, err := os.Stat(o.cfg.Rootfs); err != nil {
rootfsPath := o.cfg.RootfsPath(distro)
if _, err := os.Stat(rootfsPath); err != nil {
return fmt.Errorf("rootfs not found — run init first: %w", err)
}
goldenDir := o.goldenDir()
goldenDir := o.goldenDir(tag)
os.RemoveAll(goldenDir)
os.MkdirAll(goldenDir, 0o755)
os.MkdirAll(o.pidsDir(), 0o755)
// COW copy of rootfs for golden VM
goldenRootfs := filepath.Join(goldenDir, "rootfs.ext4")
if err := reflinkCopy(o.cfg.Rootfs, goldenRootfs); err != nil {
if err := reflinkCopy(rootfsPath, goldenRootfs); err != nil {
return fmt.Errorf("copy rootfs: %w", err)
}
@@ -301,13 +396,29 @@ func (o *Orchestrator) Golden() error {
return nil
}
// GoldenTags returns a list of all existing golden VM tags.
func (o *Orchestrator) GoldenTags() []string {
goldenDir := filepath.Join(o.cfg.BaseDir, "golden")
entries, err := os.ReadDir(goldenDir)
if err != nil {
return nil
}
var tags []string
for _, e := range entries {
if e.IsDir() {
tags = append(tags, e.Name())
}
}
return tags
}
// ——— Spawn clones ——————————————————————————————————————————————————————
func (o *Orchestrator) Spawn(count int) error {
goldenDir := o.goldenDir()
func (o *Orchestrator) Spawn(count int, tag string) error {
goldenDir := o.goldenDir(tag)
for _, f := range []string{"vmstate", "mem"} {
if _, err := os.Stat(filepath.Join(goldenDir, f)); err != nil {
return fmt.Errorf("golden %s not found — run golden first", f)
return fmt.Errorf("golden %s not found for tag %s — run golden first", f, tag)
}
}
@@ -321,7 +432,7 @@ func (o *Orchestrator) Spawn(count int) error {
for i := 0; i < count; i++ {
id := o.nextCloneID()
if err := o.spawnOne(id, o.cfg.AutoNetConfig); err != nil {
if err := o.spawnOne(id, o.cfg.AutoNetConfig, tag); err != nil {
o.log.Errorf("clone %d failed: %v", id, err)
continue
}
@@ -338,11 +449,11 @@ func (o *Orchestrator) Spawn(count int) error {
// SpawnSingle spawns one clone. net controls whether the guest receives
// automatic IP configuration via MMDS (overrides FC_AUTO_NET_CONFIG for this
// clone). Pass cfg.AutoNetConfig to preserve the global default.
func (o *Orchestrator) SpawnSingle(net bool) (int, error) {
goldenDir := o.goldenDir()
func (o *Orchestrator) SpawnSingle(net bool, tag string) (int, error) {
goldenDir := o.goldenDir(tag)
for _, f := range []string{"vmstate", "mem"} {
if _, err := os.Stat(filepath.Join(goldenDir, f)); err != nil {
return 0, fmt.Errorf("golden %s not found — run golden first", f)
return 0, fmt.Errorf("golden %s not found for tag %s — run golden first", f, tag)
}
}
os.MkdirAll(o.clonesDir(), 0o755)
@@ -353,7 +464,7 @@ func (o *Orchestrator) SpawnSingle(net bool) (int, error) {
}
}
id := o.nextCloneID()
if err := o.spawnOne(id, net); err != nil {
if err := o.spawnOne(id, net, tag); err != nil {
return 0, err
}
return id, nil
@@ -379,8 +490,8 @@ func (o *Orchestrator) KillClone(id int) error {
return nil
}
func (o *Orchestrator) spawnOne(id int, net bool) error {
goldenDir := o.goldenDir()
func (o *Orchestrator) spawnOne(id int, net bool, tag string) error {
goldenDir := o.goldenDir(tag)
cloneDir := filepath.Join(o.clonesDir(), strconv.Itoa(id))
os.MkdirAll(cloneDir, 0o755)
@@ -419,7 +530,7 @@ func (o *Orchestrator) spawnOne(id int, net bool) error {
return fmt.Errorf("resolve self path: %w", err)
}
proxyArgs := []string{"_console-proxy", "--id", strconv.Itoa(id)}
proxyArgs := []string{"_console-proxy", "--id", strconv.Itoa(id), "--tag", tag}
if o.cfg.Bridge != "none" {
proxyArgs = append(proxyArgs, "--tap", tapName)
}
@@ -544,7 +655,7 @@ func (o *Orchestrator) Kill() error {
func (o *Orchestrator) Cleanup() error {
o.Kill()
os.RemoveAll(o.clonesDir())
os.RemoveAll(o.goldenDir())
os.RemoveAll(filepath.Join(o.cfg.BaseDir, "golden"))
os.RemoveAll(o.pidsDir())
if o.cfg.Bridge != "none" {
@@ -558,6 +669,56 @@ func (o *Orchestrator) Cleanup() error {
// ——— Helpers ——————————————————————————————————————————————————————————
// installUbuntuPackages bind-mounts the virtual filesystems into mnt, then
// runs apt-get inside the chroot to install the minimal toolset required for
// network operation and general use. Bind mounts are always cleaned up on
// return regardless of whether apt-get succeeds.
func installUbuntuPackages(mnt string, logger *log.Entry) error {
type bm struct{ fstype, src, dst string }
mounts := []bm{
{"proc", "proc", "proc"},
{"sysfs", "sysfs", "sys"},
{"devtmpfs", "devtmpfs", "dev"},
{"devpts", "devpts", "dev/pts"},
}
// mount in order; on any failure unmount whatever succeeded and return.
for i, m := range mounts {
dst := filepath.Join(mnt, m.dst)
os.MkdirAll(dst, 0o755)
logger.Infof("running: mount -t %s %s %s", m.fstype, m.src, dst)
if err := run("mount", "-t", m.fstype, m.src, dst); err != nil {
for j := i - 1; j >= 0; j-- {
logger.Infof("running: umount %s", filepath.Join(mnt, mounts[j].dst))
run("umount", filepath.Join(mnt, mounts[j].dst)) //nolint:errcheck
}
return fmt.Errorf("mount %s: %w", m.dst, err)
}
}
defer func() {
for i := len(mounts) - 1; i >= 0; i-- {
logger.Infof("running: umount %s", filepath.Join(mnt, mounts[i].dst))
run("umount", filepath.Join(mnt, mounts[i].dst)) //nolint:errcheck
}
}()
// Provide DNS resolution inside the chroot so apt-get can reach the network.
if data, err := os.ReadFile("/etc/resolv.conf"); err == nil {
os.WriteFile(filepath.Join(mnt, "etc/resolv.conf"), data, 0o644) //nolint:errcheck
}
pkgs := "bash curl iproute2 wget ca-certificates"
script := "DEBIAN_FRONTEND=noninteractive apt-get update -q && " +
"DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends " + pkgs + " && " +
"apt-get clean && rm -rf /var/lib/apt/lists/*"
logger.Infof("running: chroot %s /bin/sh -c %q", mnt, script)
cmd := exec.Command("chroot", mnt, "/bin/sh", "-c", script)
cmd.Stdout = logger.Writer()
cmd.Stderr = logger.Writer()
return cmd.Run()
}
func (o *Orchestrator) nextCloneID() int {
max := 0
entries, _ := os.ReadDir(o.clonesDir())