Compare commits
3 Commits
b46d510cb7
...
82c11dd2f8
| Author | SHA1 | Date | |
|---|---|---|---|
| 82c11dd2f8 | |||
| 9089cbdbe9 | |||
| 04067f7e6b |
7
go.mod
7
go.mod
@@ -1,6 +1,6 @@
|
||||
module github.com/kacerr/fc-orchestrator
|
||||
|
||||
go 1.23
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
github.com/firecracker-microvm/firecracker-go-sdk v1.0.0
|
||||
@@ -14,6 +14,7 @@ require (
|
||||
github.com/containerd/fifo v1.0.0 // indirect
|
||||
github.com/containernetworking/cni v1.0.1 // indirect
|
||||
github.com/containernetworking/plugins v1.0.1 // indirect
|
||||
github.com/creack/pty v1.1.24 // indirect
|
||||
github.com/go-openapi/analysis v0.21.2 // indirect
|
||||
github.com/go-openapi/errors v0.20.2 // indirect
|
||||
github.com/go-openapi/jsonpointer v0.19.5 // indirect
|
||||
@@ -26,6 +27,7 @@ require (
|
||||
github.com/go-openapi/validate v0.22.0 // indirect
|
||||
github.com/go-stack/stack v1.8.1 // indirect
|
||||
github.com/google/uuid v1.3.0 // indirect
|
||||
github.com/gorilla/websocket v1.5.3 // indirect
|
||||
github.com/hashicorp/errwrap v1.0.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
@@ -38,7 +40,8 @@ require (
|
||||
github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect
|
||||
go.mongodb.org/mongo-driver v1.8.3 // indirect
|
||||
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
|
||||
golang.org/x/sys v0.43.0 // indirect
|
||||
golang.org/x/term v0.42.0 // indirect
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
)
|
||||
|
||||
11
go.sum
11
go.sum
@@ -208,6 +208,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
|
||||
github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
|
||||
github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
|
||||
github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=
|
||||
github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=
|
||||
@@ -406,6 +408,8 @@ github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z
|
||||
github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
||||
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
|
||||
@@ -444,6 +448,7 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
|
||||
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
github.com/kacerrmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
|
||||
github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
|
||||
github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
|
||||
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
|
||||
@@ -686,7 +691,7 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:
|
||||
github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
||||
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
||||
github.com/kacerrmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=
|
||||
@@ -889,9 +894,13 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
|
||||
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=
|
||||
golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
|
||||
59
main.go
59
main.go
@@ -16,13 +16,39 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
||||
"github.com/kacerr/fc-orchestrator/orchestrator"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// strip --dev flag before subcommand routing
|
||||
dev := false
|
||||
filtered := os.Args[:1]
|
||||
for _, a := range os.Args[1:] {
|
||||
if a == "--dev" {
|
||||
dev = true
|
||||
} else {
|
||||
filtered = append(filtered, a)
|
||||
}
|
||||
}
|
||||
os.Args = filtered
|
||||
|
||||
if dev {
|
||||
log.SetReportCaller(true)
|
||||
log.SetFormatter(&log.TextFormatter{
|
||||
CallerPrettyfier: func(f *runtime.Frame) (string, string) {
|
||||
return "", fmt.Sprintf("%s:%d", filepath.Base(f.File), f.Line)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// figure out if we are running as root
|
||||
if os.Geteuid() == 0 {
|
||||
fmt.Println("Running with root/sudo privileges!")
|
||||
@@ -54,6 +80,32 @@ func main() {
|
||||
fatal(orch.Kill())
|
||||
case "cleanup":
|
||||
fatal(orch.Cleanup())
|
||||
case "serve":
|
||||
addr := ":8080"
|
||||
if len(os.Args) > 2 {
|
||||
addr = os.Args[2]
|
||||
}
|
||||
fatal(orchestrator.Serve(orch, addr))
|
||||
case "console":
|
||||
if len(os.Args) < 3 {
|
||||
fmt.Fprintf(os.Stderr, "usage: %s console <id>\n", os.Args[0])
|
||||
os.Exit(1)
|
||||
}
|
||||
var id int
|
||||
fmt.Sscanf(os.Args[2], "%d", &id)
|
||||
fatal(orchestrator.ConnectConsole(orchestrator.DefaultConfig(), id))
|
||||
case "_console-proxy":
|
||||
// Internal subcommand: started by spawnOne, runs as a background daemon.
|
||||
fs := flag.NewFlagSet("_console-proxy", flag.ContinueOnError)
|
||||
var id int
|
||||
var tap string
|
||||
fs.IntVar(&id, "id", 0, "clone ID")
|
||||
fs.StringVar(&tap, "tap", "", "TAP device name")
|
||||
if err := fs.Parse(os.Args[2:]); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "console-proxy: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fatal(orchestrator.RunConsoleProxy(orchestrator.DefaultConfig(), id, tap))
|
||||
default:
|
||||
usage()
|
||||
os.Exit(1)
|
||||
@@ -61,12 +113,17 @@ func main() {
|
||||
}
|
||||
|
||||
func usage() {
|
||||
fmt.Fprintf(os.Stderr, `Usage: %s <command> [args]
|
||||
fmt.Fprintf(os.Stderr, `Usage: %s [--dev] <command> [args]
|
||||
|
||||
Flags:
|
||||
--dev log format with source file:line (e.g. file="orchestrator.go:123")
|
||||
|
||||
Commands:
|
||||
init Download kernel + create Alpine rootfs
|
||||
golden Boot golden VM → pause → snapshot
|
||||
spawn [N] Restore N clones from golden snapshot (default: 1)
|
||||
serve [addr] Start terminal web UI (default: :8080)
|
||||
console <id> Attach to the serial console of a running clone (Ctrl+] to detach)
|
||||
status Show running clones
|
||||
kill Kill all running VMs
|
||||
cleanup Kill VMs + remove all state
|
||||
|
||||
318
orchestrator/console.go
Normal file
318
orchestrator/console.go
Normal file
@@ -0,0 +1,318 @@
|
||||
package orchestrator
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/creack/pty"
|
||||
firecracker "github.com/firecracker-microvm/firecracker-go-sdk"
|
||||
"github.com/firecracker-microvm/firecracker-go-sdk/client/models"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// RunConsoleProxy is the entrypoint for the "_console-proxy" internal subcommand.
|
||||
// It restores a Firecracker clone from the golden snapshot, connecting its serial
|
||||
// console (ttyS0) to a PTY, then serves the PTY master on a Unix socket at
|
||||
// {cloneDir}/console.sock for the lifetime of the VM.
|
||||
func RunConsoleProxy(cfg Config, id int, tapName string) error {
|
||||
logger := log.WithField("component", fmt.Sprintf("console-proxy[%d]", id))
|
||||
|
||||
cloneDir := filepath.Join(cfg.BaseDir, "clones", strconv.Itoa(id))
|
||||
goldenDir := filepath.Join(cfg.BaseDir, "golden")
|
||||
sockPath := filepath.Join(cloneDir, "api.sock")
|
||||
consoleSockPath := filepath.Join(cloneDir, "console.sock")
|
||||
sharedMem := filepath.Join(goldenDir, "mem")
|
||||
cloneVmstate := filepath.Join(cloneDir, "vmstate")
|
||||
|
||||
// --- Create PTY ---
|
||||
// ptm = master (we hold and proxy), pts = slave (firecracker's stdio)
|
||||
ptm, pts, err := pty.Open()
|
||||
if err != nil {
|
||||
return fmt.Errorf("open pty: %w", err)
|
||||
}
|
||||
pty.Setsize(ptm, &pty.Winsize{Rows: 24, Cols: 80}) //nolint:errcheck
|
||||
|
||||
fcBin, err := exec.LookPath(cfg.FCBin)
|
||||
if err != nil {
|
||||
pts.Close()
|
||||
ptm.Close()
|
||||
return fmt.Errorf("firecracker not found: %w", err)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
// --- Build firecracker command with slave PTY as stdin/stdout/stderr ---
|
||||
// Setsid makes the slave PTY the controlling terminal for firecracker,
|
||||
// which is required for job control (Ctrl+C, SIGWINCH, etc.) to work.
|
||||
cmd := firecracker.VMCommandBuilder{}.
|
||||
WithBin(fcBin).
|
||||
WithSocketPath(sockPath).
|
||||
Build(ctx)
|
||||
cmd.Stdin = pts
|
||||
cmd.Stdout = pts
|
||||
cmd.Stderr = pts
|
||||
cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}
|
||||
|
||||
// --- Build Machine config (mirrors spawnOne) ---
|
||||
vcpus := cfg.VCPUs
|
||||
mem := cfg.MemMiB
|
||||
|
||||
fcCfg := firecracker.Config{
|
||||
SocketPath: sockPath,
|
||||
MachineCfg: models.MachineConfiguration{
|
||||
VcpuCount: &vcpus,
|
||||
MemSizeMib: &mem,
|
||||
},
|
||||
LogPath: sockPath + ".log",
|
||||
LogLevel: "Debug",
|
||||
FifoLogWriter: logger.Writer(),
|
||||
}
|
||||
|
||||
if cfg.Bridge != "none" && tapName != "" {
|
||||
mac := fmt.Sprintf("AA:FC:00:00:%02X:%02X", id/256, id%256)
|
||||
fcCfg.NetworkInterfaces = firecracker.NetworkInterfaces{
|
||||
{
|
||||
StaticConfiguration: &firecracker.StaticNetworkConfiguration{
|
||||
MacAddress: mac,
|
||||
HostDevName: tapName,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
m, err := firecracker.NewMachine(ctx, fcCfg,
|
||||
firecracker.WithProcessRunner(cmd),
|
||||
firecracker.WithLogger(logger),
|
||||
firecracker.WithSnapshot(sharedMem, cloneVmstate, func(sc *firecracker.SnapshotConfig) {
|
||||
sc.ResumeVM = true
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
pts.Close()
|
||||
ptm.Close()
|
||||
return fmt.Errorf("new machine: %w", err)
|
||||
}
|
||||
|
||||
// Swap in network-override snapshot handler (same as spawnOne)
|
||||
if cfg.Bridge != "none" && tapName != "" {
|
||||
m.Handlers.FcInit = m.Handlers.FcInit.Swap(firecracker.Handler{
|
||||
Name: firecracker.LoadSnapshotHandlerName,
|
||||
Fn: func(ctx context.Context, m *firecracker.Machine) error {
|
||||
return loadSnapshotWithNetworkOverride(
|
||||
ctx, sockPath, sharedMem, cloneVmstate, tapName,
|
||||
)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// --- Start VM (blocks until snapshot is loaded and VM is running) ---
|
||||
start := time.Now()
|
||||
logger.Infof("restoring clone %d from snapshot ...", id)
|
||||
if err := m.Start(ctx); err != nil {
|
||||
pts.Close()
|
||||
ptm.Close()
|
||||
return fmt.Errorf("restore clone %d: %w", id, err)
|
||||
}
|
||||
elapsed := time.Since(start)
|
||||
|
||||
// Release our copy of the slave — firecracker holds its own fd now.
|
||||
// Closing here ensures we get EOF on ptm when firecracker exits.
|
||||
pts.Close()
|
||||
|
||||
// --- Write PID file ---
|
||||
pidsDir := filepath.Join(cfg.BaseDir, "pids")
|
||||
os.MkdirAll(pidsDir, 0o755) //nolint:errcheck
|
||||
if cmd.Process != nil {
|
||||
os.WriteFile( //nolint:errcheck
|
||||
filepath.Join(pidsDir, fmt.Sprintf("clone-%d.pid", id)),
|
||||
[]byte(strconv.Itoa(cmd.Process.Pid)),
|
||||
0o644,
|
||||
)
|
||||
}
|
||||
logger.Infof("clone %d: restored in %s (pid=%d, tap=%s)",
|
||||
id, elapsed.Round(time.Millisecond), cmd.Process.Pid, tapName)
|
||||
|
||||
// --- Create console socket ---
|
||||
os.Remove(consoleSockPath) //nolint:errcheck
|
||||
listener, err := net.Listen("unix", consoleSockPath)
|
||||
if err != nil {
|
||||
ptm.Close()
|
||||
return fmt.Errorf("listen on console socket: %w", err)
|
||||
}
|
||||
|
||||
// --- Create resize sideband socket ---
|
||||
resizeSockPath := filepath.Join(cloneDir, "console-resize.sock")
|
||||
os.Remove(resizeSockPath) //nolint:errcheck
|
||||
resizeListener, err := net.Listen("unix", resizeSockPath)
|
||||
if err != nil {
|
||||
logger.Warnf("could not create resize socket, terminal resize will be unavailable: %v", err)
|
||||
resizeListener = nil
|
||||
}
|
||||
|
||||
// --- Serve until VM exits ---
|
||||
vmDone := make(chan struct{})
|
||||
go func() {
|
||||
m.Wait(ctx) //nolint:errcheck
|
||||
close(vmDone)
|
||||
}()
|
||||
|
||||
if resizeListener != nil {
|
||||
go serveResize(resizeListener, ptm, vmDone, logger)
|
||||
}
|
||||
serveConsole(listener, ptm, vmDone, logger)
|
||||
|
||||
listener.Close()
|
||||
if resizeListener != nil {
|
||||
resizeListener.Close()
|
||||
}
|
||||
ptm.Close()
|
||||
os.Remove(consoleSockPath) //nolint:errcheck
|
||||
os.Remove(resizeSockPath) //nolint:errcheck
|
||||
|
||||
if tapName != "" {
|
||||
destroyTap(tapName)
|
||||
}
|
||||
|
||||
logger.Infof("clone %d: exiting", id)
|
||||
return nil
|
||||
}
|
||||
|
||||
// resizeMsg is the JSON payload sent over the resize sideband socket.
|
||||
type resizeMsg struct {
|
||||
Rows uint16 `json:"rows"`
|
||||
Cols uint16 `json:"cols"`
|
||||
}
|
||||
|
||||
// serveResize accepts connections on the resize sideband listener and applies
|
||||
// PTY window size changes as JSON resize messages arrive.
|
||||
func serveResize(listener net.Listener, ptm *os.File, vmDone <-chan struct{}, logger *log.Entry) {
|
||||
go func() {
|
||||
<-vmDone
|
||||
listener.Close()
|
||||
}()
|
||||
for {
|
||||
conn, err := listener.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go handleResize(conn, ptm, logger)
|
||||
}
|
||||
}
|
||||
|
||||
// handleResize reads newline-delimited JSON resize messages from conn and
|
||||
// applies each one to the PTY master.
|
||||
func handleResize(conn net.Conn, ptm *os.File, logger *log.Entry) {
|
||||
defer conn.Close()
|
||||
scanner := bufio.NewScanner(conn)
|
||||
for scanner.Scan() {
|
||||
var msg resizeMsg
|
||||
if err := json.Unmarshal(scanner.Bytes(), &msg); err != nil {
|
||||
logger.Warnf("resize: bad message: %v", err)
|
||||
continue
|
||||
}
|
||||
if msg.Rows == 0 || msg.Cols == 0 {
|
||||
continue
|
||||
}
|
||||
if err := pty.Setsize(ptm, &pty.Winsize{Rows: msg.Rows, Cols: msg.Cols}); err != nil {
|
||||
logger.Warnf("resize pty: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// atomicWriter wraps an io.Writer behind a read/write lock so it can be
|
||||
// swapped between connections without holding the lock during writes.
|
||||
type atomicWriter struct {
|
||||
mu sync.RWMutex
|
||||
w io.Writer
|
||||
}
|
||||
|
||||
func (a *atomicWriter) set(w io.Writer) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
a.w = w
|
||||
}
|
||||
|
||||
func (a *atomicWriter) Write(p []byte) (int, error) {
|
||||
a.mu.RLock()
|
||||
defer a.mu.RUnlock()
|
||||
return a.w.Write(p)
|
||||
}
|
||||
|
||||
// serveConsole accepts connections on listener and proxies console I/O via ptm.
|
||||
// A background goroutine reads from the PTY master continuously (discarding
|
||||
// output when no client is connected so the VM never blocks on a full buffer).
|
||||
// Only one client is served at a time; sessions are serialised.
|
||||
func serveConsole(listener net.Listener, ptm *os.File, vmDone <-chan struct{}, logger *log.Entry) {
|
||||
aw := &atomicWriter{w: io.Discard}
|
||||
|
||||
// Background PTY reader — runs for the full VM lifetime.
|
||||
go func() {
|
||||
buf := make([]byte, 4096)
|
||||
for {
|
||||
n, err := ptm.Read(buf)
|
||||
if n > 0 {
|
||||
aw.Write(buf[:n]) //nolint:errcheck
|
||||
}
|
||||
if err != nil {
|
||||
return // PTY closed (VM exited)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Unblock Accept() when the VM exits.
|
||||
go func() {
|
||||
<-vmDone
|
||||
listener.Close()
|
||||
}()
|
||||
|
||||
for {
|
||||
conn, err := listener.Accept()
|
||||
if err != nil {
|
||||
return // listener closed
|
||||
}
|
||||
|
||||
logger.Info("console client connected")
|
||||
|
||||
// Route PTY output to this connection.
|
||||
aw.set(conn)
|
||||
|
||||
// Forward client input to the PTY master.
|
||||
clientGone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(clientGone)
|
||||
buf := make([]byte, 256)
|
||||
for {
|
||||
n, err := conn.Read(buf)
|
||||
if n > 0 {
|
||||
ptm.Write(buf[:n]) //nolint:errcheck
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Block until client disconnects or VM dies.
|
||||
select {
|
||||
case <-clientGone:
|
||||
logger.Info("console client disconnected")
|
||||
case <-vmDone:
|
||||
logger.Info("VM exited while console client was connected")
|
||||
}
|
||||
|
||||
aw.set(io.Discard)
|
||||
conn.Close()
|
||||
}
|
||||
}
|
||||
104
orchestrator/console_client.go
Normal file
104
orchestrator/console_client.go
Normal file
@@ -0,0 +1,104 @@
|
||||
package orchestrator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"sync"
|
||||
"syscall"
|
||||
|
||||
"golang.org/x/term"
|
||||
)
|
||||
|
||||
// ConnectConsole attaches the calling terminal to the serial console of the
|
||||
// clone with the given ID. The connection is made over the Unix socket at
|
||||
// {cloneDir}/console.sock served by the console-proxy process.
|
||||
//
|
||||
// Escape sequence: Ctrl+] (byte 0x1D) detaches without stopping the VM.
|
||||
func ConnectConsole(cfg Config, id int) error {
|
||||
sockPath := filepath.Join(cfg.BaseDir, "clones", strconv.Itoa(id), "console.sock")
|
||||
if _, err := os.Stat(sockPath); err != nil {
|
||||
return fmt.Errorf("clone %d has no console socket — is it running?", id)
|
||||
}
|
||||
|
||||
conn, err := net.Dial("unix", sockPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("connect to console: %w", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
if !term.IsTerminal(int(os.Stdin.Fd())) {
|
||||
return fmt.Errorf("console requires an interactive terminal")
|
||||
}
|
||||
|
||||
oldState, err := term.MakeRaw(int(os.Stdin.Fd()))
|
||||
if err != nil {
|
||||
return fmt.Errorf("set raw terminal: %w", err)
|
||||
}
|
||||
defer term.Restore(int(os.Stdin.Fd()), oldState) //nolint:errcheck
|
||||
|
||||
// Ensure the terminal is restored even on SIGTERM / SIGHUP.
|
||||
sigCh := make(chan os.Signal, 1)
|
||||
signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGHUP)
|
||||
go func() {
|
||||
<-sigCh
|
||||
term.Restore(int(os.Stdin.Fd()), oldState) //nolint:errcheck
|
||||
os.Exit(0)
|
||||
}()
|
||||
|
||||
fmt.Fprintf(os.Stderr, "Connected to clone %d console. Escape: Ctrl+]\r\n", id)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
|
||||
// stdin → VM (with Ctrl+] escape detection)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
io.Copy(conn, &escapeReader{r: os.Stdin, cancel: cancel}) //nolint:errcheck
|
||||
// Half-close so the proxy knows we're done sending.
|
||||
if uc, ok := conn.(*net.UnixConn); ok {
|
||||
uc.CloseWrite() //nolint:errcheck
|
||||
}
|
||||
}()
|
||||
|
||||
// VM → stdout
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
io.Copy(os.Stdout, conn) //nolint:errcheck
|
||||
cancel() // VM exited or proxy closed the connection
|
||||
}()
|
||||
|
||||
<-ctx.Done()
|
||||
conn.Close() // unblock both goroutines
|
||||
wg.Wait()
|
||||
|
||||
fmt.Fprintf(os.Stderr, "\r\nDetached from clone %d.\r\n", id)
|
||||
return nil
|
||||
}
|
||||
|
||||
// escapeReader wraps an io.Reader and intercepts Ctrl+] (0x1D), calling cancel
|
||||
// and returning io.EOF when the escape byte is seen.
|
||||
type escapeReader struct {
|
||||
r io.Reader
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (e *escapeReader) Read(p []byte) (int, error) {
|
||||
n, err := e.r.Read(p)
|
||||
for i := 0; i < n; i++ {
|
||||
if p[i] == 0x1D { // Ctrl+]
|
||||
e.cancel()
|
||||
// Return only the bytes before the escape so nothing leaks to the VM.
|
||||
return i, io.EOF
|
||||
}
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
firecracker "github.com/firecracker-microvm/firecracker-go-sdk"
|
||||
@@ -203,6 +204,9 @@ func (o *Orchestrator) Golden() error {
|
||||
},
|
||||
},
|
||||
NetworkInterfaces: netIfaces,
|
||||
LogPath: sockPath + ".log",
|
||||
LogLevel: "Debug",
|
||||
FifoLogWriter: o.log.Writer(),
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
@@ -218,7 +222,10 @@ func (o *Orchestrator) Golden() error {
|
||||
WithSocketPath(sockPath).
|
||||
Build(ctx)
|
||||
|
||||
m, err := firecracker.NewMachine(ctx, fcCfg, firecracker.WithProcessRunner(cmd))
|
||||
m, err := firecracker.NewMachine(ctx, fcCfg,
|
||||
firecracker.WithProcessRunner(cmd),
|
||||
firecracker.WithLogger(o.log),
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("new machine: %w", err)
|
||||
}
|
||||
@@ -298,6 +305,50 @@ func (o *Orchestrator) Spawn(count int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// SpawnSingle spawns exactly one new clone and returns its ID.
|
||||
// It is safe to call from multiple goroutines (nextCloneID is serialised by the
|
||||
// filesystem scan, and each clone gets its own directory/tap).
|
||||
func (o *Orchestrator) SpawnSingle() (int, error) {
|
||||
goldenDir := o.goldenDir()
|
||||
for _, f := range []string{"vmstate", "mem"} {
|
||||
if _, err := os.Stat(filepath.Join(goldenDir, f)); err != nil {
|
||||
return 0, fmt.Errorf("golden %s not found — run golden first", f)
|
||||
}
|
||||
}
|
||||
os.MkdirAll(o.clonesDir(), 0o755)
|
||||
os.MkdirAll(o.pidsDir(), 0o755)
|
||||
if o.cfg.Bridge != "none" {
|
||||
if err := o.setupBridge(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
id := o.nextCloneID()
|
||||
if err := o.spawnOne(id); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// KillClone kills a single clone by ID: terminates its proxy process,
|
||||
// destroys its tap device, and removes its working directory.
|
||||
func (o *Orchestrator) KillClone(id int) error {
|
||||
pidFile := filepath.Join(o.pidsDir(), fmt.Sprintf("clone-%d.proxy.pid", id))
|
||||
if data, err := os.ReadFile(pidFile); err == nil {
|
||||
if pid, err := strconv.Atoi(strings.TrimSpace(string(data))); err == nil {
|
||||
if p, err := os.FindProcess(pid); err == nil {
|
||||
_ = p.Kill()
|
||||
o.log.Infof("clone %d: killed proxy pid %d", id, pid)
|
||||
}
|
||||
}
|
||||
os.Remove(pidFile) //nolint:errcheck
|
||||
}
|
||||
tapName := fmt.Sprintf("fctap%d", id)
|
||||
destroyTap(tapName)
|
||||
os.RemoveAll(filepath.Join(o.clonesDir(), strconv.Itoa(id))) //nolint:errcheck
|
||||
o.log.Infof("clone %d: destroyed", id)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *Orchestrator) spawnOne(id int) error {
|
||||
goldenDir := o.goldenDir()
|
||||
cloneDir := filepath.Join(o.clonesDir(), strconv.Itoa(id))
|
||||
@@ -308,14 +359,11 @@ func (o *Orchestrator) spawnOne(id int) error {
|
||||
|
||||
// --- COW rootfs ---
|
||||
cloneRootfs := filepath.Join(cloneDir, "rootfs.ext4")
|
||||
o.log.Infof("clone %d: running: cp --reflink=always %s %s", id, filepath.Join(goldenDir, "rootfs.ext4"), cloneRootfs)
|
||||
if err := reflinkCopy(filepath.Join(goldenDir, "rootfs.ext4"), cloneRootfs); err != nil {
|
||||
return fmt.Errorf("copy rootfs: %w", err)
|
||||
}
|
||||
|
||||
// --- Memory: point at the shared golden mem file ---
|
||||
// Firecracker uses MAP_PRIVATE → kernel COW. No copy needed.
|
||||
sharedMem := filepath.Join(goldenDir, "mem")
|
||||
|
||||
// --- vmstate: small, cheap copy ---
|
||||
cloneVmstate := filepath.Join(cloneDir, "vmstate")
|
||||
if err := copyFile(filepath.Join(goldenDir, "vmstate"), cloneVmstate); err != nil {
|
||||
@@ -324,88 +372,64 @@ func (o *Orchestrator) spawnOne(id int) error {
|
||||
|
||||
// --- Networking ---
|
||||
tapName := fmt.Sprintf("fctap%d", id)
|
||||
var netIfaces firecracker.NetworkInterfaces
|
||||
if o.cfg.Bridge != "none" {
|
||||
o.log.Infof("clone %d: running: ip tuntap add dev %s mode tap", id, tapName)
|
||||
o.log.Infof("clone %d: running: ip link set %s up", id, tapName)
|
||||
o.log.Infof("clone %d: running: ip link set %s master %s", id, tapName, o.cfg.Bridge)
|
||||
if err := o.createTap(tapName); err != nil {
|
||||
return err
|
||||
}
|
||||
mac := fmt.Sprintf("AA:FC:00:00:%02X:%02X", id/256, id%256)
|
||||
netIfaces = firecracker.NetworkInterfaces{
|
||||
firecracker.NetworkInterface{
|
||||
StaticConfiguration: &firecracker.StaticNetworkConfiguration{
|
||||
MacAddress: mac,
|
||||
HostDevName: tapName,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// --- Restore from snapshot ---
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
fcBin, err := exec.LookPath(o.cfg.FCBin)
|
||||
// --- Launch console proxy (detached daemon) ---
|
||||
// The proxy owns the full VM lifecycle: it starts firecracker with a PTY,
|
||||
// loads the snapshot, and serves cloneDir/console.sock until the VM exits.
|
||||
selfExe, err := os.Executable()
|
||||
if err != nil {
|
||||
cancel()
|
||||
return fmt.Errorf("firecracker not found: %w", err)
|
||||
return fmt.Errorf("resolve self path: %w", err)
|
||||
}
|
||||
|
||||
cmd := firecracker.VMCommandBuilder{}.
|
||||
WithBin(fcBin).
|
||||
WithSocketPath(sockPath).
|
||||
Build(ctx)
|
||||
proxyArgs := []string{"_console-proxy", "--id", strconv.Itoa(id)}
|
||||
if o.cfg.Bridge != "none" {
|
||||
proxyArgs = append(proxyArgs, "--tap", tapName)
|
||||
}
|
||||
proxyCmd := exec.Command(selfExe, proxyArgs...)
|
||||
// New session: proxy is detached from our terminal and survives our exit.
|
||||
proxyCmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true}
|
||||
proxyCmd.Stdin = nil
|
||||
proxyCmd.Stdout = nil
|
||||
proxyCmd.Stderr = nil
|
||||
|
||||
vcpus := o.cfg.VCPUs
|
||||
mem := o.cfg.MemMiB
|
||||
if err := proxyCmd.Start(); err != nil {
|
||||
return fmt.Errorf("start console proxy: %w", err)
|
||||
}
|
||||
os.WriteFile(filepath.Join(o.pidsDir(), fmt.Sprintf("clone-%d.proxy.pid", id)),
|
||||
[]byte(strconv.Itoa(proxyCmd.Process.Pid)), 0o644) //nolint:errcheck
|
||||
|
||||
fcCfg := firecracker.Config{
|
||||
SocketPath: sockPath,
|
||||
MachineCfg: models.MachineConfiguration{
|
||||
VcpuCount: &vcpus,
|
||||
MemSizeMib: &mem,
|
||||
},
|
||||
NetworkInterfaces: netIfaces,
|
||||
// Snapshot config: tells the SDK to restore instead of fresh boot.
|
||||
Snapshot: firecracker.SnapshotConfig{
|
||||
MemFilePath: sharedMem,
|
||||
SnapshotPath: cloneVmstate,
|
||||
ResumeVM: true,
|
||||
},
|
||||
// Wait for the console socket to appear — it is created by the proxy once
|
||||
// the VM is running, so this also gates on successful snapshot restore.
|
||||
consoleSockPath := filepath.Join(cloneDir, "console.sock")
|
||||
if err := waitForSocket(consoleSockPath, 15*time.Second); err != nil {
|
||||
return fmt.Errorf("clone %d: %w", id, err)
|
||||
}
|
||||
|
||||
m, err := firecracker.NewMachine(ctx, fcCfg, firecracker.WithProcessRunner(cmd))
|
||||
if err != nil {
|
||||
cancel()
|
||||
return fmt.Errorf("new machine: %w", err)
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
if err := m.Start(ctx); err != nil {
|
||||
cancel()
|
||||
return fmt.Errorf("restore clone %d: %w", id, err)
|
||||
}
|
||||
elapsed := time.Since(start)
|
||||
|
||||
// store PID
|
||||
if cmd.Process != nil {
|
||||
os.WriteFile(filepath.Join(o.pidsDir(), fmt.Sprintf("clone-%d.pid", id)),
|
||||
[]byte(strconv.Itoa(cmd.Process.Pid)), 0o644)
|
||||
}
|
||||
|
||||
o.mu.Lock()
|
||||
o.clones[id] = &cloneInfo{
|
||||
ID: id,
|
||||
Machine: m,
|
||||
Cancel: cancel,
|
||||
Tap: tapName,
|
||||
}
|
||||
o.mu.Unlock()
|
||||
|
||||
o.log.Infof("clone %d: restored in %s (pid=%d, tap=%s)",
|
||||
id, elapsed.Round(time.Millisecond), cmd.Process.Pid, tapName)
|
||||
|
||||
o.log.Infof("clone %d: ready (proxy pid=%d, tap=%s, console=%s)",
|
||||
id, proxyCmd.Process.Pid, tapName, consoleSockPath)
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitForSocket polls path every 50 ms until it appears or timeout elapses.
|
||||
func waitForSocket(path string, timeout time.Duration) error {
|
||||
deadline := time.Now().Add(timeout)
|
||||
for time.Now().Before(deadline) {
|
||||
if _, err := os.Stat(path); err == nil {
|
||||
return nil
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
return fmt.Errorf("timed out waiting for %s", path)
|
||||
}
|
||||
|
||||
// ——— Status ————————————————————————————————————————————————————————————
|
||||
|
||||
func (o *Orchestrator) Status() {
|
||||
|
||||
218
orchestrator/serve.go
Normal file
218
orchestrator/serve.go
Normal file
@@ -0,0 +1,218 @@
|
||||
package orchestrator
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
_ "embed"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
//go:embed web/terminal.html
|
||||
var terminalHTML []byte
|
||||
|
||||
var upgrader = websocket.Upgrader{
|
||||
// Allow all origins for local/dev use. Add an origin check for production.
|
||||
CheckOrigin: func(r *http.Request) bool { return true },
|
||||
}
|
||||
|
||||
// Serve starts an HTTP server that exposes:
|
||||
//
|
||||
// GET / — terminal UI (xterm.js); ?id=N selects a clone
|
||||
// GET /clones — JSON list of running clone IDs
|
||||
// POST /clones — spawn a new clone; returns {"id": N}
|
||||
// DELETE /clones/{id} — destroy clone {id}
|
||||
// GET /ws/{id} — WebSocket console for clone {id}
|
||||
//
|
||||
// Binary WebSocket frames carry raw terminal bytes (stdin/stdout).
|
||||
// Text WebSocket frames carry JSON resize commands: {"rows":N,"cols":M}.
|
||||
func Serve(orch *Orchestrator, addr string) error {
|
||||
mux := http.NewServeMux()
|
||||
|
||||
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
w.Write(terminalHTML) //nolint:errcheck
|
||||
})
|
||||
|
||||
// /clones — list (GET) or spawn (POST)
|
||||
mux.HandleFunc("/clones", func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodGet, "":
|
||||
ids := runningCloneIDs(orch.cfg)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
json.NewEncoder(w).Encode(ids) //nolint:errcheck
|
||||
case http.MethodPost:
|
||||
id, err := orch.SpawnSingle()
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
json.NewEncoder(w).Encode(map[string]int{"id": id}) //nolint:errcheck
|
||||
default:
|
||||
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
||||
}
|
||||
})
|
||||
|
||||
// /clones/{id} — destroy (DELETE)
|
||||
mux.HandleFunc("/clones/", func(w http.ResponseWriter, r *http.Request) {
|
||||
idStr := strings.TrimPrefix(r.URL.Path, "/clones/")
|
||||
if idStr == "" {
|
||||
// redirect bare /clones/ to /clones
|
||||
http.Redirect(w, r, "/clones", http.StatusMovedPermanently)
|
||||
return
|
||||
}
|
||||
if r.Method != http.MethodDelete {
|
||||
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
id, err := strconv.Atoi(idStr)
|
||||
if err != nil {
|
||||
http.Error(w, "invalid clone id", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := orch.KillClone(id); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
})
|
||||
|
||||
mux.HandleFunc("/ws/", func(w http.ResponseWriter, r *http.Request) {
|
||||
idStr := strings.TrimPrefix(r.URL.Path, "/ws/")
|
||||
id, err := strconv.Atoi(idStr)
|
||||
if err != nil {
|
||||
http.Error(w, "invalid clone id", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
handleConsoleWS(orch.cfg, id, w, r)
|
||||
})
|
||||
|
||||
log.Infof("terminal UI: http://%s", addr)
|
||||
return http.ListenAndServe(addr, mux)
|
||||
}
|
||||
|
||||
// handleConsoleWS upgrades the request to a WebSocket and bridges it to the
|
||||
// clone's console.sock (I/O) and console-resize.sock (terminal resize).
|
||||
func handleConsoleWS(cfg Config, id int, w http.ResponseWriter, r *http.Request) {
|
||||
cloneDir := filepath.Join(cfg.BaseDir, "clones", strconv.Itoa(id))
|
||||
consoleSock := filepath.Join(cloneDir, "console.sock")
|
||||
resizeSock := filepath.Join(cloneDir, "console-resize.sock")
|
||||
|
||||
if _, err := os.Stat(consoleSock); err != nil {
|
||||
http.Error(w, fmt.Sprintf("clone %d is not running", id), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
ws, err := upgrader.Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer ws.Close()
|
||||
|
||||
consoleConn, err := net.Dial("unix", consoleSock)
|
||||
if err != nil {
|
||||
writeWSError(ws, fmt.Sprintf("console unavailable: %v", err))
|
||||
return
|
||||
}
|
||||
defer consoleConn.Close()
|
||||
|
||||
resizeConn, err := net.Dial("unix", resizeSock)
|
||||
if err != nil {
|
||||
log.Warnf("clone %d: could not connect to resize socket, resize disabled: %v", id, err)
|
||||
resizeConn = nil
|
||||
}
|
||||
if resizeConn != nil {
|
||||
defer resizeConn.Close()
|
||||
}
|
||||
|
||||
log.Infof("ws: clone %d: client connected from %s", id, r.RemoteAddr)
|
||||
bridgeWS(ws, consoleConn, resizeConn)
|
||||
log.Infof("ws: clone %d: client disconnected", id)
|
||||
}
|
||||
|
||||
// bridgeWS proxies between a WebSocket connection and a console Unix socket.
|
||||
//
|
||||
// - Binary WS frames → consoleConn (terminal input)
|
||||
// - Text WS frames → resizeConn as a JSON line (resize command)
|
||||
// - consoleConn reads → binary WS frames (terminal output)
|
||||
func bridgeWS(ws *websocket.Conn, consoleConn net.Conn, resizeConn net.Conn) {
|
||||
// console → WebSocket
|
||||
sockDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(sockDone)
|
||||
buf := make([]byte, 4096)
|
||||
for {
|
||||
n, err := consoleConn.Read(buf)
|
||||
if n > 0 {
|
||||
if werr := ws.WriteMessage(websocket.BinaryMessage, buf[:n]); werr != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// WebSocket → console / resize
|
||||
for {
|
||||
msgType, data, err := ws.ReadMessage()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
switch msgType {
|
||||
case websocket.BinaryMessage:
|
||||
consoleConn.Write(data) //nolint:errcheck
|
||||
case websocket.TextMessage:
|
||||
if resizeConn != nil {
|
||||
// Append newline so the scanner in handleResize sees a complete line.
|
||||
resizeConn.Write(append(data, '\n')) //nolint:errcheck
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
consoleConn.Close()
|
||||
<-sockDone
|
||||
}
|
||||
|
||||
// runningCloneIDs returns clone IDs that have a live console socket.
|
||||
func runningCloneIDs(cfg Config) []int {
|
||||
clonesDir := filepath.Join(cfg.BaseDir, "clones")
|
||||
entries, err := os.ReadDir(clonesDir)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var ids []int
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() {
|
||||
continue
|
||||
}
|
||||
id, err := strconv.Atoi(e.Name())
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
sock := filepath.Join(clonesDir, e.Name(), "console.sock")
|
||||
if _, err := os.Stat(sock); err == nil {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func writeWSError(ws *websocket.Conn, msg string) {
|
||||
ws.WriteMessage(websocket.TextMessage, //nolint:errcheck
|
||||
[]byte("\r\n\x1b[31m["+msg+"]\x1b[0m\r\n"))
|
||||
}
|
||||
71
orchestrator/snapshot.go
Normal file
71
orchestrator/snapshot.go
Normal file
@@ -0,0 +1,71 @@
|
||||
package orchestrator
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
type networkOverride struct {
|
||||
IfaceID string `json:"iface_id"`
|
||||
HostDevName string `json:"host_dev_name"`
|
||||
}
|
||||
|
||||
type snapshotLoadRequest struct {
|
||||
MemFilePath string `json:"mem_file_path"`
|
||||
SnapshotPath string `json:"snapshot_path"`
|
||||
ResumeVM bool `json:"resume_vm,omitempty"`
|
||||
NetworkOverrides []networkOverride `json:"network_overrides,omitempty"`
|
||||
}
|
||||
|
||||
// loadSnapshotWithNetworkOverride calls PUT /snapshot/load on the Firecracker
|
||||
// Unix socket, remapping the first network interface to tapName.
|
||||
// This bypasses the SDK's LoadSnapshotHandler which doesn't expose
|
||||
// network_overrides (added in Firecracker v1.15, SDK v1.0.0 omits it).
|
||||
func loadSnapshotWithNetworkOverride(ctx context.Context, sockPath, memPath, vmstatePath, tapName string) error {
|
||||
payload := snapshotLoadRequest{
|
||||
MemFilePath: memPath,
|
||||
SnapshotPath: vmstatePath,
|
||||
ResumeVM: true,
|
||||
NetworkOverrides: []networkOverride{
|
||||
{IfaceID: "1", HostDevName: tapName},
|
||||
},
|
||||
}
|
||||
|
||||
data, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal snapshot load params: %w", err)
|
||||
}
|
||||
|
||||
httpClient := &http.Client{
|
||||
Transport: &http.Transport{
|
||||
DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
|
||||
return net.Dial("unix", sockPath)
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut,
|
||||
"http://localhost/snapshot/load", bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return fmt.Errorf("build snapshot load request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("snapshot load request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusNoContent {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("snapshot load failed (%d): %s", resp.StatusCode, body)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
298
orchestrator/web/terminal.html
Normal file
298
orchestrator/web/terminal.html
Normal file
@@ -0,0 +1,298 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>fc-orch console</title>
|
||||
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/xterm@5/css/xterm.css"/>
|
||||
<script src="https://cdn.jsdelivr.net/npm/xterm@5/lib/xterm.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/xterm-addon-fit@0.8/lib/xterm-addon-fit.js"></script>
|
||||
<style>
|
||||
*, *::before, *::after { box-sizing: border-box; }
|
||||
html, body {
|
||||
margin: 0; padding: 0;
|
||||
width: 100%; height: 100%;
|
||||
background: #0d0d0d;
|
||||
font-family: monospace;
|
||||
color: #ccc;
|
||||
}
|
||||
|
||||
/* ── index / clone picker ── */
|
||||
#index {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
height: 100%;
|
||||
gap: 1rem;
|
||||
}
|
||||
#index h1 { margin: 0; font-size: 1.4rem; color: #8be; }
|
||||
|
||||
#clone-list {
|
||||
list-style: none;
|
||||
padding: 0; margin: 0;
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: .5rem;
|
||||
}
|
||||
.clone-entry {
|
||||
display: flex;
|
||||
align-items: stretch;
|
||||
border: 1px solid #444;
|
||||
border-radius: 4px;
|
||||
overflow: hidden;
|
||||
}
|
||||
.clone-entry a {
|
||||
display: block;
|
||||
padding: .4rem 1rem;
|
||||
color: #8be;
|
||||
text-decoration: none;
|
||||
transition: background .15s;
|
||||
}
|
||||
.clone-entry a:hover { background: #1e2e3e; }
|
||||
.clone-entry button.destroy {
|
||||
padding: .4rem .6rem;
|
||||
background: transparent;
|
||||
border: none;
|
||||
border-left: 1px solid #444;
|
||||
color: #c44;
|
||||
cursor: pointer;
|
||||
font-size: .8rem;
|
||||
transition: background .15s;
|
||||
}
|
||||
.clone-entry button.destroy:hover { background: #2a1a1a; }
|
||||
.clone-entry button.destroy:disabled { color: #555; cursor: default; }
|
||||
|
||||
#index .none { color: #666; font-size: .9rem; }
|
||||
|
||||
#spawn-btn {
|
||||
padding: .45rem 1.2rem;
|
||||
background: #1a2e1a;
|
||||
border: 1px solid #4c4;
|
||||
border-radius: 4px;
|
||||
color: #4c4;
|
||||
cursor: pointer;
|
||||
font-family: monospace;
|
||||
font-size: .9rem;
|
||||
transition: background .15s, opacity .15s;
|
||||
}
|
||||
#spawn-btn:hover:not(:disabled) { background: #243e24; }
|
||||
#spawn-btn:disabled { opacity: .5; cursor: default; }
|
||||
|
||||
#error-msg {
|
||||
color: #c44;
|
||||
font-size: .85rem;
|
||||
display: none;
|
||||
}
|
||||
|
||||
/* ── terminal view ── */
|
||||
#terminal-wrap {
|
||||
display: none;
|
||||
flex-direction: column;
|
||||
height: 100%;
|
||||
}
|
||||
#topbar {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: .75rem;
|
||||
padding: .35rem .75rem;
|
||||
background: #111;
|
||||
border-bottom: 1px solid #222;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
#topbar .title { font-size: .85rem; color: #8be; }
|
||||
#status {
|
||||
font-size: .75rem;
|
||||
padding: .15rem .5rem;
|
||||
border-radius: 3px;
|
||||
background: #1a2a1a;
|
||||
color: #4c4;
|
||||
}
|
||||
#status.disconnected { background: #2a1a1a; color: #c44; }
|
||||
#terminal-container { flex: 1; overflow: hidden; padding: 4px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<!-- clone picker (shown when no ?id= param) -->
|
||||
<div id="index">
|
||||
<h1>fc-orch console</h1>
|
||||
<ul id="clone-list"></ul>
|
||||
<p class="none" id="no-clones" style="display:none">No running clones.</p>
|
||||
<button id="spawn-btn">+ Spawn clone</button>
|
||||
<p id="error-msg"></p>
|
||||
</div>
|
||||
|
||||
<!-- terminal (shown when ?id=N) -->
|
||||
<div id="terminal-wrap">
|
||||
<div id="topbar">
|
||||
<span class="title" id="topbar-title"></span>
|
||||
<span id="status">connecting…</span>
|
||||
<a href="/" style="margin-left:auto;font-size:.75rem;color:#666;text-decoration:none">← all clones</a>
|
||||
</div>
|
||||
<div id="terminal-container"></div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
(function () {
|
||||
const params = new URLSearchParams(location.search);
|
||||
const id = params.get('id');
|
||||
|
||||
// ── index view ──────────────────────────────────────────────
|
||||
if (!id) {
|
||||
const ul = document.getElementById('clone-list');
|
||||
const noneEl = document.getElementById('no-clones');
|
||||
const spawnBtn = document.getElementById('spawn-btn');
|
||||
const errEl = document.getElementById('error-msg');
|
||||
|
||||
function showError(msg) {
|
||||
errEl.textContent = msg;
|
||||
errEl.style.display = '';
|
||||
}
|
||||
function clearError() {
|
||||
errEl.style.display = 'none';
|
||||
}
|
||||
|
||||
function addCloneEntry(c) {
|
||||
noneEl.style.display = 'none';
|
||||
const li = document.createElement('li');
|
||||
li.className = 'clone-entry';
|
||||
li.dataset.id = c;
|
||||
li.innerHTML =
|
||||
`<a href="/?id=${c}">clone ${c}</a>` +
|
||||
`<button class="destroy" title="Destroy clone ${c}">✕</button>`;
|
||||
li.querySelector('.destroy').addEventListener('click', () => destroyClone(c, li));
|
||||
ul.appendChild(li);
|
||||
}
|
||||
|
||||
function refreshList() {
|
||||
fetch('/clones')
|
||||
.then(r => r.json())
|
||||
.then(clones => {
|
||||
ul.innerHTML = '';
|
||||
if (!clones || clones.length === 0) {
|
||||
noneEl.style.display = '';
|
||||
return;
|
||||
}
|
||||
noneEl.style.display = 'none';
|
||||
clones.forEach(addCloneEntry);
|
||||
})
|
||||
.catch(() => { noneEl.style.display = ''; });
|
||||
}
|
||||
|
||||
function destroyClone(cid, li) {
|
||||
const btn = li.querySelector('.destroy');
|
||||
btn.disabled = true;
|
||||
clearError();
|
||||
fetch(`/clones/${cid}`, { method: 'DELETE' })
|
||||
.then(r => {
|
||||
if (!r.ok) return r.text().then(t => { throw new Error(t); });
|
||||
li.remove();
|
||||
if (ul.children.length === 0) noneEl.style.display = '';
|
||||
})
|
||||
.catch(e => {
|
||||
btn.disabled = false;
|
||||
showError(`destroy failed: ${e.message}`);
|
||||
});
|
||||
}
|
||||
|
||||
spawnBtn.addEventListener('click', () => {
|
||||
spawnBtn.disabled = true;
|
||||
spawnBtn.textContent = 'Spawning…';
|
||||
clearError();
|
||||
fetch('/clones', { method: 'POST' })
|
||||
.then(r => {
|
||||
if (!r.ok) return r.text().then(t => { throw new Error(t); });
|
||||
return r.json();
|
||||
})
|
||||
.then(data => {
|
||||
location.href = `/?id=${data.id}`;
|
||||
})
|
||||
.catch(e => {
|
||||
spawnBtn.disabled = false;
|
||||
spawnBtn.textContent = '+ Spawn clone';
|
||||
showError(`spawn failed: ${e.message}`);
|
||||
});
|
||||
});
|
||||
|
||||
refreshList();
|
||||
return;
|
||||
}
|
||||
|
||||
// ── terminal view ────────────────────────────────────────────
|
||||
document.getElementById('index').style.display = 'none';
|
||||
const wrap = document.getElementById('terminal-wrap');
|
||||
wrap.style.display = 'flex';
|
||||
document.getElementById('topbar-title').textContent = `clone ${id}`;
|
||||
document.title = `clone ${id} — fc-orch`;
|
||||
|
||||
const term = new Terminal({
|
||||
cursorBlink: true,
|
||||
scrollback: 5000,
|
||||
theme: {
|
||||
background: '#0d0d0d',
|
||||
foreground: '#d0d0d0',
|
||||
cursor: '#8be',
|
||||
selectionBackground: '#2a4a6a',
|
||||
},
|
||||
});
|
||||
const fitAddon = new FitAddon.FitAddon();
|
||||
term.loadAddon(fitAddon);
|
||||
term.open(document.getElementById('terminal-container'));
|
||||
fitAddon.fit();
|
||||
|
||||
const statusEl = document.getElementById('status');
|
||||
function setStatus(text, ok) {
|
||||
statusEl.textContent = text;
|
||||
statusEl.className = ok ? '' : 'disconnected';
|
||||
}
|
||||
|
||||
const proto = location.protocol === 'https:' ? 'wss:' : 'ws:';
|
||||
const ws = new WebSocket(`${proto}//${location.host}/ws/${id}`);
|
||||
ws.binaryType = 'arraybuffer';
|
||||
|
||||
ws.onopen = () => {
|
||||
setStatus('connected', true);
|
||||
sendResize();
|
||||
term.focus();
|
||||
};
|
||||
|
||||
ws.onmessage = e => {
|
||||
if (e.data instanceof ArrayBuffer) {
|
||||
term.write(new Uint8Array(e.data));
|
||||
}
|
||||
};
|
||||
|
||||
ws.onclose = () => {
|
||||
setStatus('disconnected', false);
|
||||
term.write('\r\n\x1b[31m[connection closed]\x1b[0m\r\n');
|
||||
};
|
||||
|
||||
ws.onerror = () => setStatus('error', false);
|
||||
|
||||
// Keystrokes → VM (binary frame so the server can distinguish from resize)
|
||||
term.onData(data => {
|
||||
if (ws.readyState === WebSocket.OPEN) {
|
||||
const bytes = new TextEncoder().encode(data);
|
||||
ws.send(bytes.buffer);
|
||||
}
|
||||
});
|
||||
|
||||
// Resize handling
|
||||
function sendResize() {
|
||||
if (ws.readyState === WebSocket.OPEN) {
|
||||
ws.send(JSON.stringify({ rows: term.rows, cols: term.cols }));
|
||||
}
|
||||
}
|
||||
|
||||
term.onResize(() => sendResize());
|
||||
|
||||
let resizeTimer;
|
||||
window.addEventListener('resize', () => {
|
||||
clearTimeout(resizeTimer);
|
||||
resizeTimer = setTimeout(() => { fitAddon.fit(); }, 50);
|
||||
});
|
||||
}());
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
Reference in New Issue
Block a user