Add the mock provider with a real CONNECT proxy per port (Step 3)

An in-memory provider.Provider whose state (Provisioning -> Running ->
Terminated -> purged) is a pure function of an injectable clock, not
background timers, so it's deterministic under tests and correct under
real time with no goroutine lifecycle to leak.

Once an instance is observed Running, it lazily acquires a real HTTP
CONNECT proxy listener so the health engine's through-the-proxy probe
(later steps) genuinely tunnels a request end to end, instead of the
healthcheck being simulated or bypassed for local development.

Redesigned the listener sharing model from what the plan assumed: the
plan's "one loopback IP per instance" doesn't work on macOS (only
127.0.0.1 binds without a privileged ifconfig alias, unlike Linux where
the whole 127.0.0.0/8 routes to loopback by default), and there's no
channel for a provider to report a port back to the reconciler anyway
(EffectivePort() is spec-only). Instances now share one real listener
per port, reference-counted at the package level rather than per
Provider instance, since a bound TCP port is a genuinely process-global
OS resource -- two separately configured mock-typed provider entries
must not both try to bind the same default port.

Fault injection wired both ways: MockConfig.FailNextCreates/FailWith for
demos, InjectCreateFailures(n, class) for tests. Create is idempotent by
name.

Caught and fixed a real test flake (not a logic bug): the freePort test
helper asked the OS for a free port via bind-then-close, a TOCTOU race
under t.Parallel() that let two tests collide on the same "free" port.
Replaced it with a monotonic counter, since these tests only need
uniqueness within the test run.

internal/provider/mock at 91.1% coverage, including an end-to-end test
that opens real sockets: Create -> Get past provisionDelay -> a real
http.Client tunnelling a CONNECT through the mock to a real TLS origin.
make test green across the whole repo.

Co-Authored-By: Claude <noreply@anthropic.com>

.claude/settings.json

@@ -44,7 +44,8 @@
       "Bash(cd /Users/jan.novak/srv/go/egress-proxies-operator *)",
       "Bash(echo \"build: $?\")",
       "Bash(echo \"vet: $?\")",
-      "Bash(perl -i -pe 's{^\\\\t\\\\t\\\\t\\\\t\\\\t// TODO\\\\\\(user\\\\\\): Specify other spec details if needed\\\\.\\\\n}{\\\\t\\\\t\\\\t\\\\t\\\\t// A minimal, schema-valid spec so this placeholder test survives the\\\\n\\\\t\\\\t\\\\t\\\\t\\\\t// CEL/CRD validation added in Step 1. Rewritten wholesale in Step 4\\\\n\\\\t\\\\t\\\\t\\\\t\\\\t// alongside the real reconciler and envtest suite.\\\\n\\\\t\\\\t\\\\t\\\\t\\\\tSpec: crawlv1alpha1.ProxySpec{\\\\n\\\\t\\\\t\\\\t\\\\t\\\\t\\\\tMode:     crawlv1alpha1.ModeExternal,\\\\n\\\\t\\\\t\\\\t\\\\t\\\\t\\\\tEndpoint: &crawlv1alpha1.EndpointSpec{Host: \"10.0.0.1\"},\\\\n\\\\t\\\\t\\\\t\\\\t\\\\t},\\\\n}' internal/controller/proxy_controller_test.go)"
+      "Bash(perl -i -pe 's{^\\\\t\\\\t\\\\t\\\\t\\\\t// TODO\\\\\\(user\\\\\\): Specify other spec details if needed\\\\.\\\\n}{\\\\t\\\\t\\\\t\\\\t\\\\t// A minimal, schema-valid spec so this placeholder test survives the\\\\n\\\\t\\\\t\\\\t\\\\t\\\\t// CEL/CRD validation added in Step 1. Rewritten wholesale in Step 4\\\\n\\\\t\\\\t\\\\t\\\\t\\\\t// alongside the real reconciler and envtest suite.\\\\n\\\\t\\\\t\\\\t\\\\t\\\\tSpec: crawlv1alpha1.ProxySpec{\\\\n\\\\t\\\\t\\\\t\\\\t\\\\t\\\\tMode:     crawlv1alpha1.ModeExternal,\\\\n\\\\t\\\\t\\\\t\\\\t\\\\t\\\\tEndpoint: &crawlv1alpha1.EndpointSpec{Host: \"10.0.0.1\"},\\\\n\\\\t\\\\t\\\\t\\\\t\\\\t},\\\\n}' internal/controller/proxy_controller_test.go)",
+      "Bash(go tool *)"
     ],
     "additionalDirectories": [
       "/Users/jan.novak/srv/go/egress-proxies-operator/.claude",

docs/plans-executions/2026-08-07-1747-proxy-operator.md

@@ -7,7 +7,7 @@ Pairs with [docs/plans/2026-08-07-1747-proxy-operator.md](../plans/2026-08-07-17
 - [x] Step 0 — Branch and scaffold
 - [x] Step 1 — API types (`api/v1alpha1/proxy_types.go`)
 - [x] Step 2 — Provider contract (`internal/provider/`)
-- [ ] Step 3 — Mock provider (`internal/provider/mock/`)
+- [x] Step 3 — Mock provider (`internal/provider/mock/`)
 - [ ] Step 4 — Reconciler (`internal/controller/`)
 - [ ] Step 5 — Health engine (`internal/health/`)
 - [ ] Step 6 — Lease store (`internal/lease/`)
@@ -199,3 +199,107 @@ built in (rejects unknown fields, which is what "fail fast on unknown type"
 in the plan actually needs) and was already pulled in transitively by the
 k8s.io toolchain, so no new dependency was added — `go mod tidy` just
 promoted it from indirect to direct.
+
+## Step 3 — Mock provider (`internal/provider/mock/`)
+
+Started from the plan's design (state as a pure function of an injectable
+clock, no background timers) but had to redesign the "real proxy" part
+before writing any code, once a load-bearing assumption turned out false.
+
+The plan (and the earlier decision to make the mock run a real proxy)
+assumed each instance could get its own loopback address —
+`127.0.0.1:<port>` per instance, "a fake IP from a private range." Verified
+that assumption directly before committing to it:
+
+```bash
+cat <<'EOF' > /tmp/loopbacktest.go
+package main
+import ("fmt"; "net")
+func main() {
+	for _, addr := range []string{"127.0.0.2:0", "127.0.0.55:0", "127.1.2.3:0"} {
+		l, err := net.Listen("tcp", addr)
+		if err != nil { fmt.Printf("%s: FAIL: %v\n", addr, err); continue }
+		fmt.Printf("%s: OK\n", addr)
+		l.Close()
+	}
+}
+EOF
+go run /tmp/loopbacktest.go
+# 127.0.0.2:0: FAIL: listen tcp 127.0.0.2:0: bind: can't assign requested address
+# 127.0.0.55:0: FAIL: listen tcp 127.0.0.55:0: bind: can't assign requested address
+# 127.1.2.3:0: FAIL: listen tcp 127.1.2.3:0: bind: can't assign requested address
+```
+
+Only `127.0.0.1` binds on macOS without `sudo ifconfig lo0 alias ... up` —
+Linux routes the whole `127.0.0.0/8` block to loopback by default, macOS
+doesn't. That's not something the operator can or should do at runtime, so
+per-instance loopback IPs were out. There's also a second problem the
+per-instance-IP design didn't solve anyway: `EffectivePort()`
+(`api/v1alpha1/helpers.go`, Step 1) is computed purely from `spec.port`
+with no channel for a provider to report back a different *port* — so
+whatever a mock instance actually listens on has to be the literal port the
+reconciler will pass through `CreateRequest.Port`, not an OS-assigned
+ephemeral one.
+
+Redesigned around one real listener **per port**, shared and
+reference-counted across every instance that uses it, rather than one
+listener per instance (`proxy.go`, `sharedProxies`). This fixes both
+problems at once: every instance binds the same `127.0.0.1` (no OS issue),
+and any number of instances can share a port without conflict since it's
+the exact same underlying listener. Deliberately made the refcounting
+**package-level**, not a field on `mock.Provider`, because a bound TCP port
+is a genuinely process-global OS resource — two separately configured
+mock-typed provider entries (e.g. two named `"mock"` instances in
+`providers-config.yaml`) would otherwise both try to bind the same default
+port and the second one would just fail. This is a case where a package
+global is the correct model, not a shortcut: it mirrors an OS-level
+singleton, not application state.
+
+Instance lifecycle otherwise follows the plan exactly: `Get`/`ListByTag`
+derive `Provisioning → Running → Terminated → purged (ErrNotFound)` from
+`createdAt`/`deletedAt` compared against an injectable clock, with the real
+proxy listener acquired lazily on the first observed `Running` and released
+on `Delete` (or lazily on purge, so orphaned records can't leak a
+reference). `Create` is idempotent by name. Fault injection is wired both
+ways per the plan: `MockConfig.FailNextCreates`/`FailWith` for the demo
+config, `InjectCreateFailures(n, class)` for tests.
+
+Tests initially had a real flake, caught by running with `-race -count=3`
+rather than trusting one green run:
+
+```bash
+go test -race -v ./internal/provider/mock/... 2>&1 | tail -5
+# --- FAIL: TestAcquireProxy_sharedAcrossAcquires
+#     proxy_test.go:37: port not released after last reference: bind: address already in use
+```
+
+Root cause wasn't the refcounting logic — it was the test helper. `freePort`
+asked the OS for a free port by binding to `:0` and immediately closing it,
+which is a classic TOCTOU race under `t.Parallel()`: two tests can be handed
+the same "free" port before either actually claims it, since nothing holds
+it open in between. Fixed by replacing the OS-asks approach with a
+monotonic counter (`20000 + atomic.Int32`) — these tests only need a port
+unique *within this test run*, not one verified free by the OS at an
+instant in time, so guaranteeing uniqueness outright is both simpler and
+correct where the "ask and hope" approach wasn't. Reran `-race -count=3`
+clean afterward.
+
+Added tests beyond the plan's list to close real coverage gaps rather than
+stopping at "green": config-override branches in `New`, all four
+`failClassFromString` branches, `ListByTag`'s purge-on-list and
+Running/IP-inclusion paths, and a plain-`http://` forwarding test
+(`handleForward`) alongside the CONNECT one, since a `probeURL` override
+could use either scheme. Landed at 91.1% coverage; the remainder is
+OS-failure branches (bind errors, hijack failures) not worth simulating for
+a prototype.
+
+The `TestProvider_realProxyTunnelsConnect` test is the one that actually
+matters most here: it opens real sockets end to end — mock `Create` →
+`Get` past `provisionDelay` → real `http.Client` with
+`Transport.Proxy` dialing through the mock's CONNECT tunnel to a real
+`httptest.NewTLSServer` — and gets a real `204` back. That's the concrete
+proof the "mock runs a real proxy" decision actually delivers a genuine
+end-to-end healthcheck, not a simulated one.
+
+`internal/provider/mock` at 91.1% coverage. `make test` green across the
+whole repo.

internal/provider/mock/mock.go

@@ -0,0 +1,257 @@
+// Package mock is an in-memory provider.Provider for local development and
+// tests. State transitions are a pure function of an injectable clock, not
+// background timers, so behavior is deterministic under a fake clock and
+// there is no goroutine lifecycle for provisioning/deletion to leak.
+//
+// Once an instance's state resolves to Running, the provider starts (or
+// joins) a real, minimal HTTP CONNECT proxy listener — see proxy.go — so
+// that a through-the-proxy healthcheck (internal/health) genuinely
+// succeeds against it. That's the difference between this being a true
+// end-to-end local demo and one that quietly bypasses the operator's core
+// mechanism.
+package mock
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+const (
+	defaultProvisionDelay = 5 * time.Second
+	defaultDeleteDelay    = 1 * time.Second
+)
+
+// Provider is a thread-safe, in-memory provider.Provider.
+type Provider struct {
+	mu   sync.Mutex
+	name string
+	now  func() time.Time
+
+	provisionDelay time.Duration
+	deleteDelay    time.Duration
+
+	instances map[string]*record // keyed by instance name (== providerID)
+
+	failNext  int
+	failClass error
+}
+
+type record struct {
+	uid       string
+	port      int32
+	createdAt time.Time
+	deletedAt time.Time // zero == not deleted
+
+	proxyAcquired bool
+	releaseProxy  func()
+}
+
+// New builds a mock Provider from its config block. Satisfies
+// registry.Constructor.
+func New(_ context.Context, cfg provider.ProviderConfig) (provider.Provider, error) {
+	p := &Provider{
+		name:           cfg.Name,
+		now:            time.Now,
+		provisionDelay: defaultProvisionDelay,
+		deleteDelay:    defaultDeleteDelay,
+		instances:      make(map[string]*record),
+	}
+	if cfg.Mock == nil {
+		return p, nil
+	}
+	if cfg.Mock.ProvisionDelaySeconds > 0 {
+		p.provisionDelay = time.Duration(cfg.Mock.ProvisionDelaySeconds) * time.Second
+	}
+	if cfg.Mock.DeleteDelaySeconds > 0 {
+		p.deleteDelay = time.Duration(cfg.Mock.DeleteDelaySeconds) * time.Second
+	}
+	if cfg.Mock.FailNextCreates > 0 {
+		class, err := failClassFromString(cfg.Mock.FailWith)
+		if err != nil {
+			return nil, fmt.Errorf("mock provider %q: %w", cfg.Name, err)
+		}
+		p.failNext = cfg.Mock.FailNextCreates
+		p.failClass = class
+	}
+	return p, nil
+}
+
+func failClassFromString(s string) (error, error) {
+	switch s {
+	case "", provider.FailWithTransient:
+		return provider.ErrTransient, nil
+	case provider.FailWithNotFound:
+		return provider.ErrNotFound, nil
+	case provider.FailWithQuota:
+		return provider.ErrQuotaExceeded, nil
+	case provider.FailWithPermanent:
+		return provider.ErrPermanent, nil
+	default:
+		return nil, fmt.Errorf("unknown failWith %q", s)
+	}
+}
+
+// InjectCreateFailures makes the next n calls to Create fail, each
+// returning an error classified as class. Exported for tests exercising
+// the reconciler's error handling; config-driven FailNextCreates/FailWith
+// (config.go) drives the same mechanism for demos.
+func (p *Provider) InjectCreateFailures(n int, class error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.failNext = n
+	p.failClass = class
+}
+
+// Create is idempotent by req.Name: a repeat call for an existing,
+// non-deleted instance returns its existing name rather than creating a
+// duplicate. This is what lets the reconciler recover cleanly if it
+// crashes between calling Create and persisting providerID — the next
+// reconcile's Create call finds the same instance by its deterministic
+// name.
+func (p *Provider) Create(_ context.Context, req provider.CreateRequest) (string, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	if existing, ok := p.instances[req.Name]; ok && existing.deletedAt.IsZero() {
+		return req.Name, nil
+	}
+
+	if p.failNext > 0 {
+		p.failNext--
+		return "", provider.Wrap(p.failClass, "create", p.name, req.Name, nil)
+	}
+
+	p.instances[req.Name] = &record{
+		uid:       req.UID,
+		port:      req.Port,
+		createdAt: p.now(),
+	}
+	return req.Name, nil
+}
+
+// Get returns the current state of a previously created instance,
+// deriving it from the record's timestamps against the provider's clock.
+// The first Get to observe a Running instance lazily acquires its real
+// proxy listener.
+func (p *Provider) Get(_ context.Context, providerID string) (*provider.Instance, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	rec, ok := p.instances[providerID]
+	if !ok {
+		return nil, provider.Wrap(provider.ErrNotFound, "get", p.name, providerID, nil)
+	}
+
+	state, purge := p.stateLocked(rec)
+	if purge {
+		p.releaseLocked(rec)
+		delete(p.instances, providerID)
+		return nil, provider.Wrap(provider.ErrNotFound, "get", p.name, providerID, nil)
+	}
+
+	inst := &provider.Instance{
+		ID:        providerID,
+		State:     state,
+		UID:       rec.uid,
+		CreatedAt: rec.createdAt,
+	}
+
+	if state == provider.StateRunning {
+		if !rec.proxyAcquired {
+			release, err := acquireProxy(rec.port)
+			if err != nil {
+				return nil, provider.Wrap(provider.ErrTransient, "get", p.name, providerID, err)
+			}
+			rec.releaseProxy = release
+			rec.proxyAcquired = true
+		}
+		inst.IP = mockProxyIP
+	}
+
+	return inst, nil
+}
+
+// Delete is idempotent: deleting an unknown or already-deleted instance is
+// not an error. The real proxy listener (if any) is released immediately;
+// the record itself lingers, reporting Terminated, until deleteDelay has
+// passed and a later Get/ListByTag purges it — mirroring a real provider
+// where the API stops accepting the ID before the resource fully vanishes.
+func (p *Provider) Delete(_ context.Context, providerID string) error {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	rec, ok := p.instances[providerID]
+	if !ok {
+		return nil
+	}
+	if rec.deletedAt.IsZero() {
+		rec.deletedAt = p.now()
+	}
+	p.releaseLocked(rec)
+	return nil
+}
+
+// ListByTag returns every non-purged instance, mirroring what a real
+// provider's tag/label-filtered list call would return. Also lazily purges
+// (and releases) any instance whose deleteDelay has elapsed, since orphan
+// GC — the only caller that runs regardless of whether anyone still calls
+// Get on a given proxy — is what's responsible for eventually reclaiming
+// deleted-and-expired instances in a real fleet.
+func (p *Provider) ListByTag(_ context.Context) ([]provider.Instance, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	var out []provider.Instance
+	for id, rec := range p.instances {
+		state, purge := p.stateLocked(rec)
+		if purge {
+			p.releaseLocked(rec)
+			delete(p.instances, id)
+			continue
+		}
+		inst := provider.Instance{
+			ID:        id,
+			State:     state,
+			UID:       rec.uid,
+			CreatedAt: rec.createdAt,
+		}
+		if state == provider.StateRunning && rec.proxyAcquired {
+			inst.IP = mockProxyIP
+		}
+		out = append(out, inst)
+	}
+	return out, nil
+}
+
+// stateLocked derives state from rec's timestamps against p.now(). Must be
+// called with p.mu held. purge is true once the instance has been deleted
+// long enough that it should behave as fully gone (ErrNotFound to Get,
+// absent from ListByTag).
+func (p *Provider) stateLocked(rec *record) (state provider.InstanceState, purge bool) {
+	now := p.now()
+	if !rec.deletedAt.IsZero() {
+		if now.Sub(rec.deletedAt) < p.deleteDelay {
+			return provider.StateTerminated, false
+		}
+		return "", true
+	}
+	if now.Sub(rec.createdAt) < p.provisionDelay {
+		return provider.StateProvisioning, false
+	}
+	return provider.StateRunning, false
+}
+
+// releaseLocked releases rec's proxy listener reference, if it holds one.
+// Must be called with p.mu held; acquireProxy/release use a separate lock
+// (sharedProxies.mu), so this never deadlocks against it.
+func (p *Provider) releaseLocked(rec *record) {
+	if rec.proxyAcquired {
+		rec.releaseProxy()
+		rec.proxyAcquired = false
+		rec.releaseProxy = nil
+	}
+}

internal/provider/mock/mock_test.go

@@ -0,0 +1,430 @@
+package mock
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strconv"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"gitea.home.hrajfrisbee.cz/kacerr/egress-proxies-operator/internal/provider"
+)
+
+// fakeClock lets tests drive Provider's internal state machine
+// deterministically instead of racing real time.
+type fakeClock struct {
+	mu  sync.Mutex
+	now time.Time
+}
+
+func newFakeClock() *fakeClock {
+	return &fakeClock{now: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)}
+}
+
+func (c *fakeClock) Now() time.Time {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.now
+}
+
+func (c *fakeClock) Advance(d time.Duration) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.now = c.now.Add(d)
+}
+
+func newTestProvider(clock *fakeClock) *Provider {
+	return &Provider{
+		name:           "test",
+		now:            clock.Now,
+		provisionDelay: 5 * time.Second,
+		deleteDelay:    1 * time.Second,
+		instances:      make(map[string]*record),
+	}
+}
+
+// testPortCounter hands out distinct ports across this package's test run.
+// A "bind to :0, read back the port, close it" approach looks more
+// realistic but is a TOCTOU race under t.Parallel(): two tests can be
+// handed the same "free" port before either actually claims it, since
+// nothing holds it open in between. What these tests actually need is a
+// port no *other test in this run* will also try — a monotonic counter
+// guarantees that outright, at the acceptable cost of a (much rarer) clash
+// with an unrelated process already using something in this range.
+var testPortCounter atomic.Int32
+
+func freePort(t *testing.T) int32 {
+	t.Helper()
+	return 20000 + testPortCounter.Add(1)
+}
+
+func TestProvider_Create_idempotent(t *testing.T) {
+	t.Parallel()
+	p := newTestProvider(newFakeClock())
+	ctx := context.Background()
+	req := provider.CreateRequest{Name: "proxy-abc", UID: "uid-1", Port: freePort(t)}
+
+	id1, err := p.Create(ctx, req)
+	if err != nil {
+		t.Fatalf("Create() error = %v", err)
+	}
+	id2, err := p.Create(ctx, req)
+	if err != nil {
+		t.Fatalf("Create() (repeat) error = %v", err)
+	}
+	if id1 != id2 {
+		t.Errorf("Create() not idempotent: %q != %q", id1, id2)
+	}
+
+	instances, err := p.ListByTag(ctx)
+	if err != nil {
+		t.Fatalf("ListByTag() error = %v", err)
+	}
+	if len(instances) != 1 {
+		t.Errorf("len(instances) = %d, want 1 (repeat Create must not duplicate)", len(instances))
+	}
+}
+
+func TestProvider_Get_notFound(t *testing.T) {
+	t.Parallel()
+	p := newTestProvider(newFakeClock())
+	_, err := p.Get(context.Background(), "does-not-exist")
+	if !errors.Is(err, provider.ErrNotFound) {
+		t.Errorf("Get() error = %v, want ErrNotFound", err)
+	}
+}
+
+func TestProvider_stateTransitions(t *testing.T) {
+	t.Parallel()
+	clock := newFakeClock()
+	p := newTestProvider(clock)
+	ctx := context.Background()
+	port := freePort(t)
+
+	id, err := p.Create(ctx, provider.CreateRequest{Name: "proxy-xyz", UID: "uid-2", Port: port})
+	if err != nil {
+		t.Fatalf("Create() error = %v", err)
+	}
+
+	inst, err := p.Get(ctx, id)
+	if err != nil {
+		t.Fatalf("Get() error = %v", err)
+	}
+	if inst.State != provider.StateProvisioning {
+		t.Errorf("State = %v, want Provisioning immediately after Create", inst.State)
+	}
+	if inst.IP != "" {
+		t.Errorf("IP = %q, want empty while Provisioning", inst.IP)
+	}
+
+	clock.Advance(5*time.Second + time.Millisecond)
+	inst, err = p.Get(ctx, id)
+	if err != nil {
+		t.Fatalf("Get() error = %v", err)
+	}
+	if inst.State != provider.StateRunning {
+		t.Errorf("State = %v, want Running after provisionDelay", inst.State)
+	}
+	if inst.IP == "" {
+		t.Error("IP is empty, want a real address once Running")
+	}
+	if inst.UID != "uid-2" {
+		t.Errorf("UID = %q, want %q", inst.UID, "uid-2")
+	}
+
+	if err := p.Delete(ctx, id); err != nil {
+		t.Fatalf("Delete() error = %v", err)
+	}
+	inst, err = p.Get(ctx, id)
+	if err != nil {
+		t.Fatalf("Get() error = %v", err)
+	}
+	if inst.State != provider.StateTerminated {
+		t.Errorf("State = %v, want Terminated immediately after Delete", inst.State)
+	}
+
+	clock.Advance(time.Second + time.Millisecond)
+	_, err = p.Get(ctx, id)
+	if !errors.Is(err, provider.ErrNotFound) {
+		t.Errorf("Get() error = %v, want ErrNotFound after deleteDelay", err)
+	}
+}
+
+func TestProvider_Delete_idempotent(t *testing.T) {
+	t.Parallel()
+	p := newTestProvider(newFakeClock())
+	ctx := context.Background()
+	if err := p.Delete(ctx, "never-existed"); err != nil {
+		t.Errorf("Delete() on unknown ID error = %v, want nil", err)
+	}
+
+	id, _ := p.Create(ctx, provider.CreateRequest{Name: "proxy-del", UID: "uid-3", Port: freePort(t)})
+	if err := p.Delete(ctx, id); err != nil {
+		t.Fatalf("Delete() error = %v", err)
+	}
+	if err := p.Delete(ctx, id); err != nil {
+		t.Errorf("Delete() (repeat) error = %v, want nil", err)
+	}
+}
+
+func TestProvider_FaultInjection_configDriven(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+	prov, err := New(ctx, provider.ProviderConfig{
+		Name: "flaky",
+		Type: "mock",
+		Mock: &provider.MockConfig{FailNextCreates: 1, FailWith: provider.FailWithQuota},
+	})
+	if err != nil {
+		t.Fatalf("New() error = %v", err)
+	}
+
+	_, err = prov.Create(ctx, provider.CreateRequest{Name: "proxy-1", UID: "uid-4", Port: freePort(t)})
+	if !errors.Is(err, provider.ErrQuotaExceeded) {
+		t.Fatalf("first Create() error = %v, want ErrQuotaExceeded", err)
+	}
+
+	_, err = prov.Create(ctx, provider.CreateRequest{Name: "proxy-1", UID: "uid-4", Port: freePort(t)})
+	if err != nil {
+		t.Fatalf("second Create() error = %v, want nil (failure budget exhausted)", err)
+	}
+}
+
+func TestProvider_InjectCreateFailures(t *testing.T) {
+	t.Parallel()
+	p := newTestProvider(newFakeClock())
+	ctx := context.Background()
+	p.InjectCreateFailures(2, provider.ErrPermanent)
+
+	for i := range 2 {
+		_, err := p.Create(ctx, provider.CreateRequest{Name: "proxy-fail", UID: "uid-5", Port: freePort(t)})
+		if !errors.Is(err, provider.ErrPermanent) {
+			t.Fatalf("Create() #%d error = %v, want ErrPermanent", i, err)
+		}
+	}
+	_, err := p.Create(ctx, provider.CreateRequest{Name: "proxy-fail", UID: "uid-5", Port: freePort(t)})
+	if err != nil {
+		t.Fatalf("Create() after budget exhausted, error = %v, want nil", err)
+	}
+}
+
+func TestProvider_New_unknownFailWith(t *testing.T) {
+	t.Parallel()
+	_, err := New(context.Background(), provider.ProviderConfig{
+		Name: "bad",
+		Type: "mock",
+		Mock: &provider.MockConfig{FailNextCreates: 1, FailWith: "oops"},
+	})
+	if err == nil {
+		t.Fatal("New() error = nil, want error for unknown failWith")
+	}
+}
+
+func TestNew_appliesConfigOverrides(t *testing.T) {
+	t.Parallel()
+	prov, err := New(context.Background(), provider.ProviderConfig{
+		Name: "custom",
+		Type: "mock",
+		Mock: &provider.MockConfig{ProvisionDelaySeconds: 30, DeleteDelaySeconds: 10},
+	})
+	if err != nil {
+		t.Fatalf("New() error = %v", err)
+	}
+	p := prov.(*Provider)
+	if p.provisionDelay != 30*time.Second {
+		t.Errorf("provisionDelay = %v, want 30s", p.provisionDelay)
+	}
+	if p.deleteDelay != 10*time.Second {
+		t.Errorf("deleteDelay = %v, want 10s", p.deleteDelay)
+	}
+}
+
+func TestFailClassFromString(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		in      string
+		want    error
+		wantErr bool
+	}{
+		{in: "", want: provider.ErrTransient},
+		{in: provider.FailWithTransient, want: provider.ErrTransient},
+		{in: provider.FailWithNotFound, want: provider.ErrNotFound},
+		{in: provider.FailWithQuota, want: provider.ErrQuotaExceeded},
+		{in: provider.FailWithPermanent, want: provider.ErrPermanent},
+		{in: "bogus", wantErr: true},
+	}
+	for _, tc := range tests {
+		t.Run(tc.in, func(t *testing.T) {
+			t.Parallel()
+			got, err := failClassFromString(tc.in)
+			if tc.wantErr {
+				if err == nil {
+					t.Fatal("failClassFromString() error = nil, want error")
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("failClassFromString() error = %v, want nil", err)
+			}
+			if got != tc.want {
+				t.Errorf("failClassFromString(%q) = %v, want %v", tc.in, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestProvider_ListByTag_includesRunningAndPurgesExpired(t *testing.T) {
+	t.Parallel()
+	clock := newFakeClock()
+	p := newTestProvider(clock)
+	ctx := context.Background()
+
+	running, err := p.Create(ctx, provider.CreateRequest{Name: "proxy-running", UID: "uid-running", Port: freePort(t)})
+	if err != nil {
+		t.Fatalf("Create() error = %v", err)
+	}
+	expiring, err := p.Create(ctx, provider.CreateRequest{Name: "proxy-expiring", UID: "uid-expiring", Port: freePort(t)})
+	if err != nil {
+		t.Fatalf("Create() error = %v", err)
+	}
+
+	clock.Advance(5*time.Second + time.Millisecond)
+	if _, err := p.Get(ctx, running); err != nil {
+		t.Fatalf("Get(running) error = %v", err)
+	}
+	if err := p.Delete(ctx, expiring); err != nil {
+		t.Fatalf("Delete(expiring) error = %v", err)
+	}
+	clock.Advance(time.Second + time.Millisecond) // past deleteDelay
+
+	instances, err := p.ListByTag(ctx)
+	if err != nil {
+		t.Fatalf("ListByTag() error = %v", err)
+	}
+	if len(instances) != 1 {
+		t.Fatalf("len(instances) = %d, want 1 (expired instance should be purged)", len(instances))
+	}
+	if instances[0].ID != running {
+		t.Errorf("instances[0].ID = %q, want %q", instances[0].ID, running)
+	}
+	if instances[0].State != provider.StateRunning {
+		t.Errorf("instances[0].State = %v, want Running", instances[0].State)
+	}
+	if instances[0].IP == "" {
+		t.Error("instances[0].IP is empty, want a real address for a Running instance")
+	}
+
+	if _, err := p.Get(ctx, expiring); !errors.Is(err, provider.ErrNotFound) {
+		t.Errorf("Get(expiring) error = %v, want ErrNotFound (ListByTag should have purged it)", err)
+	}
+}
+
+// TestProvider_realProxyForwardsPlainHTTP covers the non-CONNECT path: a
+// probeURL override using plain http:// instead of the default https://
+// should also work, going through handleForward rather than handleConnect.
+func TestProvider_realProxyForwardsPlainHTTP(t *testing.T) {
+	t.Parallel()
+	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer origin.Close()
+
+	clock := newFakeClock()
+	p := newTestProvider(clock)
+	ctx := context.Background()
+	port := freePort(t)
+
+	id, err := p.Create(ctx, provider.CreateRequest{Name: "proxy-forward", UID: "uid-forward", Port: port})
+	if err != nil {
+		t.Fatalf("Create() error = %v", err)
+	}
+	clock.Advance(5*time.Second + time.Millisecond)
+	inst, err := p.Get(ctx, id)
+	if err != nil {
+		t.Fatalf("Get() error = %v", err)
+	}
+
+	proxyURL, err := url.Parse("http://" + net.JoinHostPort(inst.IP, strconv.Itoa(int(port))))
+	if err != nil {
+		t.Fatalf("parsing proxy URL: %v", err)
+	}
+	client := &http.Client{
+		Transport: &http.Transport{Proxy: http.ProxyURL(proxyURL)},
+		Timeout:   5 * time.Second,
+	}
+
+	resp, err := client.Get(origin.URL)
+	if err != nil {
+		t.Fatalf("GET through mock proxy failed: %v", err)
+	}
+	defer resp.Body.Close()
+	_, _ = io.Copy(io.Discard, resp.Body)
+	if resp.StatusCode != http.StatusOK {
+		t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusOK)
+	}
+}
+
+// TestProvider_realProxyTunnelsConnect is the load-bearing test for this
+// package's whole reason to exist: once an instance is Running, a real
+// http.Client using it as an HTTP proxy must genuinely tunnel a CONNECT
+// request to a real origin — the exact mechanism internal/health depends
+// on. This is not simulated; it opens real sockets.
+func TestProvider_realProxyTunnelsConnect(t *testing.T) {
+	t.Parallel()
+	origin := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusNoContent)
+	}))
+	defer origin.Close()
+
+	clock := newFakeClock()
+	p := newTestProvider(clock)
+	ctx := context.Background()
+	port := freePort(t)
+
+	id, err := p.Create(ctx, provider.CreateRequest{Name: "proxy-e2e", UID: "uid-6", Port: port})
+	if err != nil {
+		t.Fatalf("Create() error = %v", err)
+	}
+	clock.Advance(5*time.Second + time.Millisecond)
+
+	inst, err := p.Get(ctx, id)
+	if err != nil {
+		t.Fatalf("Get() error = %v", err)
+	}
+	if inst.State != provider.StateRunning {
+		t.Fatalf("State = %v, want Running", inst.State)
+	}
+
+	proxyURL, err := url.Parse("http://" + net.JoinHostPort(inst.IP, strconv.Itoa(int(port))))
+	if err != nil {
+		t.Fatalf("parsing proxy URL: %v", err)
+	}
+	client := &http.Client{
+		Transport: &http.Transport{
+			Proxy: http.ProxyURL(proxyURL),
+			// origin is an httptest TLS server with a self-signed cert;
+			// this test is about the CONNECT tunnel, not certificate
+			// trust, so skip verification the same way httptest's own
+			// .Client() helper would.
+			TLSClientConfig: origin.Client().Transport.(*http.Transport).TLSClientConfig,
+		},
+		Timeout: 5 * time.Second,
+	}
+
+	resp, err := client.Get(origin.URL)
+	if err != nil {
+		t.Fatalf("GET through mock proxy failed: %v", err)
+	}
+	defer resp.Body.Close()
+	_, _ = io.Copy(io.Discard, resp.Body)
+	if resp.StatusCode != http.StatusNoContent {
+		t.Errorf("status = %d, want %d", resp.StatusCode, http.StatusNoContent)
+	}
+}

internal/provider/mock/proxy.go

@@ -0,0 +1,171 @@
+package mock
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"sync"
+	"time"
+)
+
+// mockProxyIP is the address every real listener binds to. Instances don't
+// get distinct addresses (unlike a real cloud provider): only 127.0.0.1 is
+// guaranteed bindable without elevated privileges across platforms — macOS
+// does not, by default, route the rest of 127.0.0.0/8 the way Linux does.
+// Instances sharing one port are told apart by which listener they share,
+// not by IP.
+const mockProxyIP = "127.0.0.1"
+
+// connectProxy is a minimal HTTP proxy: it tunnels CONNECT requests
+// (hijack + bidirectional copy) and forwards plain absolute-form HTTP
+// requests. It exists so the health engine's through-the-proxy probe
+// genuinely exercises a CONNECT tunnel against the mock provider, rather
+// than the healthcheck being simulated or bypassed for local development.
+type connectProxy struct {
+	ln net.Listener
+	sv *http.Server
+}
+
+func newConnectProxy(addr string) (*connectProxy, error) {
+	ln, err := net.Listen("tcp", addr)
+	if err != nil {
+		return nil, fmt.Errorf("mock proxy: listen %s: %w", addr, err)
+	}
+	sv := &http.Server{Handler: http.HandlerFunc(handleProxyRequest)}
+	go func() {
+		// Serve returns http.ErrServerClosed on a clean Close; there is no
+		// caller left to report anything else to by the time it returns.
+		_ = sv.Serve(ln)
+	}()
+	return &connectProxy{ln: ln, sv: sv}, nil
+}
+
+func (c *connectProxy) close() {
+	_ = c.sv.Close()
+}
+
+func handleProxyRequest(w http.ResponseWriter, r *http.Request) {
+	if r.Method == http.MethodConnect {
+		handleConnect(w, r)
+		return
+	}
+	handleForward(w, r)
+}
+
+// handleConnect implements the CONNECT tunnel: dial the real destination,
+// hijack the client connection, and splice the two together. This is the
+// exact mechanism the health engine's default https:// probe URL depends
+// on — a proxy that TCP-accepts but can't actually tunnel must fail here,
+// not succeed.
+func handleConnect(w http.ResponseWriter, r *http.Request) {
+	dst, err := net.DialTimeout("tcp", r.Host, 10*time.Second)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadGateway)
+		return
+	}
+	defer dst.Close()
+
+	hijacker, ok := w.(http.Hijacker)
+	if !ok {
+		http.Error(w, "hijack unsupported", http.StatusInternalServerError)
+		return
+	}
+	src, buf, err := hijacker.Hijack()
+	if err != nil {
+		return
+	}
+	defer src.Close()
+
+	if _, err := src.Write([]byte("HTTP/1.1 200 Connection Established\r\n\r\n")); err != nil {
+		return
+	}
+
+	// Any bytes the client already sent past the CONNECT request line
+	// before we hijacked are sitting in buf's reader; forward them before
+	// starting the raw splice loop.
+	if n := buf.Reader.Buffered(); n > 0 {
+		if _, err := io.CopyN(dst, buf.Reader, int64(n)); err != nil {
+			return
+		}
+	}
+
+	done := make(chan struct{}, 2)
+	go func() { io.Copy(dst, src); done <- struct{}{} }()
+	go func() { io.Copy(src, dst); done <- struct{}{} }()
+	<-done
+}
+
+// handleForward proxies a plain absolute-form HTTP request. CONNECT is the
+// path the health engine's default probe exercises, but a probeURL
+// override using plain http:// should work too.
+func handleForward(w http.ResponseWriter, r *http.Request) {
+	outReq := r.Clone(r.Context())
+	outReq.RequestURI = ""
+	resp, err := http.DefaultTransport.RoundTrip(outReq)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadGateway)
+		return
+	}
+	defer resp.Body.Close()
+	for k, vv := range resp.Header {
+		for _, v := range vv {
+			w.Header().Add(k, v)
+		}
+	}
+	w.WriteHeader(resp.StatusCode)
+	_, _ = io.Copy(w, resp.Body)
+}
+
+// sharedProxies tracks one real listener per port, reference-counted
+// across every mock.Provider in the process. This is package-level rather
+// than a field on Provider because a bound TCP port is a process-global OS
+// resource: two independently configured mock-typed provider entries (e.g.
+// two named "mock" instances in providers-config.yaml) must not both try
+// to bind 127.0.0.1:<port> — the second bind would simply fail. Sharing by
+// port, refcounted, means any number of instances across any number of
+// Provider values can use the same port safely, and the listener is torn
+// down once nothing needs it anymore.
+var sharedProxies = struct {
+	mu     sync.Mutex
+	byPort map[int32]*sharedProxyEntry
+}{byPort: make(map[int32]*sharedProxyEntry)}
+
+type sharedProxyEntry struct {
+	proxy *connectProxy
+	refs  int
+}
+
+// acquireProxy returns a release func for a real listener on mockProxyIP:
+// port, starting one if this is the first acquire for that port. Safe to
+// call concurrently; each returned release func must be called exactly
+// once.
+func acquireProxy(port int32) (release func(), err error) {
+	sharedProxies.mu.Lock()
+	defer sharedProxies.mu.Unlock()
+
+	entry, ok := sharedProxies.byPort[port]
+	if !ok {
+		p, err := newConnectProxy(fmt.Sprintf("%s:%d", mockProxyIP, port))
+		if err != nil {
+			return nil, err
+		}
+		entry = &sharedProxyEntry{proxy: p}
+		sharedProxies.byPort[port] = entry
+	}
+	entry.refs++
+
+	var once sync.Once
+	release = func() {
+		once.Do(func() {
+			sharedProxies.mu.Lock()
+			defer sharedProxies.mu.Unlock()
+			entry.refs--
+			if entry.refs <= 0 {
+				entry.proxy.close()
+				delete(sharedProxies.byPort, port)
+			}
+		})
+	}
+	return release, nil
+}

internal/provider/mock/proxy_test.go

@@ -0,0 +1,51 @@
+package mock
+
+import (
+	"net"
+	"strconv"
+	"testing"
+)
+
+func TestAcquireProxy_sharedAcrossAcquires(t *testing.T) {
+	t.Parallel()
+	port := freePort(t)
+	addr := net.JoinHostPort(mockProxyIP, strconv.Itoa(int(port)))
+
+	release1, err := acquireProxy(port)
+	if err != nil {
+		t.Fatalf("acquireProxy() #1 error = %v", err)
+	}
+	// A second acquire for the same port must join the existing listener
+	// rather than fail trying to bind it again — this is the whole point
+	// of sharing by port instead of by IP.
+	release2, err := acquireProxy(port)
+	if err != nil {
+		t.Fatalf("acquireProxy() #2 error = %v, want nil (should share the existing listener)", err)
+	}
+
+	release1()
+	// One reference remains; the port must still be in use.
+	if ln, err := net.Listen("tcp", addr); err == nil {
+		ln.Close()
+		t.Fatal("port became bindable after releasing only one of two references")
+	}
+
+	release2()
+	// Last reference released: the port must now be free.
+	ln, err := net.Listen("tcp", addr)
+	if err != nil {
+		t.Fatalf("port not released after last reference: %v", err)
+	}
+	ln.Close()
+}
+
+func TestAcquireProxy_releaseIsIdempotent(t *testing.T) {
+	t.Parallel()
+	port := freePort(t)
+	release, err := acquireProxy(port)
+	if err != nil {
+		t.Fatalf("acquireProxy() error = %v", err)
+	}
+	release()
+	release() // must not panic or double-decrement into a negative refcount
+}