Add body-drain experiment: observe HTTP body drain/close/leak effects
Local counting server + httptrace/goroutine metrics showing connection reuse (drain), pool bypass (close only) and fd/goroutine leak (no close). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GbnJj9ZqEx9nUvzkjdwfMq
This commit is contained in:
Claude Fable 5
@@ -0,0 +1,176 @@
|
||||
// Experiment — what happens when you don't drain/close an HTTP response body
|
||||
//
|
||||
// A local server hands out 1 MiB responses and counts every TCP connection it
|
||||
// accepts. The client performs N sequential GETs against it in one of three
|
||||
// modes and we observe:
|
||||
//
|
||||
// - server-side: how many *new* TCP connections were accepted (ideal: 1)
|
||||
// - client-side: httptrace's GotConnInfo.Reused for every request
|
||||
// - client-side: goroutine count (each pooled/leaked conn owns 2 goroutines:
|
||||
// the transport's readLoop and writeLoop)
|
||||
//
|
||||
// Modes:
|
||||
//
|
||||
// drain — io.Copy(io.Discard, body); body.Close() → connection reused
|
||||
// close — body.Close() only → connection thrown away, new dial every time
|
||||
// leak — neither → connection AND 2 goroutines AND 1 fd leaked per request
|
||||
//
|
||||
// Run: go run ./99-experiments/body-drain -mode drain
|
||||
// go run ./99-experiments/body-drain -mode close
|
||||
// go run ./99-experiments/body-drain -mode leak
|
||||
// go run ./99-experiments/body-drain -mode all (all three, fresh transport each)
|
||||
//
|
||||
// To see the leak actually kill the process, lower the fd limit first:
|
||||
// ulimit -n 64 && go run ./99-experiments/body-drain -mode leak -n 200
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/http/httptrace"
|
||||
"os"
|
||||
"runtime"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
const bodySize = 1 << 20 // 1 MiB — large enough that it is never fully buffered in the transport
|
||||
|
||||
type mode string
|
||||
|
||||
const (
|
||||
modeDrain mode = "drain"
|
||||
modeClose mode = "close"
|
||||
modeLeak mode = "leak"
|
||||
)
|
||||
|
||||
// countingServer wraps httptest.Server and counts accepted connections.
|
||||
type countingServer struct {
|
||||
*httptest.Server
|
||||
newConns atomic.Int64
|
||||
activeConns atomic.Int64
|
||||
}
|
||||
|
||||
func newCountingServer() *countingServer {
|
||||
payload := make([]byte, bodySize)
|
||||
s := &countingServer{}
|
||||
s.Server = httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Length", fmt.Sprint(bodySize))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(payload)
|
||||
}))
|
||||
// ConnState is the server-side hook that lets us see the connection lifecycle.
|
||||
s.Config.ConnState = func(c net.Conn, st http.ConnState) {
|
||||
switch st {
|
||||
case http.StateNew:
|
||||
s.newConns.Add(1)
|
||||
s.activeConns.Add(1)
|
||||
case http.StateClosed, http.StateHijacked:
|
||||
s.activeConns.Add(-1)
|
||||
}
|
||||
}
|
||||
s.Start()
|
||||
return s
|
||||
}
|
||||
|
||||
// run performs n GETs in the given mode and reports what it observed.
|
||||
func run(m mode, n int, url string) {
|
||||
// Fresh transport per run so runs don't share a pool. Cloning keeps sane defaults.
|
||||
tr := http.DefaultTransport.(*http.Transport).Clone()
|
||||
tr.MaxIdleConnsPerHost = 10
|
||||
// NOTE: deliberately no Client.Timeout. With a timeout, a leaked body gets
|
||||
// force-closed when the timer fires, which hides the leak (try adding one
|
||||
// and watch the goroutine count drop after the timeout).
|
||||
client := &http.Client{Transport: tr}
|
||||
|
||||
var reused, fresh int
|
||||
trace := &httptrace.ClientTrace{
|
||||
GotConn: func(info httptrace.GotConnInfo) {
|
||||
if info.Reused {
|
||||
reused++
|
||||
} else {
|
||||
fresh++
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
goroutinesBefore := runtime.NumGoroutine()
|
||||
start := time.Now()
|
||||
|
||||
for i := 0; i < n; i++ {
|
||||
req, err := http.NewRequestWithContext(httptrace.WithClientTrace(context.Background(), trace), http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
fmt.Printf(" request %d failed: %v\n", i, err)
|
||||
// This is where `leak` eventually dies with "socket: too many open files".
|
||||
break
|
||||
}
|
||||
|
||||
switch m {
|
||||
case modeDrain:
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
resp.Body.Close()
|
||||
case modeClose:
|
||||
resp.Body.Close()
|
||||
case modeLeak:
|
||||
// nothing — the body, the conn, its fd and its 2 goroutines are now orphaned
|
||||
}
|
||||
}
|
||||
|
||||
elapsed := time.Since(start)
|
||||
// Give the transport a moment to settle bookkeeping (put conn back to idle etc.).
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
goroutinesAfter := runtime.NumGoroutine()
|
||||
|
||||
fmt.Printf(" requests : %d in %v\n", n, elapsed.Round(time.Millisecond))
|
||||
fmt.Printf(" httptrace GotConn : reused=%d fresh=%d\n", reused, fresh)
|
||||
fmt.Printf(" goroutines : before=%d after=%d (delta %+d)\n",
|
||||
goroutinesBefore, goroutinesAfter, goroutinesAfter-goroutinesBefore)
|
||||
|
||||
// Closing idle conns cleans up whatever the transport still *knows* about.
|
||||
// Leaked bodies are not idle, so they survive this — that's the point.
|
||||
tr.CloseIdleConnections()
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
fmt.Printf(" after CloseIdleConnections: goroutines=%d\n", runtime.NumGoroutine())
|
||||
}
|
||||
|
||||
func main() {
|
||||
modeFlag := flag.String("mode", "all", "drain | close | leak | all")
|
||||
n := flag.Int("n", 20, "number of requests per mode")
|
||||
flag.Parse()
|
||||
|
||||
modes := []mode{modeDrain, modeClose, modeLeak}
|
||||
if *modeFlag != "all" {
|
||||
modes = []mode{mode(*modeFlag)}
|
||||
}
|
||||
|
||||
for _, m := range modes {
|
||||
srv := newCountingServer()
|
||||
fmt.Printf("== mode=%s ==\n", m)
|
||||
run(m, *n, srv.URL)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
fmt.Printf(" server new conns : %d (ideal: 1)\n", srv.newConns.Load())
|
||||
fmt.Printf(" server active conns: %d (ideal after cleanup: 0)\n", srv.activeConns.Load())
|
||||
fmt.Println()
|
||||
// In leak mode the server's handlers are STILL BLOCKED in Write(): the
|
||||
// kernel socket buffers filled up because nobody on the client side is
|
||||
// reading the 1 MiB body. srv.Close() waits for active conns and would
|
||||
// hang forever (it logs "blocked in Close after 5 seconds" and lists
|
||||
// them). That's the server-side symptom of a client leak: goroutines
|
||||
// and memory pinned per stuck connection. Force-close them first.
|
||||
srv.CloseClientConnections()
|
||||
srv.Close()
|
||||
}
|
||||
|
||||
if len(modes) == 1 && modes[0] == modeLeak {
|
||||
fmt.Fprintln(os.Stderr, "tip: rerun with `ulimit -n 64` and -n 200 to watch it hit 'too many open files'")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user