Initial commit: Go SRE interview exercises
Includes solved 02-worker-pool and 09-safe-counter (mutex, atomic, sharded). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GbnJj9ZqEx9nUvzkjdwfMq
This commit is contained in:
Claude Fable 5
@@ -0,0 +1,54 @@
|
||||
# 02 — Worker Pool
|
||||
|
||||
The fundamental fan-out/fan-in shape. Everything else (pipelines, errgroup,
|
||||
batch processors) is a decoration of this.
|
||||
|
||||
## The channel-closing protocol
|
||||
|
||||
The whole exercise is really about one question: **who closes what, and when?**
|
||||
|
||||
Rules worth engraving:
|
||||
|
||||
1. **Only the sender closes.** A receiver closing a channel = eventual panic on
|
||||
the sender side (`send on closed channel`).
|
||||
2. **Close is a broadcast.** Every `range` and every `v, ok := <-ch` across all
|
||||
receivers unblocks. That's why `close(jobs)` cleanly retires all workers.
|
||||
3. **With N senders, no single sender may close.** That's the "closer
|
||||
goroutine" pattern you need here:
|
||||
|
||||
```go
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(out)
|
||||
}()
|
||||
```
|
||||
|
||||
All N workers send to `out`; a dedicated goroutine waits for all of them and
|
||||
then closes. This is the only race-free way to close a multi-producer channel.
|
||||
|
||||
4. **You don't have to close channels at all.** Channels are GC'd like anything
|
||||
else. Closing is *signaling*, not resource cleanup. You close `out` here only
|
||||
because the consumer ranges over it.
|
||||
|
||||
## The subtle leak in the naive version
|
||||
|
||||
If you forget `close(out)`, the consumer's `range` blocks forever — the test
|
||||
catches this with a timeout. In prod this presents as goroutine count creeping
|
||||
up one per request, which you'd see in `go_goroutines` on your dashboard long
|
||||
before OOM. Well... ideally you would.
|
||||
|
||||
## Why unbuffered `out` is fine (and interesting)
|
||||
|
||||
With `out` unbuffered, workers block until the consumer reads → natural
|
||||
backpressure, memory bounded by `workers`. Buffering `out` decouples throughput
|
||||
spikes but the buffer is a queue, and queues hide latency — a very SRE
|
||||
trade-off to mention out loud. (Little's law applies to your channels too.)
|
||||
|
||||
## Interview follow-ups to prep
|
||||
|
||||
- "Add context cancellation" → every `out <- fn(j)` becomes a `select` with
|
||||
`ctx.Done()`, and job consumption too.
|
||||
- "A job can fail" → either a Result{V, Err} envelope type, or switch to
|
||||
`errgroup` and talk about fail-fast vs collect-all-errors semantics.
|
||||
- "Jobs arrive faster than they're processed" → bounded jobs channel =
|
||||
backpressure at the producer; discuss shedding vs queueing.
|
||||
@@ -0,0 +1,61 @@
|
||||
// Exercise 02 — Worker Pool
|
||||
//
|
||||
// ASSIGNMENT
|
||||
// ----------
|
||||
// Implement Pool so that it:
|
||||
//
|
||||
// 1. Spawns exactly `workers` goroutines.
|
||||
// 2. Each worker consumes jobs from `jobs` until the channel is closed.
|
||||
// 3. Results of fn(job) are sent to the returned channel.
|
||||
// 4. The returned channel is CLOSED after ALL workers finish.
|
||||
// (Whoever ranges over the output must terminate — no deadlocks.)
|
||||
// 5. No goroutines may leak.
|
||||
//
|
||||
// Run the tests: go test -race ./02-worker-pool/
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Pool fans `jobs` out to `workers` goroutines running fn, fans results in.
|
||||
// TODO: implement me.
|
||||
func Pool[T, R any](jobs <-chan T, workers int, fn func(T) R) <-chan R {
|
||||
out := make(chan R)
|
||||
var wg sync.WaitGroup
|
||||
// start all the workers and increment the waitgroup
|
||||
for i := 0; i < workers; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for j := range jobs {
|
||||
out <- fn(j)
|
||||
}
|
||||
}()
|
||||
}
|
||||
// TODO:
|
||||
// - start `workers` goroutines, each: for j := range jobs { out <- fn(j) }
|
||||
// - close(out) exactly once, only after every worker returned
|
||||
// (hint: sync.WaitGroup + one extra "closer" goroutine)
|
||||
|
||||
// start go routine that closes out channel when all workers are done
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(out)
|
||||
}()
|
||||
return out
|
||||
}
|
||||
|
||||
func main() {
|
||||
jobs := make(chan int)
|
||||
go func() {
|
||||
for i := 1; i <= 9; i++ {
|
||||
jobs <- i
|
||||
}
|
||||
close(jobs)
|
||||
}()
|
||||
for r := range Pool(jobs, 3, func(n int) int { return n * n }) {
|
||||
fmt.Println(r)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestAllJobsProcessedAndOutputCloses(t *testing.T) {
|
||||
jobs := make(chan int)
|
||||
go func() {
|
||||
for i := 0; i < 100; i++ {
|
||||
jobs <- i
|
||||
}
|
||||
close(jobs)
|
||||
}()
|
||||
|
||||
out := Pool(jobs, 5, func(n int) int { return n * 2 })
|
||||
|
||||
got := []int{}
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
for v := range out { // terminates only if out is closed
|
||||
got = append(got, v)
|
||||
}
|
||||
close(done)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("output channel never closed — deadlock or missing close(out)")
|
||||
}
|
||||
|
||||
if len(got) != 100 {
|
||||
t.Fatalf("want 100 results, got %d", len(got))
|
||||
}
|
||||
sort.Ints(got)
|
||||
for i, v := range got {
|
||||
if v != i*2 {
|
||||
t.Fatalf("missing/wrong result: index %d = %d", i, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestActuallyConcurrent(t *testing.T) {
|
||||
var inFlight, peak atomic.Int32
|
||||
jobs := make(chan int)
|
||||
go func() {
|
||||
for i := 0; i < 12; i++ {
|
||||
jobs <- i
|
||||
}
|
||||
close(jobs)
|
||||
}()
|
||||
|
||||
out := Pool(jobs, 4, func(n int) int {
|
||||
cur := inFlight.Add(1)
|
||||
for {
|
||||
p := peak.Load()
|
||||
if cur <= p || peak.CompareAndSwap(p, cur) {
|
||||
break
|
||||
}
|
||||
}
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
inFlight.Add(-1)
|
||||
return n
|
||||
})
|
||||
drained := make(chan struct{})
|
||||
go func() {
|
||||
for range out {
|
||||
}
|
||||
close(drained)
|
||||
}()
|
||||
select {
|
||||
case <-drained:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("output never closed")
|
||||
}
|
||||
|
||||
if p := peak.Load(); p < 2 {
|
||||
t.Errorf("expected concurrent execution, peak parallelism was %d", p)
|
||||
} else if p > 4 {
|
||||
t.Errorf("more than `workers` goroutines running fn: peak %d > 4", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyJobs(t *testing.T) {
|
||||
jobs := make(chan int)
|
||||
close(jobs)
|
||||
out := Pool(jobs, 3, func(n int) int { return n })
|
||||
select {
|
||||
case _, ok := <-out:
|
||||
if ok {
|
||||
t.Fatal("got a value from empty job stream")
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("output not closed for empty job stream")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user