Initial commit: Go SRE interview exercises
Includes solved 02-worker-pool and 09-safe-counter (mutex, atomic, sharded). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GbnJj9ZqEx9nUvzkjdwfMq
This commit is contained in:
@@ -0,0 +1,54 @@
|
||||
# 04 — Log Crunching: Top-N
|
||||
|
||||
Looks like a warm-up. It's actually a trap-density championship.
|
||||
|
||||
## Trap 1: `bufio.Scanner`'s 64KB guillotine
|
||||
|
||||
`Scanner` refuses tokens above 64KB by default and stops with
|
||||
`bufio.ErrTooLong` — and if you forget to check `sc.Err()`, it stops
|
||||
*silently*, and your report just... misses the tail of the file. Real-world
|
||||
lines that exceed 64KB: JSON-formatted logs, stack traces, requests with fat
|
||||
query strings. The fix:
|
||||
|
||||
```go
|
||||
sc.Buffer(make([]byte, 0, 1024*1024), 1024*1024)
|
||||
```
|
||||
|
||||
An enormous number of production log processors have shipped with this bug.
|
||||
Checking `sc.Err()` after the loop is the difference between a tool and a liar.
|
||||
|
||||
## Trap 2: maps have no order — by design, with a great backstory
|
||||
|
||||
Go **deliberately randomizes** map iteration order (a random seed per map, and
|
||||
the runtime even randomizes the starting bucket) so nobody can accidentally
|
||||
depend on it. To rank by count you must extract into a slice and sort:
|
||||
|
||||
```go
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Count > out[j].Count })
|
||||
```
|
||||
|
||||
Since 1.24 maps use a Swiss-table implementation (flat, SIMD-probed, cache
|
||||
friendlier) — fun fact to drop if asked "what's inside a Go map."
|
||||
|
||||
## Trap 3: `strings.Fields` vs `strings.Split`
|
||||
|
||||
`Split(s, " ")` produces empty strings for repeated spaces; `Fields` splits on
|
||||
any run of whitespace (incl. tabs) and never yields empties. For log parsing,
|
||||
`Fields` is almost always what you mean.
|
||||
|
||||
## The senior follow-up: "the file is 500 GB"
|
||||
|
||||
Have a ladder ready:
|
||||
|
||||
1. **This solution already streams** — memory is O(distinct endpoints), not
|
||||
O(file). Say that first; many candidates don't realize their answer scales.
|
||||
2. **Cardinality explosion** (endpoints contain IDs): normalize paths
|
||||
(`/users/123` → `/users/:id`) or cap the map with count-min sketch /
|
||||
space-saving algorithm — approximate top-K in bounded memory. Naming
|
||||
"count-min sketch" in an SRE interview is a mic-drop; it's what real
|
||||
metrics pipelines do to survive high-cardinality labels.
|
||||
3. **Parallelize**: shard by hash(endpoint) across workers, merge partials —
|
||||
map-reduce in one sentence.
|
||||
4. **Only need top-N, not full sort?** A min-heap of size N gives
|
||||
O(distinct · log N) instead of O(distinct · log distinct). `container/heap`
|
||||
exists; nobody remembers its interface; everyone respects you for trying.
|
||||
@@ -0,0 +1,52 @@
|
||||
// Exercise 04 — Log Crunching: Top-N Endpoints
|
||||
//
|
||||
// ASSIGNMENT
|
||||
// ----------
|
||||
// Log lines look like:
|
||||
//
|
||||
// 2026-08-28T10:00:01Z GET /api/users 200 123ms
|
||||
//
|
||||
// Implement TopN(r, n) so that it:
|
||||
//
|
||||
// 1. Streams the input (must handle logs far larger than memory — no
|
||||
// io.ReadAll).
|
||||
// 2. Counts requests per endpoint (3rd whitespace-separated field).
|
||||
// 3. Skips malformed lines (fewer than 3 fields) without failing.
|
||||
// 4. Handles lines longer than 64KB (hint: bufio.Scanner's default limit).
|
||||
// 5. Returns the n most frequent endpoints, most frequent first.
|
||||
// Ties: order among equal counts is unspecified.
|
||||
//
|
||||
// Run the tests: go test -race ./04-log-topn/
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type KV struct {
|
||||
Endpoint string
|
||||
Count int
|
||||
}
|
||||
|
||||
// TopN returns the n most frequent endpoints in the log stream.
|
||||
// TODO: implement me.
|
||||
func TopN(r io.Reader, n int) ([]KV, error) {
|
||||
// TODO:
|
||||
// - bufio.Scanner with an enlarged buffer
|
||||
// - map[string]int for counting
|
||||
// - copy map into []KV and sort by count desc (maps can't be sorted)
|
||||
// - truncate to n
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
logs := `2026-08-28T10:00:01Z GET /api/users 200 12ms
|
||||
2026-08-28T10:00:02Z GET /api/orders 200 40ms
|
||||
2026-08-28T10:00:03Z GET /api/users 500 3ms
|
||||
garbage line
|
||||
2026-08-28T10:00:04Z POST /api/users 201 88ms`
|
||||
top, err := TopN(strings.NewReader(logs), 2)
|
||||
fmt.Println(top, err)
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTopNBasic(t *testing.T) {
|
||||
logs := strings.Join([]string{
|
||||
"t GET /a 200 1ms",
|
||||
"t GET /b 200 1ms",
|
||||
"t GET /a 200 1ms",
|
||||
"t GET /c 200 1ms",
|
||||
"t GET /a 200 1ms",
|
||||
"t GET /b 200 1ms",
|
||||
}, "\n")
|
||||
|
||||
got, err := TopN(strings.NewReader(logs), 2)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("want 2 results, got %d: %v", len(got), got)
|
||||
}
|
||||
if got[0].Endpoint != "/a" || got[0].Count != 3 {
|
||||
t.Errorf("want /a x3 first, got %+v", got[0])
|
||||
}
|
||||
if got[1].Endpoint != "/b" || got[1].Count != 2 {
|
||||
t.Errorf("want /b x2 second, got %+v", got[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestMalformedLinesSkipped(t *testing.T) {
|
||||
logs := "t GET /a 200 1ms\n\ngarbage\nonly two\nt GET /a 200 1ms\n"
|
||||
got, err := TopN(strings.NewReader(logs), 5)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Count != 2 {
|
||||
t.Fatalf("malformed lines mishandled: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNLargerThanDistinct(t *testing.T) {
|
||||
got, err := TopN(strings.NewReader("t GET /a 200 1ms"), 10)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("want 1 result, got %d", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestLongLines(t *testing.T) {
|
||||
// A single line > 64KB (bufio.Scanner default max token size).
|
||||
long := "t GET /huge?q=" + strings.Repeat("x", 80*1024) + " 200 1ms"
|
||||
logs := long + "\nt GET /small 200 1ms\nt GET /small 200 1ms\n"
|
||||
got, err := TopN(strings.NewReader(logs), 1)
|
||||
if err != nil {
|
||||
t.Fatalf("long line broke the scanner: %v (raise Scanner.Buffer)", err)
|
||||
}
|
||||
if len(got) == 0 || got[0].Endpoint != "/small" {
|
||||
t.Fatalf("want /small on top, got %v", got)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user