Add global-ratelimit experiment: token-bucket limiter as an HTTP service
limiter/ package (ex03 bucket + ex09 sharded keyed store + idle sweep, race-tested), server/ with /allow /stats /healthz and X-RateLimit headers, client/ demo replica with fail-open/closed and latency percentiles. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GbnJj9ZqEx9nUvzkjdwfMq
This commit is contained in:
Claude Fable 5
@@ -0,0 +1,185 @@
|
||||
// Package limiter is the token bucket from exercise 03 turned into a keyed
|
||||
// store suitable for a shared rate-limiter service: many buckets addressed by
|
||||
// a string key (tenant, client id, API key), sharded like exercise 09, with
|
||||
// eviction of idle buckets so the map does not grow forever.
|
||||
package limiter
|
||||
|
||||
import (
|
||||
"hash/fnv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ---- Bucket: a single token bucket (exercise 03) ----
|
||||
|
||||
// Bucket is a lazily refilled token bucket. Safe for concurrent use.
|
||||
type Bucket struct {
|
||||
mu sync.Mutex
|
||||
tokens float64
|
||||
max float64
|
||||
rate float64 // tokens per second
|
||||
last time.Time
|
||||
}
|
||||
|
||||
// NewBucket returns a full bucket allowing `rate` tokens/s with bursts up to `burst`.
|
||||
func NewBucket(rate float64, burst int) *Bucket {
|
||||
return &Bucket{tokens: float64(burst), max: float64(burst), rate: rate, last: time.Now()}
|
||||
}
|
||||
|
||||
// Decision is the outcome of an AllowN call.
|
||||
type Decision struct {
|
||||
Allowed bool
|
||||
Remaining float64 // tokens left after this decision
|
||||
Limit int // burst (bucket capacity)
|
||||
Rate float64 // refill rate, tokens/s
|
||||
RetryAfter time.Duration // when denied: how long until n tokens would be available
|
||||
}
|
||||
|
||||
// AllowN tries to take n tokens. Non-blocking.
|
||||
func (b *Bucket) AllowN(n int) Decision {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
|
||||
// 1. lazy refill, capped at max; `last` advances on every call
|
||||
now := time.Now()
|
||||
b.tokens += now.Sub(b.last).Seconds() * b.rate
|
||||
if b.tokens > b.max {
|
||||
b.tokens = b.max
|
||||
}
|
||||
b.last = now
|
||||
|
||||
d := Decision{Limit: int(b.max), Rate: b.rate}
|
||||
// 2. consume or compute how long until we could
|
||||
if b.tokens >= float64(n) {
|
||||
b.tokens -= float64(n)
|
||||
d.Allowed = true
|
||||
} else if b.rate > 0 {
|
||||
d.RetryAfter = time.Duration((float64(n) - b.tokens) / b.rate * float64(time.Second))
|
||||
}
|
||||
d.Remaining = b.tokens
|
||||
return d
|
||||
}
|
||||
|
||||
// idleSince reports the last time the bucket was touched and whether it is
|
||||
// full (i.e. nothing would be lost by forgetting it).
|
||||
func (b *Bucket) idleSince() (time.Time, bool) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
// A bucket untouched for `max/rate` seconds is full again by definition,
|
||||
// so idle time alone decides; we don't need to refill here.
|
||||
return b.last, true
|
||||
}
|
||||
|
||||
// ---- Store: many buckets by key (exercise 09's sharding) ----
|
||||
|
||||
const shards = 16
|
||||
|
||||
type shard struct {
|
||||
mu sync.Mutex
|
||||
m map[string]*Bucket
|
||||
}
|
||||
|
||||
// Limit is a (rate, burst) pair.
|
||||
type Limit struct {
|
||||
Rate float64
|
||||
Burst int
|
||||
}
|
||||
|
||||
// Store holds one Bucket per key, created lazily with the default Limit or a
|
||||
// per-key override.
|
||||
type Store struct {
|
||||
def Limit
|
||||
overrides map[string]Limit // read-only after construction
|
||||
shards [shards]shard
|
||||
|
||||
allowed atomic.Int64
|
||||
denied atomic.Int64
|
||||
}
|
||||
|
||||
// NewStore creates a store; unknown keys get `def`, keys in `overrides` get theirs.
|
||||
func NewStore(def Limit, overrides map[string]Limit) *Store {
|
||||
s := &Store{def: def, overrides: overrides}
|
||||
for i := range s.shards {
|
||||
s.shards[i].m = make(map[string]*Bucket)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *Store) shardFor(key string) *shard {
|
||||
h := fnv.New32a()
|
||||
h.Write([]byte(key))
|
||||
return &s.shards[h.Sum32()%shards]
|
||||
}
|
||||
|
||||
func (s *Store) limitFor(key string) Limit {
|
||||
if l, ok := s.overrides[key]; ok {
|
||||
return l
|
||||
}
|
||||
return s.def
|
||||
}
|
||||
|
||||
// bucket returns the bucket for key, creating it on first sight.
|
||||
func (s *Store) bucket(key string) *Bucket {
|
||||
sh := s.shardFor(key)
|
||||
sh.mu.Lock()
|
||||
defer sh.mu.Unlock()
|
||||
b, ok := sh.m[key]
|
||||
if !ok {
|
||||
l := s.limitFor(key)
|
||||
b = NewBucket(l.Rate, l.Burst)
|
||||
sh.m[key] = b
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// Allow tries to take n tokens from key's bucket.
|
||||
func (s *Store) Allow(key string, n int) Decision {
|
||||
d := s.bucket(key).AllowN(n)
|
||||
if d.Allowed {
|
||||
s.allowed.Add(1)
|
||||
} else {
|
||||
s.denied.Add(1)
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// Sweep removes buckets not touched for at least `idle`. Such a bucket has
|
||||
// refilled to full anyway (as long as idle >= burst/rate), so dropping it and
|
||||
// recreating it later is unobservable to clients. Returns how many were removed.
|
||||
func (s *Store) Sweep(idle time.Duration) int {
|
||||
cutoff := time.Now().Add(-idle)
|
||||
removed := 0
|
||||
for i := range s.shards {
|
||||
sh := &s.shards[i]
|
||||
sh.mu.Lock()
|
||||
for k, b := range sh.m {
|
||||
if last, _ := b.idleSince(); last.Before(cutoff) {
|
||||
delete(sh.m, k)
|
||||
removed++
|
||||
}
|
||||
}
|
||||
sh.mu.Unlock()
|
||||
}
|
||||
return removed
|
||||
}
|
||||
|
||||
// Stats is a point-in-time snapshot for observability.
|
||||
type Stats struct {
|
||||
Keys int `json:"keys"`
|
||||
Allowed int64 `json:"allowed_total"`
|
||||
Denied int64 `json:"denied_total"`
|
||||
}
|
||||
|
||||
// Stats returns counters. Keys is counted shard by shard, not atomically
|
||||
// across shards (fine for metrics; see exercise 09 Snapshot discussion).
|
||||
func (s *Store) Stats() Stats {
|
||||
st := Stats{Allowed: s.allowed.Load(), Denied: s.denied.Load()}
|
||||
for i := range s.shards {
|
||||
sh := &s.shards[i]
|
||||
sh.mu.Lock()
|
||||
st.Keys += len(sh.m)
|
||||
sh.mu.Unlock()
|
||||
}
|
||||
return st
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
package limiter
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestBurstThenDeny(t *testing.T) {
|
||||
s := NewStore(Limit{Rate: 1, Burst: 3}, nil)
|
||||
for i := 0; i < 3; i++ {
|
||||
if !s.Allow("k", 1).Allowed {
|
||||
t.Fatalf("request %d within burst should be allowed", i)
|
||||
}
|
||||
}
|
||||
d := s.Allow("k", 1)
|
||||
if d.Allowed {
|
||||
t.Fatal("4th immediate request should be denied")
|
||||
}
|
||||
if d.RetryAfter <= 0 || d.RetryAfter > time.Second {
|
||||
t.Fatalf("retry-after should be ~1s at rate 1, got %v", d.RetryAfter)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefill(t *testing.T) {
|
||||
s := NewStore(Limit{Rate: 20, Burst: 1}, nil)
|
||||
if !s.Allow("k", 1).Allowed {
|
||||
t.Fatal("initial token missing")
|
||||
}
|
||||
if s.Allow("k", 1).Allowed {
|
||||
t.Fatal("should be empty immediately after")
|
||||
}
|
||||
time.Sleep(120 * time.Millisecond)
|
||||
if !s.Allow("k", 1).Allowed {
|
||||
t.Fatal("should have refilled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBurstIsCapped(t *testing.T) {
|
||||
s := NewStore(Limit{Rate: 1000, Burst: 2}, nil)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
allowed := 0
|
||||
for i := 0; i < 10; i++ {
|
||||
if s.Allow("k", 1).Allowed {
|
||||
allowed++
|
||||
}
|
||||
}
|
||||
if allowed > 2 {
|
||||
t.Fatalf("idle time built credit beyond burst: %d", allowed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeysAreIsolatedAndOverridden(t *testing.T) {
|
||||
s := NewStore(Limit{Rate: 0.0001, Burst: 1}, map[string]Limit{"vip": {Rate: 0.0001, Burst: 5}})
|
||||
if !s.Allow("a", 1).Allowed || s.Allow("a", 1).Allowed {
|
||||
t.Fatal("key a should allow exactly one")
|
||||
}
|
||||
if !s.Allow("b", 1).Allowed {
|
||||
t.Fatal("key b must have its own bucket")
|
||||
}
|
||||
for i := 0; i < 5; i++ {
|
||||
if !s.Allow("vip", 1).Allowed {
|
||||
t.Fatalf("vip request %d should be allowed (burst 5)", i)
|
||||
}
|
||||
}
|
||||
if s.Allow("vip", 1).Allowed {
|
||||
t.Fatal("vip 6th should be denied")
|
||||
}
|
||||
if got := s.Stats().Keys; got != 3 {
|
||||
t.Fatalf("expected 3 keys, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllowN(t *testing.T) {
|
||||
s := NewStore(Limit{Rate: 0.0001, Burst: 10}, nil)
|
||||
if !s.Allow("k", 7).Allowed {
|
||||
t.Fatal("7 of 10 should be allowed")
|
||||
}
|
||||
if s.Allow("k", 4).Allowed {
|
||||
t.Fatal("4 more (3 left) should be denied")
|
||||
}
|
||||
if !s.Allow("k", 3).Allowed {
|
||||
t.Fatal("exactly 3 left should be allowed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrentNoOverAdmission(t *testing.T) {
|
||||
s := NewStore(Limit{Rate: 0.0001, Burst: 100}, nil)
|
||||
var allowed atomic.Int64
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 50; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for j := 0; j < 10; j++ {
|
||||
if s.Allow("shared", 1).Allowed {
|
||||
allowed.Add(1)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if got := allowed.Load(); got != 100 {
|
||||
t.Fatalf("expected exactly 100 admissions, got %d", got)
|
||||
}
|
||||
st := s.Stats()
|
||||
if st.Allowed != 100 || st.Denied != 400 {
|
||||
t.Fatalf("stats mismatch: %+v", st)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSweepEvictsIdleOnly(t *testing.T) {
|
||||
s := NewStore(Limit{Rate: 1000, Burst: 1}, nil)
|
||||
s.Allow("old", 1)
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
s.Allow("fresh", 1)
|
||||
if n := s.Sweep(20 * time.Millisecond); n != 1 {
|
||||
t.Fatalf("expected 1 evicted, got %d", n)
|
||||
}
|
||||
if got := s.Stats().Keys; got != 1 {
|
||||
t.Fatalf("expected 1 key left, got %d", got)
|
||||
}
|
||||
// evicted key comes back full, which is what an idle bucket would be anyway
|
||||
if !s.Allow("old", 1).Allowed {
|
||||
t.Fatal("recreated bucket should start full")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user