From f609d8124fa2cd5ca589decb0e36d07c3223042a Mon Sep 17 00:00:00 2001 From: Jan Novak Date: Wed, 1 Jul 2026 12:07:44 +0200 Subject: [PATCH] =?UTF-8?q?feat(py):=20step=202=20=E2=80=94=20per-phase=20?= =?UTF-8?q?latency=20measurement=20(phases.py)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hand-drives raw sockets to time each HTTP phase individually — DNS (getaddrinfo), TCP connect, TLS handshake (ssl.wrap_socket, https only), TTFB (sendall → first recv), Transfer (first byte → EOF), Total. Partial phases are preserved on failure (same invariant as Go's probe.go). Error classification mirrors Go's priority: dns → timeout → tls → connect. Output format matches Go's single-sample text layout. Input: bare URL args or plain-text config file (same format as simple.py). Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 15 ++ docs/plans/2026-07-01-12-04-py-phases.md | 68 ++++++ docs/usage/py-phases.md | 151 ++++++++++++ python/phases.py | 286 +++++++++++++++++++++++ 4 files changed, 520 insertions(+) create mode 100644 docs/plans/2026-07-01-12-04-py-phases.md create mode 100644 docs/usage/py-phases.md create mode 100644 python/phases.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c37add7..a1050e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,21 @@ All completed features are logged here in reverse-chronological order. --- +## 2026-07-01 12:04 — Per-phase latency measurement (Python, Step 2) + +- `python/phases.py`: self-contained script; hand-drives raw sockets to time + each HTTP phase individually — DNS (`getaddrinfo`), TCP connect, TLS handshake + (`ssl.wrap_socket`, HTTPS only), TTFB (sendall → first recv), Transfer, Total +- Partial phases preserved on failure (same invariant as Go's `probe.go`) +- Error classification mirrors Go's priority: dns → timeout → tls → connect +- Output format matches Go's single-sample text layout (14-char labels, `─` + separator, `%8.2f ms` alignment) +- Input: bare URL args or plain-text config file (same format as `simple.py`) +- Exits 0 if all URLs complete without network error; 1 if any failed +- User doc: `docs/usage/py-phases.md` + +--- + ## 2026-07-01 11:30 — Error-path example configs for simple.py (Python, Step 1 refinement) - `python/configs/` directory with 7 purpose-built config files, one per error class: diff --git a/docs/plans/2026-07-01-12-04-py-phases.md b/docs/plans/2026-07-01-12-04-py-phases.md new file mode 100644 index 0000000..b424adb --- /dev/null +++ b/docs/plans/2026-07-01-12-04-py-phases.md @@ -0,0 +1,68 @@ +# Python Port — Step 2: `phases.py` + +## Goal + +Introduce the manual per-phase timing technique — the Python answer to Go's +`net/http/httptrace`. A single self-contained script that hand-drives a raw +socket for each URL and times each phase individually with `time.perf_counter()`. + +## Why raw sockets + +Python's `urllib` / `httpx` / `requests` give no per-phase callbacks, unlike +Go's `httptrace.ClientTrace`. The only way to time DNS, TCP connect, TLS +handshake, TTFB, and transfer independently is to drive the connection at the +socket level: +- `socket.getaddrinfo()` → DNS +- `sock.connect()` → TCP +- `ssl.SSLContext.wrap_socket()` → TLS (HTTPS only) +- `sock.sendall(request)` + `sock.recv()` → TTFB +- drain to EOF → Transfer + +## Input + +- Bare URL(s) as positional args: `python phases.py https://example.com` +- Or a plain-text config file: `python phases.py configs/all-ok.txt` + (detected by whether the first arg starts with `http://` / `https://`) + +## Output + +Mirrors Go's single-sample text layout for each URL: + +``` +https://example.com (200) + DNS lookup : 18.21 ms + TCP connect : 10.12 ms + TLS handshake : 36.11 ms + Server (TTFB) : 21.95 ms + Transfer : 0.18 ms + ───────────────────────────── + Total : 88.00 ms +``` + +- TLS row omitted for `http://` URLs. +- On failure: header shows `(FAILED)`, partial phases shown, error at the end. +- Multiple URLs separated by a blank line. + +## Error classification + +Mirrors Go's priority order (dns → timeout → tls → connect): +- `socket.gaierror` → `"dns"` +- `socket.timeout` / `TimeoutError` → `"timeout"` (regardless of phase) +- `ssl.SSLError` or `OSError` during TLS wrap → `"tls"` +- `ConnectionRefusedError` / other `OSError` during connect → `"connect"` +- Error after first byte → `"transfer"` + +Partial phases are preserved on failure (same invariant as Go). + +## Known limitations (documented in usage doc) + +- No redirect following (3xx responses are reported with their raw status code). +- `Connection: close` + read-to-EOF; no keep-alive, no HTTP/2. +- Timeout hard-coded at 10 s (no `--timeout` flag — that's in the full version). +- Bodies fully drained to get accurate Transfer timing. + +## Files + +- `python/phases.py` — the script +- `docs/usage/py-phases.md` — user-facing doc +- CHANGELOG.md entry diff --git a/docs/usage/py-phases.md b/docs/usage/py-phases.md new file mode 100644 index 0000000..6891580 --- /dev/null +++ b/docs/usage/py-phases.md @@ -0,0 +1,151 @@ +# `phases.py` — Per-Phase HTTP Latency Measurement + +## What it does + +Measures the latency of each phase of an HTTP request by hand-driving a raw +socket, timing each step individually with `time.perf_counter()`. This is the +Python answer to Go's `net/http/httptrace` — there is no equivalent callback +API in Python's stdlib, so we instrument at the socket level. + +Phases measured: + +| Phase | What is timed | +|-------|---------------| +| DNS lookup | `socket.getaddrinfo()` — hostname resolution | +| TCP connect | `sock.connect()` — SYN to connection established | +| TLS handshake | `ssl.wrap_socket()` — full handshake (HTTPS only) | +| Server (TTFB) | `sendall()` return → first `recv()` byte — server processing time | +| Transfer | First byte → EOF — body download time | +| Total | DNS start → body EOF — wall-clock end-to-end | + +The TLS row is omitted automatically for `http://` URLs. + +## Flags / arguments + +``` +python phases.py [url ...] +python phases.py +``` + +| Argument | Meaning | +|----------|---------| +| ` [url …]` | One or more URLs starting with `http://` or `https://` | +| `` | Path to a plain-text URL list (same format as `simple.py`) | + +Detection is automatic: if the first argument starts with `http://` or +`https://`, all arguments are treated as URLs; otherwise the single argument +is treated as a config file path. + +**Exit codes:** + +| Code | Meaning | +|------|---------| +| 0 | All URLs completed without a network error | +| 1 | One or more URLs failed, or a usage/config error | + +HTTP error statuses (4xx, 5xx) do **not** set exit code 1 — the request +completed successfully at the network level. The status code is visible in +the output header. + +## Example — single URL + +```sh +python phases.py https://example.com +``` + +``` +https://example.com (200) + DNS lookup : 21.74 ms + TCP connect : 10.79 ms + TLS handshake : 18.48 ms + Server (TTFB) : 73.72 ms + Transfer : 0.13 ms + ───────────────────────────── + Total : 132.95 ms +``` + +## Example — HTTP URL (no TLS row) + +```sh +python phases.py http://example.com +``` + +``` +http://example.com (200) + DNS lookup : 3.37 ms + TCP connect : 13.51 ms + Server (TTFB) : 20.01 ms + Transfer : 2.85 ms + ───────────────────────────── + Total : 39.77 ms +``` + +## Example — DNS failure (partial phases) + +```sh +python phases.py https://no.such.host.invalid +``` + +``` +https://no.such.host.invalid (FAILED) + ───────────────────────────── + Total : 0.67 ms + ✗ dns: [Errno 8] nodename nor servname provided, or not known +``` + +## Example — TLS failure (partial phases preserved) + +```sh +python phases.py https://expired.badssl.com/ +``` + +``` +https://expired.badssl.com/ (FAILED) + DNS lookup : 28.32 ms + TCP connect : 129.01 ms + TLS handshake : 305.30 ms + ───────────────────────────── + Total : 473.17 ms + ✗ tls: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired … +``` + +Note: DNS and TCP phases are populated even though the overall request failed. +This is the same behaviour as the Go tool — partial timing is preserved up to +the point of failure. + +## Example — config file + +```sh +python phases.py configs/all-ok.txt +``` + +Multiple URLs are separated by a blank line, matching Go's text output style. + +## Using the error-path example configs + +The `python/configs/` files from `simple.py` work identically with `phases.py`. +Compare the error handling between the two tools: + +```sh +# DNS failure +python phases.py configs/dns-failure.txt + +# Connection refused — note DNS phase is present, TCP is partial +python phases.py configs/connection-refused.txt + +# TLS errors — all three phases up to TLS are present +python phases.py configs/tls-errors.txt +``` + +## Limitations + +- **No redirect following.** 3xx responses are reported with their raw status + code; the redirect target is not probed. Use `latprobe/` (the full version) + or `simple.py` (which uses `urllib`, which follows redirects) if you need + the final destination's timing. +- **HTTP 1.1 + `Connection: close` only.** No keep-alive, no HTTP/2, no auth, + no custom headers beyond `Host` and `User-Agent`. +- **Timeout hard-coded at 10 s.** Use `latprobe/` for a `--timeout` flag. +- **No sampling.** Each URL is probed once. Use `latprobe/` for `-n` (min/avg/max). +- **Body is fully drained.** Transfer time includes reading the entire response + body, so it is real wall-clock transfer time. diff --git a/python/phases.py b/python/phases.py new file mode 100644 index 0000000..20f2ce8 --- /dev/null +++ b/python/phases.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python3 +"""phases.py — measure per-phase HTTP latency using raw sockets. + +Phases: DNS lookup, TCP connect, TLS handshake (HTTPS only), +Server/TTFB (sent → first byte), Transfer (first byte → EOF), Total. + +Usage: + python phases.py [url ...] # one or more URLs + python phases.py # plain-text list of URLs + +Config file: one URL per line; '#' lines and blank lines ignored. +Exits 0 if all URLs complete without network error, 1 if any failed. +""" + +import socket +import ssl +import sys +import time +import urllib.parse +from dataclasses import dataclass, field + +_TIMEOUT = 10.0 +_SEP = "─" * 29 + +# (dataclass field name, 14-char display label) +_PHASE_LABELS = [ + ("dns", "DNS lookup "), + ("connect", "TCP connect "), + ("tls", "TLS handshake "), + ("ttfb", "Server (TTFB) "), + ("transfer", "Transfer "), +] + + +@dataclass +class Phase: + ms: float = 0.0 + present: bool = False + + +@dataclass +class Result: + url: str + dns: Phase = field(default_factory=Phase) + connect: Phase = field(default_factory=Phase) + tls: Phase = field(default_factory=Phase) + ttfb: Phase = field(default_factory=Phase) + transfer: Phase = field(default_factory=Phase) + total: Phase = field(default_factory=Phase) + status_code: int = 0 + fail_phase: str = "" + err: Exception | None = None + + +def _phase(start: float, end: float) -> Phase: + return Phase(ms=(end - start) * 1000, present=True) + + +def _parse_status(data: bytes) -> int: + """Extract HTTP status code from the first recv() chunk.""" + eol = data.find(b"\r\n") + if eol == -1: + eol = data.find(b"\n") + if eol == -1: + return 0 + parts = data[:eol].decode("latin-1", errors="replace").split(None, 2) + if len(parts) >= 2: + try: + return int(parts[1]) + except ValueError: + pass + return 0 + + +def measure(raw_url: str) -> Result: + """Probe raw_url and return a Result with per-phase timings. + + Partial phases are preserved when the request fails mid-flight. + Method is always GET; bodies are fully drained so Transfer timing is real. + Redirects are NOT followed — the raw HTTP response status is reported. + """ + r = Result(url=raw_url) + + parsed = urllib.parse.urlparse(raw_url) + scheme = parsed.scheme.lower() + if scheme not in ("http", "https"): + r.fail_phase = "request" + r.err = ValueError(f"unsupported scheme: {scheme!r}") + return r + + host = parsed.hostname or "" + port = parsed.port or (443 if scheme == "https" else 80) + path = parsed.path or "/" + if parsed.query: + path = path + "?" + parsed.query + use_tls = scheme == "https" + + t_start = time.perf_counter() + + # ── DNS ────────────────────────────────────────────────────────────────── + + t0 = time.perf_counter() + try: + infos = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM) + except socket.gaierror as exc: + r.fail_phase = "dns" + r.err = exc + r.total = _phase(t_start, time.perf_counter()) + return r + r.dns = _phase(t0, time.perf_counter()) + + # ── TCP connect ─────────────────────────────────────────────────────────── + + addr = infos[0][4] + family = infos[0][0] + sock = socket.socket(family, socket.SOCK_STREAM) + sock.settimeout(_TIMEOUT) + t0 = time.perf_counter() + try: + sock.connect(addr) + except socket.timeout as exc: + sock.close() + r.connect = _phase(t0, time.perf_counter()) + r.fail_phase = "timeout" + r.err = exc + r.total = _phase(t_start, time.perf_counter()) + return r + except OSError as exc: + sock.close() + r.connect = _phase(t0, time.perf_counter()) + r.fail_phase = "connect" + r.err = exc + r.total = _phase(t_start, time.perf_counter()) + return r + r.connect = _phase(t0, time.perf_counter()) + + # ── TLS handshake (HTTPS only) ──────────────────────────────────────────── + + if use_tls: + ctx = ssl.create_default_context() + t0 = time.perf_counter() + try: + # wrap_socket blocks until the handshake completes (do_handshake_on_connect=True) + sock = ctx.wrap_socket(sock, server_hostname=host) + except socket.timeout as exc: + # Timeout during handshake classifies as timeout, not tls (mirrors Go) + r.tls = _phase(t0, time.perf_counter()) + r.fail_phase = "timeout" + r.err = exc + r.total = _phase(t_start, time.perf_counter()) + return r + except (ssl.SSLError, OSError) as exc: + r.tls = _phase(t0, time.perf_counter()) + r.fail_phase = "tls" + r.err = exc + r.total = _phase(t_start, time.perf_counter()) + return r + r.tls = _phase(t0, time.perf_counter()) + + # ── Send request ────────────────────────────────────────────────────────── + + request = ( + f"GET {path} HTTP/1.1\r\n" + f"Host: {host}\r\n" + f"Connection: close\r\n" + f"User-Agent: latprobe-phases/1.0\r\n" + f"\r\n" + ).encode() + try: + sock.sendall(request) + except OSError as exc: + sock.close() + r.fail_phase = "request" + r.err = exc + r.total = _phase(t_start, time.perf_counter()) + return r + t_wrote = time.perf_counter() + + # ── TTFB — sent → first response byte ──────────────────────────────────── + + try: + first_chunk = b"" + while not first_chunk: + chunk = sock.recv(4096) + if not chunk: + raise OSError("server closed connection before sending a response") + first_chunk = chunk + except socket.timeout as exc: + sock.close() + r.fail_phase = "timeout" + r.err = exc + r.total = _phase(t_start, time.perf_counter()) + return r + except OSError as exc: + sock.close() + r.fail_phase = "transfer" + r.err = exc + r.total = _phase(t_start, time.perf_counter()) + return r + + t_first_byte = time.perf_counter() + r.ttfb = _phase(t_wrote, t_first_byte) + r.status_code = _parse_status(first_chunk) + + # ── Transfer — drain remaining body ─────────────────────────────────────── + + try: + while True: + chunk = sock.recv(65536) + if not chunk: + break + except OSError: + pass # connection reset during body drain is acceptable; timing is captured + finally: + sock.close() + + t_end = time.perf_counter() + r.transfer = _phase(t_first_byte, t_end) + r.total = _phase(t_start, t_end) + return r + + +def print_result(r: Result) -> None: + if r.err is not None and r.status_code == 0: + print(f"{r.url} (FAILED)") + else: + print(f"{r.url} ({r.status_code})") + + for attr, label in _PHASE_LABELS: + phase: Phase = getattr(r, attr) + if phase.present: + print(f" {label} : {phase.ms:8.2f} ms") + + print(f" {_SEP}") + if r.total.present: + print(f" {'Total '} : {r.total.ms:8.2f} ms") + if r.err is not None: + print(f" ✗ {r.fail_phase}: {r.err}") + + +def load_config(path: str) -> list[str]: + urls = [] + with open(path) as f: + for line in f: + line = line.strip() + if not line or line.startswith("#"): + continue + urls.append(line.split()[0]) + return urls + + +def main(argv: list[str]) -> int: + if not argv: + print(__doc__, file=sys.stderr) + return 1 + + if argv[0].startswith(("http://", "https://")): + urls = argv + else: + try: + urls = load_config(argv[0]) + except FileNotFoundError: + print(f"error: file not found: {argv[0]}", file=sys.stderr) + return 1 + except OSError as exc: + print(f"error: cannot read config: {exc}", file=sys.stderr) + return 1 + + if not urls: + print("error: no URLs found", file=sys.stderr) + return 1 + + any_failed = False + for i, url in enumerate(urls): + if i > 0: + print() + r = measure(url) + print_result(r) + if r.err is not None: + any_failed = True + + return 1 if any_failed else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:]))