feat(py): step 3 — full latprobe package with --verbose flag
Adds the complete Python port of the Go latprobe tool plus diagnostic verbose output that the Go version does not yet have. Package (python/latprobe/): - probe.py: raw-socket HTTP timing with CertInfo/VerboseDetail dataclasses; TTFB loop accumulates to \r\n\r\n so headers are parseable without changing t_first_byte semantics; captures resolved IP, TLS version/cipher/bits, verified cert (via getpeercert()), and all response headers when verbose=True - aggregate.py: summarize() → per-phase min/avg/max PhaseStats - cli.py: injectable run(args,stdout,stderr)->int; argparse with injected streams; ThreadPoolExecutor concurrency across URLs; four text-rendering branches; JSON output; worst-exit-code accumulation; -v/--verbose flag appends IP/TLS/cert/header block after every timing table; JSON extended with "verbose" object (omitted when flag absent) - duration.py: parse Go-style duration strings (10s, 500ms, 2m, bare seconds) - Exit codes: 0 ok, 1 usage, 2 dns, 3 connect, 4 timeout, 5 tls, 6 http≥400 Tests: - test_probe.py: 18 hermetic tests (success, 404, DNS/connect/timeout/scheme failures, partial-phase invariants, verbose detail fields) - test_cli.py: 38 hermetic tests (usage errors, single/aggregate/failure text, worst-code, --fail, JSON schema/ordering/grouping, verbose text and JSON) - test_integration.py: 45 tests against live internet services (badssl.com for TLS errors, real cert/IP/header validation in verbose mode) Makefile: PYTHONPATH fix, fnmatch pattern test_[!i]*.py to exclude integration tests from py-test, new py-test-integration target Docs: docs/usage/py-latprobe.md, python/configs/usage-latprobe.md (runnable reference with captured output), plans for both the package and verbose feature Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1
python/latprobe/__init__.py
Normal file
1
python/latprobe/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""latprobe — measure per-phase HTTP request latency."""
|
||||
5
python/latprobe/__main__.py
Normal file
5
python/latprobe/__main__.py
Normal file
@@ -0,0 +1,5 @@
|
||||
import sys
|
||||
|
||||
from .cli import run
|
||||
|
||||
sys.exit(run(sys.argv[1:], sys.stdout, sys.stderr))
|
||||
53
python/latprobe/aggregate.py
Normal file
53
python/latprobe/aggregate.py
Normal file
@@ -0,0 +1,53 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from .probe import Result
|
||||
|
||||
|
||||
@dataclass
|
||||
class PhaseStats:
|
||||
min_ms: float = 0.0
|
||||
avg_ms: float = 0.0
|
||||
max_ms: float = 0.0
|
||||
present: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class Aggregate:
|
||||
url: str = ""
|
||||
count: int = 0
|
||||
status_code: int = 0
|
||||
dns: PhaseStats = field(default_factory=PhaseStats)
|
||||
connect: PhaseStats = field(default_factory=PhaseStats)
|
||||
tls: PhaseStats = field(default_factory=PhaseStats)
|
||||
ttfb: PhaseStats = field(default_factory=PhaseStats)
|
||||
transfer: PhaseStats = field(default_factory=PhaseStats)
|
||||
total: PhaseStats = field(default_factory=PhaseStats)
|
||||
|
||||
|
||||
def summarize(results: list[Result]) -> Aggregate:
|
||||
"""Compute per-phase min/avg/max over a list of succeeded Results."""
|
||||
if not results:
|
||||
return Aggregate()
|
||||
a = Aggregate(url=results[0].url, count=len(results))
|
||||
a.status_code = results[-1].status_code
|
||||
|
||||
def _stats(attr: str) -> PhaseStats:
|
||||
vals = [getattr(r, attr).ms for r in results if getattr(r, attr).present]
|
||||
if not vals:
|
||||
return PhaseStats()
|
||||
return PhaseStats(
|
||||
min_ms=min(vals),
|
||||
avg_ms=sum(vals) / len(vals),
|
||||
max_ms=max(vals),
|
||||
present=True,
|
||||
)
|
||||
|
||||
a.dns = _stats("dns")
|
||||
a.connect = _stats("connect")
|
||||
a.tls = _stats("tls")
|
||||
a.ttfb = _stats("ttfb")
|
||||
a.transfer = _stats("transfer")
|
||||
a.total = _stats("total")
|
||||
return a
|
||||
441
python/latprobe/cli.py
Normal file
441
python/latprobe/cli.py
Normal file
@@ -0,0 +1,441 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import datetime
|
||||
import json
|
||||
import sys
|
||||
from typing import IO
|
||||
|
||||
from .aggregate import Aggregate, PhaseStats, summarize
|
||||
from .duration import parse_duration
|
||||
from .probe import Options, Result, VerboseDetail, measure
|
||||
|
||||
# ── exit codes (mirrors Go) ───────────────────────────────────────────────────
|
||||
|
||||
EXIT_OK = 0
|
||||
EXIT_USAGE = 1
|
||||
EXIT_DNS = 2
|
||||
EXIT_CONNECT = 3
|
||||
EXIT_TIMEOUT = 4
|
||||
EXIT_TLS = 5
|
||||
EXIT_HTTP = 6
|
||||
|
||||
_PHASE_EXIT: dict[str, int] = {
|
||||
"dns": EXIT_DNS,
|
||||
"timeout": EXIT_TIMEOUT,
|
||||
"tls": EXIT_TLS,
|
||||
}
|
||||
|
||||
|
||||
def _phase_code(fail_phase: str) -> int:
|
||||
return _PHASE_EXIT.get(fail_phase, EXIT_CONNECT)
|
||||
|
||||
|
||||
# ── display constants ─────────────────────────────────────────────────────────
|
||||
|
||||
_SINGLE_SEP = " " + "─" * 29
|
||||
_AGG_SEP = " " + "─" * 49
|
||||
|
||||
_PHASE_LABELS = [
|
||||
("dns", "DNS lookup "),
|
||||
("connect", "TCP connect "),
|
||||
("tls", "TLS handshake "),
|
||||
("ttfb", "Server (TTFB) "),
|
||||
("transfer", "Transfer "),
|
||||
]
|
||||
|
||||
# Response headers shown in text verbose block, in priority order.
|
||||
# In JSON verbose mode all parsed headers are included.
|
||||
_VERBOSE_HEADERS = [
|
||||
"Location",
|
||||
"Server",
|
||||
"Content-Type",
|
||||
"X-Cache",
|
||||
"CF-Cache-Status",
|
||||
"Cache-Control",
|
||||
"Via",
|
||||
"X-Powered-By",
|
||||
]
|
||||
|
||||
# ── argparse with injectable streams ─────────────────────────────────────────
|
||||
|
||||
|
||||
class _ArgExit(Exception):
|
||||
def __init__(self, code: int) -> None:
|
||||
self.code = code
|
||||
|
||||
|
||||
class _Parser(argparse.ArgumentParser):
|
||||
"""ArgumentParser that writes to injected streams and raises instead of exiting."""
|
||||
|
||||
def __init__(self, *args, out: IO, err: IO, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self._out = out
|
||||
self._err = err
|
||||
|
||||
def _print_message(self, message: str, file=None) -> None:
|
||||
if message:
|
||||
(file if file is not None else self._out).write(message)
|
||||
|
||||
def print_help(self, file=None) -> None:
|
||||
self._print_message(self.format_help(), self._out)
|
||||
|
||||
def print_usage(self, file=None) -> None:
|
||||
self._print_message(self.format_usage(), self._err)
|
||||
|
||||
def error(self, message: str) -> None:
|
||||
self.print_usage()
|
||||
self._err.write(f"{self.prog}: error: {message}\n")
|
||||
raise _ArgExit(EXIT_USAGE)
|
||||
|
||||
def exit(self, status: int = 0, message: str | None = None) -> None:
|
||||
if message:
|
||||
self._err.write(message)
|
||||
raise _ArgExit(int(status) if status else EXIT_OK)
|
||||
|
||||
|
||||
# ── sampling ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _run_samples(
|
||||
url: str, count: int, opts: Options
|
||||
) -> tuple[list[Result], list[Result]]:
|
||||
succeeded, failed = [], []
|
||||
for _ in range(count):
|
||||
r = measure(url, opts)
|
||||
(failed if r.err else succeeded).append(r)
|
||||
return succeeded, failed
|
||||
|
||||
|
||||
# ── verbose rendering ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _cert_expired(expiry: str) -> bool:
|
||||
try:
|
||||
return datetime.date.fromisoformat(expiry) < datetime.date.today()
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
|
||||
def _print_verbose_block(detail: VerboseDetail, out: IO) -> None:
|
||||
"""Write the verbose detail block (IP, TLS, cert, headers) after timing rows."""
|
||||
def _row(label: str, value: str) -> None:
|
||||
out.write(f" {label:<14} : {value}\n")
|
||||
|
||||
if detail.resolved_ip:
|
||||
_row("IP", detail.resolved_ip)
|
||||
|
||||
if detail.tls_version:
|
||||
parts = [detail.tls_version]
|
||||
if detail.tls_cipher:
|
||||
parts.append(detail.tls_cipher)
|
||||
if detail.tls_bits:
|
||||
parts.append(f"{detail.tls_bits} bit")
|
||||
_row("TLS", " ".join(parts))
|
||||
|
||||
if detail.cert:
|
||||
c = detail.cert
|
||||
label = "Cert (unvrf.)" if not c.verified else "Cert"
|
||||
cert_parts = []
|
||||
if c.cn:
|
||||
cert_parts.append(f"CN={c.cn}")
|
||||
if c.expiry:
|
||||
tag = "EXPIRED" if _cert_expired(c.expiry) else "valid until"
|
||||
cert_parts.append(f"{tag} {c.expiry}")
|
||||
if c.issuer_cn:
|
||||
cert_parts.append(c.issuer_cn)
|
||||
_row(label, " ".join(cert_parts))
|
||||
|
||||
for name in _VERBOSE_HEADERS:
|
||||
value = detail.headers.get(name)
|
||||
if value:
|
||||
_row(name, value)
|
||||
|
||||
|
||||
# ── text rendering ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _status_str(status_code: int, fail_flag: bool) -> str:
|
||||
s = str(status_code)
|
||||
if fail_flag and status_code >= 400:
|
||||
s += " ✗"
|
||||
return s
|
||||
|
||||
|
||||
def _print_single(r: Result, fail_flag: bool, out: IO) -> None:
|
||||
status = _status_str(r.status_code, fail_flag)
|
||||
out.write(f"{r.url} ({status})\n")
|
||||
for attr, label in _PHASE_LABELS:
|
||||
ph = getattr(r, attr)
|
||||
if ph.present:
|
||||
out.write(f" {label} : {ph.ms:8.2f} ms\n")
|
||||
out.write(_SINGLE_SEP + "\n")
|
||||
if r.total.present:
|
||||
out.write(f" {'Total '} : {r.total.ms:8.2f} ms\n")
|
||||
if r.err is not None:
|
||||
out.write(f" ✗ {r.fail_phase}: {r.err}\n")
|
||||
if r.detail is not None:
|
||||
_print_verbose_block(r.detail, out)
|
||||
|
||||
|
||||
def _print_aggregate(
|
||||
agg: Aggregate,
|
||||
failed: list[Result],
|
||||
fail_flag: bool,
|
||||
out: IO,
|
||||
detail: VerboseDetail | None = None,
|
||||
) -> None:
|
||||
status = _status_str(agg.status_code, fail_flag)
|
||||
header = f"{agg.url} ({status}, {agg.count} samples"
|
||||
if failed:
|
||||
header += f", {len(failed)} failed"
|
||||
out.write(header + ")\n")
|
||||
|
||||
if agg.total.present:
|
||||
out.write(f" {'':14} {'min':>9} {'avg':>9} {'max':>9}\n")
|
||||
for attr, label in _PHASE_LABELS:
|
||||
ps: PhaseStats = getattr(agg, attr)
|
||||
if ps.present:
|
||||
out.write(
|
||||
f" {label} : {ps.min_ms:6.2f} ms"
|
||||
f" {ps.avg_ms:6.2f} ms"
|
||||
f" {ps.max_ms:6.2f} ms\n"
|
||||
)
|
||||
out.write(_AGG_SEP + "\n")
|
||||
out.write(
|
||||
f" {'Total '} : {agg.total.min_ms:6.2f} ms"
|
||||
f" {agg.total.avg_ms:6.2f} ms"
|
||||
f" {agg.total.max_ms:6.2f} ms\n"
|
||||
)
|
||||
_print_failure_summary(failed, out)
|
||||
if detail is not None:
|
||||
_print_verbose_block(detail, out)
|
||||
|
||||
|
||||
def _print_all_failed(
|
||||
url: str, failed: list[Result], total_count: int, out: IO
|
||||
) -> None:
|
||||
header = f"{url} (FAILED"
|
||||
if total_count > 1:
|
||||
header += f", 0/{total_count} succeeded"
|
||||
out.write(header + ")\n")
|
||||
|
||||
last = failed[-1]
|
||||
any_phase = False
|
||||
for attr, label in _PHASE_LABELS:
|
||||
ph = getattr(last, attr)
|
||||
if ph.present:
|
||||
out.write(f" {label} : {ph.ms:8.2f} ms\n")
|
||||
any_phase = True
|
||||
if last.total.present:
|
||||
if any_phase:
|
||||
out.write(_SINGLE_SEP + "\n")
|
||||
out.write(f" {'Total '} : {last.total.ms:8.2f} ms\n")
|
||||
_print_failure_summary(failed, out)
|
||||
if last.detail is not None:
|
||||
_print_verbose_block(last.detail, out)
|
||||
|
||||
|
||||
def _print_failure_summary(failed: list[Result], out: IO) -> None:
|
||||
if not failed:
|
||||
return
|
||||
counts: dict[tuple[str, str], int] = {}
|
||||
order: list[tuple[str, str]] = []
|
||||
for r in failed:
|
||||
key = (r.fail_phase, str(r.err))
|
||||
if key not in counts:
|
||||
order.append(key)
|
||||
counts[key] = 0
|
||||
counts[key] += 1
|
||||
for phase, msg in order:
|
||||
n = counts[(phase, msg)]
|
||||
if n == 1:
|
||||
out.write(f" ✗ {phase}: {msg}\n")
|
||||
else:
|
||||
out.write(f" ✗ {n} × {phase}: {msg}\n")
|
||||
|
||||
|
||||
def _print_url(
|
||||
url: str,
|
||||
succeeded: list[Result],
|
||||
failed: list[Result],
|
||||
total_count: int,
|
||||
fail_flag: bool,
|
||||
out: IO,
|
||||
) -> None:
|
||||
n_ok = len(succeeded)
|
||||
n_fail = len(failed)
|
||||
last_detail = succeeded[-1].detail if succeeded else None
|
||||
|
||||
if n_fail == 0 and total_count == 1:
|
||||
_print_single(succeeded[0], fail_flag, out)
|
||||
elif n_fail == 0:
|
||||
_print_aggregate(summarize(succeeded), [], fail_flag, out, last_detail)
|
||||
elif n_ok == 0:
|
||||
_print_all_failed(url, failed, total_count, out)
|
||||
else:
|
||||
_print_aggregate(summarize(succeeded), failed, fail_flag, out, last_detail)
|
||||
|
||||
|
||||
# ── JSON rendering ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _build_json_entry(
|
||||
url: str,
|
||||
succeeded: list[Result],
|
||||
failed: list[Result],
|
||||
detail: VerboseDetail | None = None,
|
||||
) -> dict:
|
||||
entry: dict = {
|
||||
"url": url,
|
||||
"status": 0,
|
||||
"succeeded": len(succeeded),
|
||||
"failed": len(failed),
|
||||
}
|
||||
|
||||
if succeeded:
|
||||
agg = summarize(succeeded)
|
||||
entry["status"] = agg.status_code
|
||||
phases: dict = {}
|
||||
for attr in ("dns", "connect", "tls", "ttfb", "transfer", "total"):
|
||||
ps: PhaseStats = getattr(agg, attr)
|
||||
if ps.present:
|
||||
phases[attr] = {
|
||||
"min_ms": ps.min_ms,
|
||||
"avg_ms": ps.avg_ms,
|
||||
"max_ms": ps.max_ms,
|
||||
}
|
||||
if phases:
|
||||
entry["phases"] = phases
|
||||
|
||||
if failed:
|
||||
counts: dict[tuple[str, str], int] = {}
|
||||
order: list[tuple[str, str]] = []
|
||||
for r in failed:
|
||||
key = (r.fail_phase, str(r.err))
|
||||
if key not in counts:
|
||||
order.append(key)
|
||||
counts[key] = 0
|
||||
counts[key] += 1
|
||||
entry["errors"] = [
|
||||
{"phase": ph, "count": counts[(ph, msg)], "message": msg}
|
||||
for ph, msg in order
|
||||
]
|
||||
|
||||
if detail is not None:
|
||||
v: dict = {}
|
||||
if detail.resolved_ip:
|
||||
v["ip"] = detail.resolved_ip
|
||||
if detail.tls_version:
|
||||
v["tls_version"] = detail.tls_version
|
||||
v["tls_cipher"] = detail.tls_cipher
|
||||
v["tls_bits"] = detail.tls_bits
|
||||
if detail.cert:
|
||||
c = detail.cert
|
||||
v["cert"] = {
|
||||
"cn": c.cn,
|
||||
"sans": c.sans,
|
||||
"expiry": c.expiry,
|
||||
"issuer_cn": c.issuer_cn,
|
||||
"verified": c.verified,
|
||||
}
|
||||
if detail.headers:
|
||||
v["headers"] = dict(detail.headers)
|
||||
if v:
|
||||
entry["verbose"] = v
|
||||
|
||||
return entry
|
||||
|
||||
|
||||
# ── entry point ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def run(args: list[str], stdout: IO, stderr: IO) -> int:
|
||||
parser = _Parser(
|
||||
prog="latprobe",
|
||||
description="measure per-phase HTTP request latency",
|
||||
out=stdout,
|
||||
err=stderr,
|
||||
)
|
||||
parser.add_argument("urls", nargs="+", metavar="url")
|
||||
parser.add_argument(
|
||||
"-n", "--count", type=int, default=1, metavar="N",
|
||||
help="number of requests per URL (default: 1)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-c", "--concurrency", type=int, default=0, metavar="N",
|
||||
help="max parallel URLs, 0=auto (default: 0)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timeout", default="10s", metavar="DURATION",
|
||||
help="per-request timeout, e.g. 10s, 500ms (default: 10s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--fail", action="store_true",
|
||||
help="exit non-zero on HTTP status >= 400",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--json", action="store_true", dest="json_out",
|
||||
help="output results as JSON instead of text",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-v", "--verbose", action="store_true",
|
||||
help="show resolved IP, TLS version/cipher, certificate, and response headers",
|
||||
)
|
||||
|
||||
try:
|
||||
ns = parser.parse_args(args)
|
||||
except _ArgExit as exc:
|
||||
return exc.code
|
||||
|
||||
try:
|
||||
timeout_secs = parse_duration(ns.timeout)
|
||||
except ValueError:
|
||||
stderr.write(f"latprobe: error: invalid timeout: {ns.timeout!r}\n")
|
||||
return EXIT_USAGE
|
||||
|
||||
opts = Options(timeout=timeout_secs, verbose=ns.verbose)
|
||||
urls = ns.urls
|
||||
count = ns.count
|
||||
|
||||
workers = ns.concurrency
|
||||
if workers <= 0:
|
||||
workers = min(len(urls), 8)
|
||||
workers = max(1, min(workers, len(urls)))
|
||||
|
||||
def _probe(url: str) -> tuple[list[Result], list[Result]]:
|
||||
return _run_samples(url, count, opts)
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as ex:
|
||||
all_results = list(ex.map(_probe, urls))
|
||||
|
||||
worst = EXIT_OK
|
||||
json_items = []
|
||||
|
||||
for i, (url, (succeeded, failed)) in enumerate(zip(urls, all_results)):
|
||||
for r in failed:
|
||||
c = _phase_code(r.fail_phase)
|
||||
if c > worst:
|
||||
worst = c
|
||||
if ns.fail:
|
||||
for r in succeeded:
|
||||
if r.status_code >= 400:
|
||||
worst = max(worst, EXIT_HTTP)
|
||||
|
||||
last_detail = succeeded[-1].detail if succeeded else (
|
||||
failed[-1].detail if failed else None
|
||||
)
|
||||
|
||||
if ns.json_out:
|
||||
json_items.append(_build_json_entry(url, succeeded, failed, last_detail))
|
||||
continue
|
||||
|
||||
if i > 0:
|
||||
stdout.write("\n")
|
||||
_print_url(url, succeeded, failed, count, ns.fail, stdout)
|
||||
|
||||
if ns.json_out:
|
||||
stdout.write(json.dumps(json_items, indent=2) + "\n")
|
||||
|
||||
return worst
|
||||
14
python/latprobe/duration.py
Normal file
14
python/latprobe/duration.py
Normal file
@@ -0,0 +1,14 @@
|
||||
def parse_duration(s: str) -> float:
|
||||
"""Parse a duration string to seconds.
|
||||
|
||||
Suffixes: ms (milliseconds), s (seconds), m (minutes).
|
||||
A bare number is treated as seconds.
|
||||
"""
|
||||
s = s.strip()
|
||||
if s.endswith("ms"):
|
||||
return float(s[:-2]) / 1000
|
||||
if s.endswith("s"):
|
||||
return float(s[:-1])
|
||||
if s.endswith("m"):
|
||||
return float(s[:-1]) * 60
|
||||
return float(s)
|
||||
306
python/latprobe/probe.py
Normal file
306
python/latprobe/probe.py
Normal file
@@ -0,0 +1,306 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import socket
|
||||
import ssl
|
||||
import time
|
||||
import urllib.parse
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
# ── verbose detail dataclasses ────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass
|
||||
class CertInfo:
|
||||
cn: str = ""
|
||||
sans: list[str] = field(default_factory=list)
|
||||
expiry: str = "" # ISO date "YYYY-MM-DD"
|
||||
issuer_cn: str = ""
|
||||
verified: bool = False # True = TLS handshake passed verification
|
||||
|
||||
|
||||
@dataclass
|
||||
class VerboseDetail:
|
||||
resolved_ip: str = ""
|
||||
tls_version: str = ""
|
||||
tls_cipher: str = ""
|
||||
tls_bits: int = 0
|
||||
cert: CertInfo | None = None
|
||||
headers: dict[str, str] = field(default_factory=dict) # all parsed response headers
|
||||
|
||||
|
||||
# ── core dataclasses ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass
|
||||
class Options:
|
||||
timeout: float = 10.0
|
||||
verbose: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class Phase:
|
||||
ms: float = 0.0
|
||||
present: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class Result:
|
||||
url: str
|
||||
dns: Phase = field(default_factory=Phase)
|
||||
connect: Phase = field(default_factory=Phase)
|
||||
tls: Phase = field(default_factory=Phase)
|
||||
ttfb: Phase = field(default_factory=Phase)
|
||||
transfer: Phase = field(default_factory=Phase)
|
||||
total: Phase = field(default_factory=Phase)
|
||||
status_code: int = 0
|
||||
fail_phase: str = ""
|
||||
err: Exception | None = None
|
||||
detail: VerboseDetail | None = None # populated only when opts.verbose=True
|
||||
|
||||
|
||||
# ── internal helpers ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _p(start: float, end: float) -> Phase:
|
||||
return Phase(ms=(end - start) * 1000, present=True)
|
||||
|
||||
|
||||
def _parse_status(data: bytes) -> int:
|
||||
eol = data.find(b"\r\n")
|
||||
if eol == -1:
|
||||
eol = data.find(b"\n")
|
||||
if eol == -1:
|
||||
return 0
|
||||
parts = data[:eol].decode("latin-1", errors="replace").split(None, 2)
|
||||
if len(parts) >= 2:
|
||||
try:
|
||||
return int(parts[1])
|
||||
except ValueError:
|
||||
pass
|
||||
return 0
|
||||
|
||||
|
||||
def _parse_cert_date(s: str) -> str:
|
||||
"""Convert SSL cert date 'Jun 14 00:00:00 2025 GMT' → '2025-06-14'."""
|
||||
if not s:
|
||||
return ""
|
||||
s = " ".join(s.split()) # collapse double-spaces ("May 5 ..." → "May 5 ...")
|
||||
for fmt in ("%b %d %H:%M:%S %Y %Z", "%b %d %H:%M:%S %Y"):
|
||||
try:
|
||||
return datetime.datetime.strptime(s, fmt).strftime("%Y-%m-%d")
|
||||
except ValueError:
|
||||
pass
|
||||
return ""
|
||||
|
||||
|
||||
def _parse_cert(peer: dict, *, verified: bool) -> CertInfo:
|
||||
"""Build CertInfo from the dict returned by SSLSocket.getpeercert()."""
|
||||
def _attr(rdns, key: str) -> str:
|
||||
for rdn in rdns:
|
||||
for k, v in rdn:
|
||||
if k == key:
|
||||
return v
|
||||
return ""
|
||||
|
||||
issuer = peer.get("issuer", ())
|
||||
# Prefer organizationName for issuer (more human-readable than CA CN)
|
||||
issuer_cn = _attr(issuer, "organizationName") or _attr(issuer, "commonName")
|
||||
|
||||
return CertInfo(
|
||||
cn=_attr(peer.get("subject", ()), "commonName"),
|
||||
sans=[v for k, v in peer.get("subjectAltName", ()) if k == "DNS"],
|
||||
expiry=_parse_cert_date(peer.get("notAfter", "")),
|
||||
issuer_cn=issuer_cn,
|
||||
verified=verified,
|
||||
)
|
||||
|
||||
|
||||
def _parse_response_headers(buf: bytes) -> dict[str, str]:
|
||||
"""Parse all HTTP response headers from a raw buffer (up to \\r\\n\\r\\n)."""
|
||||
header_section = buf.split(b"\r\n\r\n", 1)[0]
|
||||
lines = header_section.decode("latin-1", errors="replace").splitlines()
|
||||
headers: dict[str, str] = {}
|
||||
for line in lines[1:]: # skip status line
|
||||
if ":" in line:
|
||||
name, _, value = line.partition(":")
|
||||
headers[name.strip()] = value.strip()
|
||||
return headers
|
||||
|
||||
|
||||
# ── measure ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def measure(raw_url: str, opts: Options | None = None) -> Result:
|
||||
"""Probe raw_url and return a Result with per-phase timings.
|
||||
|
||||
Partial phases are preserved when the request fails mid-flight.
|
||||
Redirects are not followed; bodies are drained so Transfer timing is real.
|
||||
When opts.verbose=True, Result.detail is populated with resolved IP,
|
||||
TLS metadata, certificate info, and response headers.
|
||||
"""
|
||||
if opts is None:
|
||||
opts = Options()
|
||||
r = Result(url=raw_url)
|
||||
|
||||
if opts.verbose:
|
||||
r.detail = VerboseDetail()
|
||||
|
||||
parsed = urllib.parse.urlparse(raw_url)
|
||||
scheme = parsed.scheme.lower()
|
||||
if scheme not in ("http", "https"):
|
||||
r.fail_phase = "request"
|
||||
r.err = ValueError(f"unsupported scheme: {scheme!r}")
|
||||
return r
|
||||
|
||||
host = parsed.hostname or ""
|
||||
port = parsed.port or (443 if scheme == "https" else 80)
|
||||
path = parsed.path or "/"
|
||||
if parsed.query:
|
||||
path = path + "?" + parsed.query
|
||||
use_tls = scheme == "https"
|
||||
|
||||
t_start = time.perf_counter()
|
||||
|
||||
# ── DNS ──────────────────────────────────────────────────────────────────
|
||||
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
infos = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)
|
||||
except socket.gaierror as exc:
|
||||
r.fail_phase = "dns"
|
||||
r.err = exc
|
||||
r.total = _p(t_start, time.perf_counter())
|
||||
return r
|
||||
r.dns = _p(t0, time.perf_counter())
|
||||
|
||||
if r.detail is not None:
|
||||
r.detail.resolved_ip = str(infos[0][4][0])
|
||||
|
||||
# ── TCP connect ───────────────────────────────────────────────────────────
|
||||
|
||||
addr = infos[0][4]
|
||||
family = infos[0][0]
|
||||
sock = socket.socket(family, socket.SOCK_STREAM)
|
||||
sock.settimeout(opts.timeout)
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
sock.connect(addr)
|
||||
except socket.timeout as exc:
|
||||
sock.close()
|
||||
r.connect = _p(t0, time.perf_counter())
|
||||
r.fail_phase = "timeout"
|
||||
r.err = exc
|
||||
r.total = _p(t_start, time.perf_counter())
|
||||
return r
|
||||
except OSError as exc:
|
||||
sock.close()
|
||||
r.connect = _p(t0, time.perf_counter())
|
||||
r.fail_phase = "connect"
|
||||
r.err = exc
|
||||
r.total = _p(t_start, time.perf_counter())
|
||||
return r
|
||||
r.connect = _p(t0, time.perf_counter())
|
||||
|
||||
# ── TLS handshake (HTTPS only) ────────────────────────────────────────────
|
||||
|
||||
if use_tls:
|
||||
ctx = ssl.create_default_context()
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
sock = ctx.wrap_socket(sock, server_hostname=host)
|
||||
except socket.timeout as exc:
|
||||
r.tls = _p(t0, time.perf_counter())
|
||||
r.fail_phase = "timeout"
|
||||
r.err = exc
|
||||
r.total = _p(t_start, time.perf_counter())
|
||||
return r
|
||||
except (ssl.SSLError, OSError) as exc:
|
||||
r.tls = _p(t0, time.perf_counter())
|
||||
r.fail_phase = "tls"
|
||||
r.err = exc
|
||||
r.total = _p(t_start, time.perf_counter())
|
||||
return r
|
||||
r.tls = _p(t0, time.perf_counter())
|
||||
|
||||
if r.detail is not None:
|
||||
r.detail.tls_version = sock.version() or ""
|
||||
cipher_name, _, bits = sock.cipher()
|
||||
r.detail.tls_cipher = cipher_name or ""
|
||||
r.detail.tls_bits = bits or 0
|
||||
peer = sock.getpeercert()
|
||||
if peer:
|
||||
r.detail.cert = _parse_cert(peer, verified=True)
|
||||
|
||||
# ── Send request ──────────────────────────────────────────────────────────
|
||||
|
||||
request = (
|
||||
f"GET {path} HTTP/1.1\r\n"
|
||||
f"Host: {host}\r\n"
|
||||
f"Connection: close\r\n"
|
||||
f"User-Agent: latprobe/1.0\r\n"
|
||||
f"\r\n"
|
||||
).encode()
|
||||
try:
|
||||
sock.sendall(request)
|
||||
except OSError as exc:
|
||||
sock.close()
|
||||
r.fail_phase = "request"
|
||||
r.err = exc
|
||||
r.total = _p(t_start, time.perf_counter())
|
||||
return r
|
||||
t_wrote = time.perf_counter()
|
||||
|
||||
# ── TTFB — accumulate until end of headers (\r\n\r\n) ────────────────────
|
||||
#
|
||||
# t_first_byte is stamped on the first recv() that returns data (unchanged
|
||||
# semantics vs the old single-recv approach). We keep reading until the
|
||||
# full header section is in buf so that response headers can be parsed when
|
||||
# opts.verbose=True.
|
||||
|
||||
try:
|
||||
buf = b""
|
||||
t_first_byte: float | None = None
|
||||
while b"\r\n\r\n" not in buf:
|
||||
chunk = sock.recv(4096)
|
||||
if not chunk:
|
||||
raise OSError("server closed connection before headers complete")
|
||||
if t_first_byte is None:
|
||||
t_first_byte = time.perf_counter()
|
||||
buf += chunk
|
||||
except socket.timeout as exc:
|
||||
sock.close()
|
||||
r.fail_phase = "timeout"
|
||||
r.err = exc
|
||||
r.total = _p(t_start, time.perf_counter())
|
||||
return r
|
||||
except OSError as exc:
|
||||
sock.close()
|
||||
r.fail_phase = "transfer"
|
||||
r.err = exc
|
||||
r.total = _p(t_start, time.perf_counter())
|
||||
return r
|
||||
|
||||
t_first_byte = t_first_byte or time.perf_counter()
|
||||
r.ttfb = _p(t_wrote, t_first_byte)
|
||||
r.status_code = _parse_status(buf)
|
||||
|
||||
if r.detail is not None:
|
||||
r.detail.headers = _parse_response_headers(buf)
|
||||
|
||||
# ── Transfer — drain remaining body ───────────────────────────────────────
|
||||
|
||||
try:
|
||||
while True:
|
||||
chunk = sock.recv(65536)
|
||||
if not chunk:
|
||||
break
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
t_end = time.perf_counter()
|
||||
r.transfer = _p(t_first_byte, t_end)
|
||||
r.total = _p(t_start, t_end)
|
||||
return r
|
||||
Reference in New Issue
Block a user