Merge pull request 'feat(py): M5.3 — Python /api/* shadow endpoints' (#18) from feat/go-m5-3-python-api-shadow into main

Reviewed-on: #18

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 2026-05-07 22:37 CEST — feat(py): M5.3 — Python /api/* shadow endpoints
+
+- `app.py`: four new JSON routes (`/api/version`, `/api/adults`, `/api/juniors`, `/api/payments`) mirroring the Go `/api/*` handlers; `_unwrap_view_model_for_api()` helper expands pre-serialised JSON strings and renames `month_labels_json` → `month_labels`, `raw_payments_json` → `raw_payments` to match Go wire contract.
+- `tests/test_app.py`: four new smoke tests asserting top-level key sets and that unwrapped fields are objects (not strings).
+
 ## 2026-05-07 20:13 CEST — feat(go): M5.2 — HTTP handlers for /api/adults, /api/juniors, /api/payments, /api/version
 
 - `web/api/handler.go`: `Handler` struct + `ServeAdults`, `ServeJuniors`, `ServePayments`, `ServeVersion` using `membership.Sources`.

app.py

@@ -7,7 +7,7 @@ import os
 import io
 import qrcode
 import logging
-from flask import Flask, render_template, g, send_file, request
+from flask import Flask, render_template, g, send_file, request, jsonify
 
 # Configure logging, allowing override via LOG_LEVEL environment variable
 log_level = os.environ.get("LOG_LEVEL", "INFO").upper()
@@ -68,6 +68,16 @@ BUILD_META = _json.loads(_meta_path.read_text()) if _meta_path.exists() else {
     "tag": "dev", "commit": "local", "build_date": ""
 }
 
+
+def _unwrap_view_model_for_api(vm: dict) -> dict:
+    """Expand pre-stringified JSON fields and rename to match Go API contract."""
+    out = dict(vm)
+    out["member_data"] = _json.loads(out.pop("member_data"))
+    out["month_labels"] = _json.loads(out.pop("month_labels_json"))
+    out["raw_payments"] = _json.loads(out.pop("raw_payments_json"))
+    return out
+
+
 warmup_cache()
 
 @app.before_request
@@ -144,6 +154,75 @@ def sync_bank():
 def version():
     return BUILD_META
 
+@app.route("/api/version")
+def api_version():
+    return jsonify(BUILD_META)
+
+@app.route("/api/adults")
+def api_adults():
+    attendance_url = f"https://docs.google.com/spreadsheets/d/{ATTENDANCE_SHEET_ID}/edit"
+    payments_url = f"https://docs.google.com/spreadsheets/d/{PAYMENTS_SHEET_ID}/edit"
+    members_data = get_cached_data("attendance_regular", ATTENDANCE_SHEET_ID, get_members_with_fees)
+    if not members_data:
+        return jsonify({"error": "no data"}), 503
+    members, sorted_months = members_data
+    transactions = get_cached_data("payments_transactions", PAYMENTS_SHEET_ID, fetch_sheet_data, PAYMENTS_SHEET_ID, CREDENTIALS_PATH)
+    exceptions = get_cached_data(
+        "exceptions_dict", PAYMENTS_SHEET_ID, fetch_exceptions,
+        PAYMENTS_SHEET_ID, CREDENTIALS_PATH,
+        serialize=lambda d: [[list(k), v] for k, v in d.items()],
+        deserialize=lambda c: {tuple(k): v for k, v in c},
+    )
+    result = reconcile(members, sorted_months, transactions, exceptions)
+    vm = build_adults_view_model(
+        members, sorted_months, result, transactions,
+        datetime.now().strftime("%Y-%m"),
+        attendance_url=attendance_url, payments_url=payments_url, bank_account=BANK_ACCOUNT,
+    )
+    return jsonify(_unwrap_view_model_for_api(vm))
+
+@app.route("/api/juniors")
+def api_juniors():
+    attendance_url = f"https://docs.google.com/spreadsheets/d/{ATTENDANCE_SHEET_ID}/edit#gid={JUNIOR_SHEET_GID}"
+    payments_url = f"https://docs.google.com/spreadsheets/d/{PAYMENTS_SHEET_ID}/edit"
+    junior_members_data = get_cached_data("attendance_juniors", ATTENDANCE_SHEET_ID, get_junior_members_with_fees)
+    if not junior_members_data:
+        return jsonify({"error": "no data"}), 503
+    junior_members, sorted_months = junior_members_data
+    transactions = get_cached_data("payments_transactions", PAYMENTS_SHEET_ID, fetch_sheet_data, PAYMENTS_SHEET_ID, CREDENTIALS_PATH)
+    exceptions = get_cached_data(
+        "exceptions_dict", PAYMENTS_SHEET_ID, fetch_exceptions,
+        PAYMENTS_SHEET_ID, CREDENTIALS_PATH,
+        serialize=lambda d: [[list(k), v] for k, v in d.items()],
+        deserialize=lambda c: {tuple(k): v for k, v in c},
+    )
+    adapted_members = adapt_junior_members(junior_members)
+    result = reconcile(adapted_members, sorted_months, transactions, exceptions)
+    vm = build_juniors_view_model(
+        junior_members, adapted_members, sorted_months, result, transactions,
+        datetime.now().strftime("%Y-%m"),
+        attendance_url=attendance_url, payments_url=payments_url, bank_account=BANK_ACCOUNT,
+    )
+    return jsonify(_unwrap_view_model_for_api(vm))
+
+@app.route("/api/payments")
+def api_payments():
+    attendance_url = f"https://docs.google.com/spreadsheets/d/{ATTENDANCE_SHEET_ID}/edit"
+    payments_url = f"https://docs.google.com/spreadsheets/d/{PAYMENTS_SHEET_ID}/edit"
+    transactions = get_cached_data("payments_transactions", PAYMENTS_SHEET_ID, fetch_sheet_data, PAYMENTS_SHEET_ID, CREDENTIALS_PATH)
+    adults_data = get_cached_data("attendance_regular", ATTENDANCE_SHEET_ID, get_members_with_fees)
+    juniors_data = get_cached_data("attendance_juniors", ATTENDANCE_SHEET_ID, get_junior_members_with_fees)
+    member_names = []
+    if adults_data:
+        member_names.extend(name for name, _, _ in adults_data[0])
+    if juniors_data:
+        member_names.extend(name for name, _, _ in juniors_data[0])
+    vm = build_payments_view_model(
+        transactions, member_names,
+        attendance_url=attendance_url, payments_url=payments_url,
+    )
+    return jsonify(vm)
+
 @app.route("/adults")
 def adults_view():
     attendance_url = f"https://docs.google.com/spreadsheets/d/{ATTENDANCE_SHEET_ID}/edit"

docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md

@@ -99,7 +99,7 @@ Goal: byte-equal JSON between Python and Go for every route. This is the parity
 
 - [x] **M5.1** Hand-author Go structs for `/api/adults`, `/api/juniors`, `/api/payments`, `/api/version` with explicit `json:` tags matching Python keys; emit JSON Schemas via `github.com/invopop/jsonschema` to `tests/fixtures/api-schema/` — `f253e3f`
 - [x] **M5.2** Implement Go handlers for `/api/*` routes composing `services/*` results into the JSON structs — `7d48e8f`
-- [ ] **M5.3** Add Python `/api/X` shadow endpoints in [app.py](app.py): `jsonify(view_model_dict)` — no transformation
+- [x] **M5.3** Add Python `/api/X` shadow endpoints in [app.py](app.py): `jsonify(view_model_dict)` — no transformation — `40e4a9e`
 - [ ] **M5.4** Build `cmd/parity/main.go`: hits both backends' `/api/X`, normalizes allowlist (`render_time.total`, `build_meta`), prints `cmp.Diff`. Add `make parity` target
 
 **Gate:** For each route, `make parity` reports zero non-allowlisted diffs across the M3 fixture corpus.

docs/plans/2026-05-07-2114-go-rewrite-m5-3-python-api-shadow.md

@@ -0,0 +1,113 @@
+# M5.3 — Python `/api/X` shadow endpoints
+
+Companion to:
+- [2026-05-03-2349-go-backend-rewrite.md](2026-05-03-2349-go-backend-rewrite.md) (master design)
+- [2026-05-03-2349-go-backend-rewrite-progress.md](2026-05-03-2349-go-backend-rewrite-progress.md) (M5.3 row)
+- [2026-05-07-1431-m5-json-api-parity.md](2026-05-07-1431-m5-json-api-parity.md) (Python view-model extraction prep)
+- [2026-05-07-1650-go-rewrite-m5-1-api-structs-schemas.md](2026-05-07-1650-go-rewrite-m5-1-api-structs-schemas.md) (Go wire types)
+
+## Context
+
+M5.1 (Go wire types + JSON Schemas) and M5.2 (Go HTTP handlers for `/api/adults` `/api/juniors` `/api/payments` `/api/version`) have merged. M5.3 mirrors the same four endpoints on the Python Flask side so M5.4's `cmd/parity` tool can hit both backends and diff the JSON. After M5.3, every byte the Go side emits has a Python counterpart to compare against.
+
+The Python view-model builders ([scripts/views.py](scripts/views.py)) already produce dicts very close to the wire shape — except three template-only fields (`member_data`, `month_labels_json`, `raw_payments_json`) are pre-`json.dumps`'d for inline `<script>` blocks. M5.1's plan called this out explicitly: M5.3's `/api/X` is `jsonify(unwrap_json_strings(view_model_dict))` — a 4-line shim, not real transformation logic.
+
+## Approach
+
+Add four shadow routes to [app.py](app.py) and one private unwrap helper. Builders and templates are untouched.
+
+### Decisions
+
+1. **Unwrap shim lives in `app.py`**, not `scripts/views.py`. It's 4 lines, only the API routes use it, and it's parity-only scaffolding that M8 will delete. Keeping `views.py` free of HTTP-layer concerns means cleaner deletion later.
+2. **No data-loading helper extraction.** Each shadow route duplicates ~8 lines of cache loads from its sibling HTML route. A helper would have to thread `attendance_url` / `payments_url` / `bank_account` and the adults-vs-juniors-vs-payments branching back out — net negative for code that M8 will erase wholesale.
+3. **Drop `record_step` calls in API routes.** `record_step` only feeds `inject_render_time` (a Jinja `context_processor`); JSON responses don't go through templates, so timing breakdown has no consumer.
+4. **`/api/version` is a one-liner.** `BUILD_META` already has the keys (`tag`, `commit`, `build_date`) Go emits. Just `jsonify(BUILD_META)`. Existing `/version` route stays as-is — the new endpoint sits alongside.
+5. **Tests assert key sets and unwrap, not values.** Hard-code `EXPECTED_ADULTS_KEYS` etc. as module constants in [tests/test_app.py](tests/test_app.py). Catches drift in unit tests rather than waiting for M5.4 parity diffs. Type-check the unwrapped fields (`isinstance(member_data, dict)` etc.) to prove the shim ran.
+
+### The shim
+
+```python
+def _unwrap_view_model_for_api(vm: dict) -> dict:
+    out = dict(vm)
+    out["member_data"] = _json.loads(out.pop("member_data"))
+    out["month_labels"] = _json.loads(out.pop("month_labels_json"))
+    out["raw_payments"] = _json.loads(out.pop("raw_payments_json"))
+    return out
+```
+
+Note the **rename**: `month_labels_json` → `month_labels`, `raw_payments_json` → `raw_payments` (matches Go contract per [adults.go](go/internal/web/api/adults.go) JSON tags).
+
+### Route skeleton (`/api/adults`)
+
+```python
+@app.route("/api/adults")
+def api_adults():
+    attendance_url = f"https://docs.google.com/spreadsheets/d/{ATTENDANCE_SHEET_ID}/edit"
+    payments_url = f"https://docs.google.com/spreadsheets/d/{PAYMENTS_SHEET_ID}/edit"
+    members_data = get_cached_data("attendance_regular", ATTENDANCE_SHEET_ID, get_members_with_fees)
+    if not members_data:
+        return jsonify({"error": "no data"}), 503
+    members, sorted_months = members_data
+    transactions = get_cached_data("payments_transactions", PAYMENTS_SHEET_ID, fetch_sheet_data, PAYMENTS_SHEET_ID, CREDENTIALS_PATH)
+    exceptions = get_cached_data(
+        "exceptions_dict", PAYMENTS_SHEET_ID, fetch_exceptions,
+        PAYMENTS_SHEET_ID, CREDENTIALS_PATH,
+        serialize=lambda d: [[list(k), v] for k, v in d.items()],
+        deserialize=lambda c: {tuple(k): v for k, v in c},
+    )
+    result = reconcile(members, sorted_months, transactions, exceptions)
+    vm = build_adults_view_model(
+        members, sorted_months, result, transactions,
+        datetime.now().strftime("%Y-%m"),
+        attendance_url=attendance_url, payments_url=payments_url, bank_account=BANK_ACCOUNT,
+    )
+    return jsonify(_unwrap_view_model_for_api(vm))
+```
+
+`/api/juniors` mirrors with `adapt_junior_members` + `JUNIOR_SHEET_GID` + `build_juniors_view_model`. `/api/payments` skips the unwrap (its builder has no JSON-string fields): `return jsonify(vm)`. `/api/version`: `return jsonify(BUILD_META)`.
+
+## Files to modify
+
+- [app.py](app.py)
+  - L10: add `jsonify` to `from flask import ...`.
+  - Add `_unwrap_view_model_for_api` near [BUILD_META](app.py#L67) (it already imports `json as _json`).
+  - Add four routes: `/api/version`, `/api/adults`, `/api/juniors`, `/api/payments`. Place them after [`/version`](app.py#L143) for grouping.
+- [tests/test_app.py](tests/test_app.py)
+  - Add `EXPECTED_ADULTS_KEYS`, `EXPECTED_JUNIORS_KEYS`, `EXPECTED_PAYMENTS_KEYS`, `EXPECTED_VERSION_KEYS` module constants (sourced from [adults.go](go/internal/web/api/adults.go) / [juniors.go](go/internal/web/api/juniors.go) / [payments.go](go/internal/web/api/payments.go) / [version.go](go/internal/web/api/version.go) JSON tags).
+  - Four new test functions: `test_api_adults`, `test_api_juniors`, `test_api_payments`, `test_api_version`. Reuse the existing `_bypass_cache` patcher and the `fetch_sheet_data` / `fetch_exceptions` / `get_members_with_fees` / `get_junior_members_with_fees` mocks already in the file.
+  - Each adults/juniors test asserts: `200`, `response.is_json`, `set(json.keys()) == EXPECTED_*_KEYS`, `isinstance(json["member_data"], dict)`, `isinstance(json["month_labels"], dict)`, `isinstance(json["raw_payments"], dict)`.
+- [CHANGELOG.md](CHANGELOG.md): post-merge entry per CLAUDE.md format.
+- [docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md:102](docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md#L102): tick M5.3 with merge SHA.
+
+## Reusable existing code
+
+- [`build_adults_view_model`](scripts/views.py#L64), [`build_juniors_view_model`](scripts/views.py#L240), [`build_payments_view_model`](scripts/views.py#L432) — call as-is, no changes.
+- [`reconcile`](scripts/match_payments.py) and [`adapt_junior_members`](scripts/views.py#L48) — same.
+- [`get_cached_data`](app.py#L36), [`fetch_sheet_data`](scripts/match_payments.py), [`fetch_exceptions`](scripts/match_payments.py), [`get_members_with_fees`](scripts/attendance.py), [`get_junior_members_with_fees`](scripts/attendance.py) — same call sites as the HTML routes.
+- [`BUILD_META`](app.py#L67) — already shaped to match Go's `VersionResponse`.
+
+## Verification
+
+1. `make test` — all existing tests still pass; four new `test_api_*` tests pass.
+2. `python -c "import app"` — catches the missing `jsonify` import.
+3. `make web-py`, then:
+   - `curl -s localhost:5001/api/version | jq .` → `{tag, commit, build_date}`.
+   - `curl -s localhost:5001/api/adults | jq 'keys'` → 14 keys, no `_json` suffix anywhere.
+   - `curl -s localhost:5001/api/adults | jq '.member_data | type, .month_labels | type, .raw_payments | type'` → all `"object"` (proves unwrap).
+   - Same checks on `/api/juniors` and `/api/payments`.
+4. Visit `/adults`, `/juniors`, `/payments` in browser — HTML still renders identically (regression check on builders).
+5. **Optional pre-M5.4 peek:** `make web-go` on :8080 + `make web-py` on :5001, then `diff <(curl -s :5001/api/adults | jq -S .) <(curl -s :8080/api/adults | jq -S .)`. Expect non-zero diff (raw transaction key shape, see below) — that is fine; M5.4 surfaces and resolves these.
+
+## Out of scope (M5.4 will surface and resolve)
+
+These are known parity friction points; **don't fix in M5.3** — the whole point of M5.4 is to enumerate them:
+
+- **`raw_payments[name][i]` row shape**: Python emits raw Google Sheets row dicts with column-header keys (e.g. `"VS"`, `"Sync ID"`); Go's `RawTransaction` uses snake_case (`vs`, `sync_id`). Keys and types will diverge.
+- **`unmatched[]`**: same divergence (same raw row dicts).
+- **`null` vs missing keys**: Python omits keys never set; Go zero-value structs may emit `null`/`""` depending on `omitempty`.
+- **`Decimal` / float precision** in `grouped_payments` amounts.
+- **Field insertion order**: `jsonify` preserves insertion order (Python 3.7+); Go marshals struct fields in declaration order. Likely fine, parity tool will tell.
+
+## Branch + MR
+
+Per CLAUDE.md: branch `feat/go-m5-3-python-api-shadow`, push with `-u`, open MR via `tea pr create --title ... --description ... --base main --head feat/go-m5-3-python-api-shadow`. Do not merge from CLI.

tests/test_app.py

@@ -1,7 +1,17 @@
 import unittest
+import json
 from unittest.mock import patch
 from app import app
 
+EXPECTED_ADULTS_KEYS = {
+    "months", "raw_months", "results", "totals", "member_data", "month_labels",
+    "raw_payments", "credits", "debts", "unmatched", "attendance_url",
+    "payments_url", "bank_account", "current_month",
+}
+EXPECTED_JUNIORS_KEYS = EXPECTED_ADULTS_KEYS
+EXPECTED_PAYMENTS_KEYS = {"grouped_payments", "sorted_people", "attendance_url", "payments_url"}
+EXPECTED_VERSION_KEYS = {"tag", "commit", "build_date"}
+
 
 def _bypass_cache(cache_key, sheet_id, fetch_func, *args, serialize=None, deserialize=None, **kwargs):
     """Test helper: call fetch_func directly, bypassing the cache layer."""
@@ -97,5 +107,80 @@ class TestWebApp(unittest.TestCase):
         self.assertIn(b'500/500 CZK', response.data)
         self.assertIn(b'?', response.data)
 
+    def test_api_version(self):
+        """Test /api/version returns BUILD_META keys as JSON."""
+        response = self.client.get('/api/version')
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(response.is_json)
+        data = json.loads(response.data)
+        self.assertEqual(set(data.keys()), EXPECTED_VERSION_KEYS)
+
+    @patch('app.get_cached_data', side_effect=_bypass_cache)
+    @patch('app.fetch_sheet_data')
+    @patch('app.fetch_exceptions', return_value={})
+    @patch('app.get_members_with_fees')
+    def test_api_adults(self, mock_get_members, mock_exceptions, mock_fetch_sheet, mock_cache):
+        """Test /api/adults returns JSON with correct top-level keys and unwrapped fields."""
+        mock_get_members.return_value = (
+            [('Test Member', 'A', {'2026-01': (750, 4)})],
+            ['2026-01']
+        )
+        mock_fetch_sheet.return_value = [{
+            'date': '2026-01-01', 'amount': 750, 'person': 'Test Member',
+            'purpose': '2026-01', 'message': 'test payment',
+            'sender': 'External Bank User', 'inferred_amount': 750,
+        }]
+        response = self.client.get('/api/adults')
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(response.is_json)
+        data = json.loads(response.data)
+        self.assertEqual(set(data.keys()), EXPECTED_ADULTS_KEYS)
+        self.assertIsInstance(data['member_data'], dict)
+        self.assertIsInstance(data['month_labels'], dict)
+        self.assertIsInstance(data['raw_payments'], dict)
+
+    @patch('app.get_cached_data', side_effect=_bypass_cache)
+    @patch('app.fetch_sheet_data')
+    @patch('app.fetch_exceptions', return_value={})
+    @patch('app.get_junior_members_with_fees')
+    def test_api_juniors(self, mock_get_junior_members, mock_exceptions, mock_fetch_sheet, mock_cache):
+        """Test /api/juniors returns JSON with correct top-level keys and unwrapped fields."""
+        mock_get_junior_members.return_value = (
+            [
+                ('Junior One', 'J', {'2026-01': (500, 3, 0, 3)}),
+                ('Junior Two', 'X', {'2026-01': ('?', 1, 0, 1)}),
+            ],
+            ['2026-01']
+        )
+        mock_fetch_sheet.return_value = [{
+            'date': '2026-01-15', 'amount': 500, 'person': 'Junior One',
+            'purpose': '2026-01', 'message': '', 'sender': 'Parent', 'inferred_amount': 500,
+        }]
+        response = self.client.get('/api/juniors')
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(response.is_json)
+        data = json.loads(response.data)
+        self.assertEqual(set(data.keys()), EXPECTED_JUNIORS_KEYS)
+        self.assertIsInstance(data['member_data'], dict)
+        self.assertIsInstance(data['month_labels'], dict)
+        self.assertIsInstance(data['raw_payments'], dict)
+
+    @patch('app.get_cached_data', side_effect=_bypass_cache)
+    @patch('app.fetch_sheet_data')
+    def test_api_payments(self, mock_fetch_sheet, mock_cache):
+        """Test /api/payments returns JSON with correct top-level keys."""
+        mock_fetch_sheet.return_value = [{
+            'date': '2026-01-01', 'amount': 750, 'person': 'Test Member',
+            'purpose': '2026-01', 'message': 'test', 'sender': 'Someone',
+        }]
+        response = self.client.get('/api/payments')
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(response.is_json)
+        data = json.loads(response.data)
+        self.assertEqual(set(data.keys()), EXPECTED_PAYMENTS_KEYS)
+        self.assertIsInstance(data['grouped_payments'], dict)
+        self.assertIsInstance(data['sorted_people'], list)
+
+
 if __name__ == '__main__':
     unittest.main()