Compare commits
83 Commits
0.19
...
fix/py-pay
| Author | SHA1 | Date | |
|---|---|---|---|
| 65694ad378 | |||
| 092dff25a5 | |||
| 56c21bcf03 | |||
| 208f762c18 | |||
| 4d035213b5 | |||
| 2b15280d03 | |||
| 723152cdad | |||
| fe0e49a134 | |||
| e5a272b682 | |||
| 8b3064ffab | |||
| 423c3e2a4b | |||
| f4c497681f | |||
| 40e4a9e45e | |||
| 68810369bd | |||
| 2b7eff14c4 | |||
| 7d48e8f607 | |||
| be4ecef20f | |||
| da5b82fcdb | |||
| f253e3fcb1 | |||
| 59223c0da4 | |||
| 32a16ff50d | |||
| 2eec51bb34 | |||
| b562ce3201 | |||
| f0de300292 | |||
| 2164e99866 | |||
| b41b8ef29c | |||
| 80db33945d | |||
| f87adeff9f | |||
| a7cf45fc95 | |||
| f0a0f79475 | |||
| fcb83691f5 | |||
| 8275db1a63 | |||
| 36a28a40d2 | |||
| 6465e2a221 | |||
| 7afd12d9a5 | |||
| 57518a8a68 | |||
| 67d2f11d7c | |||
| 28f0e468f7 | |||
| 8386af8078 | |||
| 56aa2303a8 | |||
| ea8622a541 | |||
| 71278e6f7a | |||
| 34ce0be5a0 | |||
| c5a8a4e7b1 | |||
| 3e597242eb | |||
| 7232697e9c | |||
| e596f0000e | |||
| c2bffed1b8 | |||
| 54a783ea00 | |||
| 84a5d177e9 | |||
| 1a63bfd313 | |||
| d24d20553a | |||
| fa853780db | |||
| 0fc3b6dd9a | |||
| 57ec817044 | |||
| 6cf83a01e3 | |||
| 98f401c149 | |||
| 0a8017fffa | |||
| 6d971b61d4 | |||
| 3460f57c62 | |||
| 6ca35e2112 | |||
| 20ade6de3e | |||
| d9a61b338c | |||
| 91ac3b37cf | |||
| 394da2e6b8 | |||
| 81b36878b3 | |||
| 97f568f49f | |||
| cf0f176d3f | |||
| 5a41cdae83 | |||
| dfdf2aacb8 | |||
| ced238385e | |||
| 77743019b0 | |||
| f712198319 | |||
| 1ac5df7be5 | |||
| 109ef983f0 | |||
| 083a51023c | |||
| 54762cd421 | |||
| b2aaca5df9 | |||
| 883bc4489e | |||
| 3ad4a21f5b | |||
| 3c1604c7af | |||
| 8b3223f865 | |||
| 276e18a9c8 |
16
.claude/settings.json
Normal file
16
.claude/settings.json
Normal file
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"permissions": {
|
||||
"allow": [
|
||||
"Bash(git add:*)",
|
||||
"Bash(go version *)",
|
||||
"Bash(go mod *)",
|
||||
"Bash(golangci-lint run *)",
|
||||
"Bash(golangci-lint --version)",
|
||||
"Bash(gofumpt *)",
|
||||
"Bash(./bin/fuj help *)",
|
||||
"Bash(./bin/fuj version *)",
|
||||
"Bash(make go-test *)",
|
||||
"Bash(make go-lint *)"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -31,5 +31,35 @@ jobs:
|
||||
TAG=${{ inputs.tag }}
|
||||
fi
|
||||
IMAGE=gitea.home.hrajfrisbee.cz/${{ github.repository }}:$TAG
|
||||
docker build -f build/Dockerfile -t $IMAGE .
|
||||
docker build -f build/Dockerfile \
|
||||
--build-arg GIT_TAG=$TAG \
|
||||
--build-arg GIT_COMMIT=${{ github.sha }} \
|
||||
--build-arg BUILD_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) \
|
||||
-t $IMAGE .
|
||||
docker push $IMAGE
|
||||
|
||||
build-go:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Login to Gitea registry
|
||||
run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login -u ${{ github.actor }} --password-stdin gitea.home.hrajfrisbee.cz
|
||||
|
||||
- name: Build and push Go image
|
||||
run: |
|
||||
TAG=${{ github.ref_name }}
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
TAG=${{ inputs.tag }}
|
||||
fi
|
||||
IMAGE=gitea.home.hrajfrisbee.cz/${{ github.repository }}:$TAG-go
|
||||
docker build -f go/build/Dockerfile \
|
||||
--build-arg GIT_TAG=$TAG \
|
||||
--build-arg GIT_COMMIT=${{ github.sha }} \
|
||||
--build-arg BUILD_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) \
|
||||
-t $IMAGE go/
|
||||
docker push $IMAGE
|
||||
|
||||
3
.gitignore
vendored
3
.gitignore
vendored
@@ -4,3 +4,6 @@
|
||||
|
||||
# local tmp folder
|
||||
tmp/
|
||||
|
||||
# go build output
|
||||
bin/
|
||||
|
||||
187
CHANGELOG.md
Normal file
187
CHANGELOG.md
Normal file
@@ -0,0 +1,187 @@
|
||||
# Changelog
|
||||
|
||||
## 2026-05-07 23:37 CEST — fix(go): accept single-digit day/month in attendance date headers
|
||||
|
||||
- `go/internal/services/membership/sources.go`: `parseDates` now uses Go time formats `2.1.2006` and `1/2/2006` (single-digit reference forms, which accept both padded and unpadded inputs) instead of `02.01.2006` and `01/02/2006`. The Czech attendance sheet headers contain dates like `1.6.2026`, `23.3.2026`, `6.4.2026` — Go silently dropped those columns under the strict zero-padded format, while Python's `strptime("%d.%m.%Y")` accepted them. Effect was a missing `2026-06` month entirely on `/api/juniors` plus undercounted attendance for any month with single-digit columns; both surfaced as diffs in `make parity`.
|
||||
- `sources_test.go::TestParseDates_SingleDigitDayMonth` added as a regression guard covering both Czech and US format flavours with and without leading zeros.
|
||||
|
||||
## 2026-05-07 23:17 CEST — fix(go): pass raw value to FormatDate so numeric serial-day dates format
|
||||
|
||||
- `go/internal/services/membership/sources.go`: transaction-row parser now passes `row[idxDate]` directly to `matching.FormatDate` (via a new `getRaw` helper) instead of stringifying first via `getVal`. The Sheets API returns numeric serial-day values as `float64` for date-formatted cells; pre-stringifying them defeated `FormatDate`'s `case float64:` dispatch, causing all numeric dates to leak through as `"46147"` style strings instead of `"2026-05-05"`.
|
||||
- Surfaced by `make parity` (M5.4): every `transactions[].date` field on `/api/adults` and `/api/juniors` differed between Python and Go.
|
||||
- `sources_test.go::TestLoadTransactions` extended with a numeric-serial-day row covering the regression.
|
||||
|
||||
## 2026-05-07 23:05 CEST — fix(go): default CacheDir to `tmp/go` to avoid Python collision
|
||||
|
||||
- `go/internal/config/config.go`: `CacheDir` default changed from `tmp` to `tmp/go`. Override via `CACHE_DIR` env var still works.
|
||||
- Why: both backends used `tmp/<key>_cache.json` with the same keys (`attendance_regular`, `attendance_juniors`, `payments_transactions`, `exceptions_dict`) but different shapes — Python caches post-processed view-model tuples, Go caches raw rows. Whichever wrote last poisoned the cache; running both in parallel produced `ValueError: too many values to unpack (expected 2, got 68)` on Python's `/adults` after the Go server populated `attendance_regular_cache.json` with raw CSV rows.
|
||||
- After upgrading: stop the Go server, hit `/flush-cache` on the Python side once (rewrites `tmp/*.json` with correct shapes), then restart `make web-go` — it will use `tmp/go/` going forward. Required for the M5.4 `make parity` workflow which assumes both backends run side-by-side.
|
||||
|
||||
## 2026-05-07 22:55 CEST — feat(go): M5.4 — parity diff binary + `make parity`
|
||||
|
||||
- `go/cmd/parity/main.go`: new standalone binary that GETs `/api/adults`, `/api/juniors`, `/api/payments` from both Python (:5001) and Go (:8080) backends, scrubs an allowlist (`render_time.total`), and prints `cmp.Diff` for any remaining differences. Exits 0 on full match, 1 on diffs, 2 on fetch/parse errors — CI-friendly for M7.2. `/api/version` is excluded by design (returns binary identity — tag/commit/build_date — which differs between independently built backends); still accessible via `make parity ARGS="-route /api/version"`.
|
||||
- `go/cmd/parity/scrub_test.go`: 4 unit tests covering top-level delete, nested delete, missing path, and non-map parent.
|
||||
- `go/go.mod`: `github.com/google/go-cmp` promoted to direct dependency.
|
||||
- `Makefile`: `parity` target added (`.PHONY`, help, `cd go && go run ./cmd/parity`).
|
||||
- `docs/plans/2026-05-07-2254-m5-4-parity-binary.md`: plan archived.
|
||||
|
||||
## 2026-05-07 22:37 CEST — feat(py): M5.3 — Python /api/* shadow endpoints
|
||||
|
||||
- `app.py`: four new JSON routes (`/api/version`, `/api/adults`, `/api/juniors`, `/api/payments`) mirroring the Go `/api/*` handlers; `_unwrap_view_model_for_api()` helper expands pre-serialised JSON strings and renames `month_labels_json` → `month_labels`, `raw_payments_json` → `raw_payments` to match Go wire contract.
|
||||
- `tests/test_app.py`: four new smoke tests asserting top-level key sets and that unwrapped fields are objects (not strings).
|
||||
|
||||
## 2026-05-07 20:13 CEST — feat(go): M5.2 — HTTP handlers for /api/adults, /api/juniors, /api/payments, /api/version
|
||||
|
||||
- `web/api/handler.go`: `Handler` struct + `ServeAdults`, `ServeJuniors`, `ServePayments`, `ServeVersion` using `membership.Sources`.
|
||||
- `web/api/build_{adults,juniors,payments,common}.go`: ports of `scripts/views.py` view-model builders; `buildJuniorMemberRow` handles `"?"` sentinel, `:NJ,MA` breakdown, unknown-month skip.
|
||||
- Extended `reconcile.FeeData`/`MonthData` with `IsUnknown`, `JuniorAttendance`, `AdultAttendance`; `Transaction` with `ManualFix`, `VS`, `BankID`, `SyncID`.
|
||||
- `sources.go` exports `AdultMergedMonths`/`JuniorMergedMonths`; parses new FeeData and transaction columns.
|
||||
- `web/server.go` + `cmd/fuj/main.go` wired to register `/api/*` routes.
|
||||
- PR #17.
|
||||
|
||||
## 2026-05-07 17:37 CEST — feat(go): M5.1 — /api/* wire types + JSON Schemas
|
||||
|
||||
- New `go/internal/web/api/` package: `AdultsResponse`, `JuniorsResponse`, `PaymentsResponse`, `VersionResponse` with explicit `json:` tags matching Python view-model keys.
|
||||
- `Expected{Value int; Unknown bool}` custom `MarshalJSON` emits integer or `"?"` for junior single-attendance months.
|
||||
- `schemagen_test.go` golden-tests four JSON Schemas committed to `go/tests/fixtures/api-schema/`. `JSONSchema()` on `Expected` lives in the test file — production binary has no jsonschema dep.
|
||||
- PR #16.
|
||||
|
||||
## 2026-05-07 15:26 CEST — refactor(app): extract view-model builders into scripts/views.py
|
||||
|
||||
- Pulled ~350 lines of inline per-row computation out of `adults_view`, `juniors_view`, and `payments` into three pure functions in `scripts/views.py`: `build_adults_view_model`, `build_juniors_view_model`, `build_payments_view_model`.
|
||||
- Moved `get_month_labels`, `group_payments_by_person`, `adapt_junior_members` from `app.py` to `scripts/views.py`. Route handlers now ~25 lines each.
|
||||
- Hotfixed missing `import re` that caused 500 on `/qr` after the refactor.
|
||||
- No behaviour change; all 27 tests pass. Prep for `/api/*` shadow endpoints (M5).
|
||||
|
||||
## 2026-05-07 14:13 CEST — feat(go): --print-fio-table + Fio debug logging + date parser fix
|
||||
|
||||
- Added `--print-fio-table` flag to `fuj sync --dry-run`: prints an aligned table of every Fio transaction in the window with `STATUS=NEW/DUP`, using `text/tabwriter`. Key files: `go/internal/services/banksync/fio_table.go`, `sync.go`, `cmd/fuj/main.go`.
|
||||
- Added `LOG_LEVEL=DEBUG` debug logging on the Fio fetch path: client variant selected, full GET URL (token redacted on API path), HTTP status, body bytes, and per-parse drop-reason counters (`raw_rows`, `kept`, `dropped_bad_date`, `dropped_nonpositive_amount`). Key files: `go/internal/io/fio/{client,api,transparent}.go`.
|
||||
- Fixed `parseCzechDate` to accept `DD.MM.YY` (2-digit year) in addition to the 4-digit variant — Fio's transparent page now serves this format. Key file: `go/internal/io/fio/transparent.go`.
|
||||
- Added `make go-sync-debug [DAYS=N]` Makefile target (default 30 days).
|
||||
|
||||
## 2026-05-07 10:32 CEST — feat(go): --dry-run for fuj sync
|
||||
|
||||
- `SyncOpts.DryRun bool` added; when true, `SyncToSheets` prints planned writes (`would write header row`, `would append date=… amount=… sender=…`, `would sort by date`) and returns without calling `WriteHeader`, `AppendValues`, or `SortByDateColumn`.
|
||||
- `fuj sync --dry-run` flag wired in `cmd/fuj/main.go`; mirrors existing `fuj infer --dry-run` behaviour.
|
||||
- `TestSyncToSheets_DryRun` added to banksync test suite.
|
||||
|
||||
## 2026-05-07 01:03 CEST — feat(go/M4): IO layer behind interfaces
|
||||
|
||||
- `go/internal/io/attendance`: CSV-over-public-URL client + `Fake` for both adult and junior tabs.
|
||||
- `go/internal/io/drive`: thin Drive v3 wrapper for `modifiedTime` reads + `Fake`.
|
||||
- `go/internal/io/sheets`: Sheets v4 client (`GetValues`, `AppendValues`, `BatchUpdateValues`, `WriteHeader`, `SortByDateColumn`) + `Fake` with call-capture for assertions.
|
||||
- `go/internal/io/cache`: Drive-modifiedTime-gated `FileCache` with two TTL knobs, atomic writes, and generic `Get[T]`; Python-compatible JSON format; `Flush()` support.
|
||||
- `go/internal/io/fio`: `Client` interface backed by Fio REST API (`apiClient`) and HTML-scraper (`transparentClient`); `Fake` for tests. Fixtures in `testdata/`.
|
||||
- `go/internal/services/membership/sources.go`: `NewSources` wires attendance CSV + Sheets + cache into `LoadAdults`, `LoadJuniors`, `LoadTransactions`, `LoadExceptions`. Includes Czech month/merged-month parsing logic.
|
||||
- `go/internal/services/banksync`: `SyncToSheets` (dedup via SHA-256 Sync ID, optional sort) and `InferPayments` (name-match + `[?]` review prefix, dry-run) — fully tested with fakes.
|
||||
- `go/cmd/fuj/main.go`: `sync` and `infer` subcommands wired to real clients; `fees` and `reconcile` now use real `NewSources`.
|
||||
- All packages lint-clean (golangci-lint v1.64.8, gofumpt extra-rules).
|
||||
|
||||
## 2026-05-06 23:25 CEST — feat(go/M3): fixture capture + parity test framework
|
||||
|
||||
- `scripts/capture_fixtures.py`: dispatcher CLI that calls each ported function with seeded inputs and emits captured output as JSON fixtures.
|
||||
- `scripts/scrub_fixtures.py`: deterministic PII scrubber (SHA-256 pseudonyms, digit-preserving account/VS hashes, name-sweep in free text).
|
||||
- `scripts/_fixture_seeds.py`: handcrafted seed registry for all 10 pure functions + 10 reconcile branch-coverage cases.
|
||||
- 98 fixture files committed under `go/tests/fixtures/pure/<func>/` and `go/tests/fixtures/reconcile/`; all PII-free.
|
||||
- `go/tests/parity/parityio.go`: shared loader with generic `LoadDir`/`RunAll` helpers and typed `In`/`Out` structs for all 10 functions.
|
||||
- 11 parity test packages under `//go:build parity`: 10 pure-function tests + bespoke reconcile test with per-cell float tolerance.
|
||||
- Makefile: `go-parity`, `go-test-all`, `capture-fixtures` targets.
|
||||
- `go/tests/fixtures/README.md`: refresh workflow, PII audit guide, adding-a-fixture steps.
|
||||
|
||||
## 2026-05-06 17:49 CEST — feat(go/M2.11-12): wire fuj fees + fuj reconcile subcommands
|
||||
|
||||
- New `go/internal/services/membership` package: `AttendanceLoader`, `TransactionLoader`, `ExceptionLoader` interfaces, a stub (`NewStubSources`) that returns `ErrIOPending`, and `FeesReport` / `ReconcileReport` orchestration functions backed by real `domain/fees` + `domain/reconcile` logic.
|
||||
- Text formatters `printFeesTable` / `printReconcileReport` port the output of `calculate_fees.py` and `match_payments.py print_report` verbatim.
|
||||
- `cmd/fuj/main.go`: `fuj fees` and `fuj reconcile` subcommands now dispatch properly; `fuj sync` / `fuj infer` retain the [M4] placeholder.
|
||||
- Both subcommands exit 1 with a clean `"io layer not yet wired up; lands in milestone M4"` message until real Sheets loaders are injected in M4.
|
||||
- 13 unit tests covering stubs, all formatter branches (OK/partial/UNPAID/dash cells, credits, debts, unmatched, review annotation), and orchestration wiring via fake loaders.
|
||||
|
||||
## 2026-05-06 16:38 CEST — fix: include juniors in payment-inference roster
|
||||
|
||||
- `scripts/infer_payments.py`: union adults + junior rosters so junior-only members are visible to the matcher.
|
||||
- Root cause: `get_members_with_fees()` reads only the adults sheet; junior-only kids like Jáchym Kubík were absent from `member_names`, causing the exact-match short-circuit to never fire and a different adult sharing the first name to win via fuzzy review.
|
||||
- Two regression tests added to `tests/test_match_members.py`.
|
||||
|
||||
## 2026-05-06 16:05 CEST — feat(go/M2.10): port domain/reconcile.Reconcile
|
||||
|
||||
- New `go/internal/domain/reconcile` package porting the three-phase payment allocation from `scripts/match_payments.py reconcile()`.
|
||||
- 12 unit tests covering all Python test cases plus Go-only extras (diacritics tolerance, `[?]` stripping, `other:` purpose, out-of-window credit, inference fallback, unmatched, no-transaction guard).
|
||||
|
||||
## 2026-05-06 13:18 CEST — feat(go/M2.7-2.9): port domain/matching package
|
||||
|
||||
- New `go/internal/domain/matching` package porting three helpers from `scripts/match_payments.py`.
|
||||
- `BuildNameVariants` — extracts normalized ASCII search variants from a member name, including nickname (from parens) and separate first/last; filters variants shorter than 3 chars; `variants[0]` is always the full normalized base name.
|
||||
- `MatchMembers` — finds members in free text with `"auto"` or `"review"` confidence; exact-name short-circuit prevents nickname substrings (e.g. `tov`) from matching inside surnames (e.g. `ottova`).
|
||||
- `FormatDate` — normalizes Google Sheets date values: handles nil, empty, int/float64 serial-days since 1899-12-30 (supports fractional serials), pre-formatted `YYYY-MM-DD` strings, and garbage input — never errors.
|
||||
- `InferTransactionDetails` — composes name + month matching over sender/message/user_id; falls back to sender-only member match and date-derived month when text gives no signal.
|
||||
- 21 table-driven tests; all expected values verified against live Python on 2026-05-06.
|
||||
|
||||
## 2026-05-06 12:43 CEST — feat(go/M2.6): port domain/synch.GenerateSyncID
|
||||
|
||||
- New `go/internal/domain/synch` package with `GenerateSyncID(Transaction) string` ported from `scripts/sync_fio_to_sheets.py` `generate_sync_id`.
|
||||
- Byte-stable SHA-256 hash over `date|amount|currency|sender|vs|message|bank_id` (lowercased, UTF-8); `Currency: ""` defaults to `"CZK"` matching the Python missing-key fallback.
|
||||
- Key subtlety: Python's `str(float)` emits `"500.0"` for whole-valued floats and switches to scientific notation at `|f| >= 1e16` or `|f| < 1e-4` — replicated in `formatAmount` using `'f'`/`'e'` format selection.
|
||||
- 6 table-driven hash tests + 9 `formatAmount` tests; all expected values verified against live Python on 2026-05-06.
|
||||
|
||||
## 2026-05-06 09:38 CEST — feat(go/M2.5): port domain/money.ParseCZK
|
||||
|
||||
- New `go/internal/domain/money` package with `ParseCZK(string) (float64, error)` ported from `scripts/infer_payments.py` `parse_czk_amount`.
|
||||
- Preserves the Czech-locale heuristic: comma → decimal sep; 2+ dots → thousand seps; single dot → decimal (so `"1.500"` → `1.5`).
|
||||
- Returns `(0, ErrInvalidAmount)` on parse failure; callers wanting Python's silent-zero contract use `v, _ := ParseCZK(s)`.
|
||||
- 15 table-driven tests plus a silent-zero contract test; all expected values verified against live Python on 2026-05-06.
|
||||
|
||||
## 2026-05-06 09:24 CEST — feat(go/M2.3+M2.4): port domain/fees.CalculateFee and CalculateJuniorFee
|
||||
|
||||
- New `go/internal/domain/fees` package with adult and junior fee calculators ported from `scripts/attendance.py`.
|
||||
- `CalculateFee(count, monthKey) int` — `0→0`, `1→200`, `2+→AdultFeeMonthlyRate[month]` (fallback 700 CZK).
|
||||
- `CalculateJuniorFee(count, monthKey) Expected` — `0→{0}`, `1→{Unknown:true}` (the `"?"` sentinel, now strictly typed), `2+→JuniorFeeMonthlyRate[month]` (fallback 500 CZK).
|
||||
- 20 table-driven tests, all verified against live Python; `-race` clean; `golangci-lint` clean.
|
||||
|
||||
## 2026-05-06 00:07 CEST — feat(go/M2.2): port czech.ParseMonthReferences
|
||||
|
||||
- `internal/domain/czech.ParseMonthReferences`: three-pass regex (numeric slash, dot, Czech month names) with range wrap-around and `m≥10 → previousYear` heuristic, byte-equivalent to Python.
|
||||
- 35 table-driven tests; all expected outputs verified against live Python before locking (addresses risk #4 from the rewrite plan).
|
||||
|
||||
## 2026-05-05 23:33 CEST — feat(go/M2.1): port czech.Normalize
|
||||
|
||||
- First M2 pure-domain task: `internal/domain/czech.Normalize` (NFKD + Mn-strip + lowercase), byte-equivalent to Python `czech_utils.normalize`.
|
||||
- Adds `golang.org/x/text v0.36.0` as first external Go dependency.
|
||||
- 13-case table-driven test, all spot-checked against Python before locking.
|
||||
|
||||
## 2026-05-04 23:08 CEST — fix: payment inference exact-match short-circuit
|
||||
|
||||
- `match_members()` now short-circuits on whole-word full-name hits; nickname/partial checks only run when no full name is present.
|
||||
- Replaced bare `in` substring checks with `_word_in()` word-boundary regex throughout, closing the class of bugs where a short nickname (e.g. `tov`) matches inside another member's surname (`ottova`).
|
||||
- Added `tests/test_match_members.py` (6 cases). Affects `scripts/match_payments.py`.
|
||||
|
||||
## 2026-05-04 23:08 CEST — feat: lower adult monthly fee to 700 CZK from April 2026
|
||||
|
||||
- `ADULT_FEE_DEFAULT` reduced from 750 → 700 CZK.
|
||||
- `ADULT_FEE_MONTHLY_RATE` now pins Sep 2025 – Feb 2026 at 750 to preserve historical billing; Mar 2026 stays 350; Apr–May 2026 at 700. Affects `scripts/attendance.py`.
|
||||
|
||||
## 2026-05-04 12:02 CEST — Go rewrite M1: skeleton + tooling
|
||||
|
||||
- Created `go/` tree with module `fuj-management/go` (Go 1.26).
|
||||
- `cmd/fuj`: stdlib-flag subcommand dispatcher; `server` and `version` implemented, stubs for M2/M4 commands.
|
||||
- `internal/config`: env loader mirroring `scripts/config.py` (same env var names and defaults).
|
||||
- `internal/logging`: slog setup accepting log level from config.
|
||||
- `internal/web`: `net/http` ServeMux on `:8080`; `middleware/timer.go` logs method/path/status/ms.
|
||||
- `go/build/Dockerfile`: multi-stage (`golang:1.26` → `alpine:3`) producing a static binary image.
|
||||
- Makefile: `web` → `web-py` alias; added `web-go`, `go-build`, `go-test`, `go-run`, `go-lint`.
|
||||
- `.gitea/workflows/build.yaml`: parallel `build-go` job pushing `<tag>-go` image.
|
||||
- Gate: `make go-build`, `make go-lint`, `make go-test`, `curl :8080` all pass.
|
||||
|
||||
## 2026-05-03 20:37 CEST — Fix Balance column to correctly reflect past-month debt
|
||||
|
||||
- Balance (and Pay-All) are now computed as `sum(paid − expected)` over past months only, iterating directly over the ledger entries from `reconcile()`.
|
||||
- Previously the balance used `total_balance` (which includes current/future-month activity and out-of-window credits) plus a one-sided current-month debt adjustment. Current-month *surplus* leaked through, making the balance appear less negative than the actual past-month debt.
|
||||
- Pay-All is now `max(0, −balance)` so the two values are derived from a single source and can never disagree.
|
||||
- Affected: `adults_view()` and `juniors_view()` in `app.py`.
|
||||
|
||||
## 2026-05-03 19:26 CEST — Fee-aware allocation for multi-month payments
|
||||
|
||||
- `reconcile()` no longer splits a multi-month payment evenly. Allocation is now per-member with two phases: greedy (if amount ≥ total expected, each month gets exactly its expected fee and overflow → credit) and proportional (otherwise distribute by each month's expected). Fixes the case where e.g. 1250 CZK covering 3 months with mixed fees (750/350/150) marked two months red.
|
||||
- Out-of-window months keep the previous even-split-to-credit behavior. Fallback to even split when all matched months have `expected = 0` (prepayment before attendance is recorded).
|
||||
- Display layer only — no changes to how payments are stored in Google Sheets; `Inferred Amount` still holds the full bank amount.
|
||||
- Files: [scripts/match_payments.py](scripts/match_payments.py), [tests/test_reconcile_exceptions.py](tests/test_reconcile_exceptions.py) (6 new test cases).
|
||||
144
CLAUDE.md
144
CLAUDE.md
@@ -16,22 +16,144 @@ Flask-based financial management system for FUJ (Frisbee Ultimate Jablonec). Han
|
||||
This project uses `uv` for dependency management.
|
||||
|
||||
```bash
|
||||
uv venv # Create virtual environment
|
||||
uv sync # Install dependencies from pyproject.toml
|
||||
uv venv && uv sync
|
||||
source .venv/bin/activate
|
||||
```
|
||||
|
||||
Alternatively, use the Makefile:
|
||||
- `make sync` - Sync bank transactions to Google Sheets
|
||||
- `make infer` - Automatically infer Person/Purpose/Amount in the sheet
|
||||
- `make reconcile` - Generate balance report from Google Sheets data
|
||||
- `make fees` - Calculate expected fees from attendance
|
||||
- `make match` - (Legacy) Match bank data directly
|
||||
- `make web` - Start dashboard
|
||||
- `make image` - Build Docker image
|
||||
Set `PYTHONPATH=scripts:.` when running scripts directly (the Makefile does this automatically).
|
||||
|
||||
Requires `.secret/fuj-management-bot-credentials.json` for Google Sheets API access (configurable via `CREDENTIALS_PATH` env var).
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
make web # Start dashboard at http://localhost:5001
|
||||
make web-debug # Same with FLASK_DEBUG=1
|
||||
make test # Run all tests (unittest discover)
|
||||
make test-v # Tests with verbose output
|
||||
make fees # Print fee report from attendance sheet
|
||||
make sync-2026 # Sync Fio bank transactions for 2026 to Google Sheets
|
||||
make infer # Auto-fill Person/Purpose/Amount columns in payments sheet
|
||||
make reconcile # Print balance report from Google Sheets data
|
||||
make image # Build Docker image
|
||||
```
|
||||
|
||||
Run a single test:
|
||||
```bash
|
||||
PYTHONPATH=scripts:. python -m unittest tests.test_app.TestWebApp.test_adults_route
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
### Data flow
|
||||
|
||||
```
|
||||
Google Sheets (attendance) ──► attendance.py ──► reconcile() ──► Flask routes ──► templates/
|
||||
Google Sheets (payments) ──► match_payments.py ──┘
|
||||
Fio Bank API ──► sync_fio_to_sheets.py ──► Google Sheets (payments)
|
||||
```
|
||||
|
||||
### Key modules
|
||||
|
||||
- `app.py` — Flask app; routes for `/adults`, `/juniors`, `/payments`, `/sync-bank`, `/qr`, `/flush-cache`
|
||||
- `scripts/attendance.py` — Fetches attendance CSV from Google Sheets, computes per-member per-month fees. Contains fee rate constants (`ADULT_FEE_DEFAULT`, `JUNIOR_FEE_DEFAULT`) and `ADULT_MERGED_MONTHS` / `JUNIOR_MERGED_MONTHS` dicts.
|
||||
- `scripts/match_payments.py` — `reconcile()` matches transactions to members/months. `fetch_sheet_data()` reads the payments sheet. `fetch_exceptions()` reads the `exceptions` tab.
|
||||
- `scripts/cache_utils.py` — Invalidation via Google Drive API `modifiedTime`; falls back to 5-minute TTL buckets when Drive API is unavailable. Cache files live in `tmp/`.
|
||||
- `scripts/sync_fio_to_sheets.py` — Pulls Fio bank transactions and appends them to the payments Google Sheet.
|
||||
- `scripts/infer_payments.py` — Fills in Person/Purpose/Inferred Amount columns using name-matching heuristics.
|
||||
- `scripts/config.py` — All external IDs, paths, and tunable TTLs. Override via env vars (`CREDENTIALS_PATH`, `BANK_ACCOUNT`, `CACHE_TTL_SECONDS`).
|
||||
|
||||
### Member tiers
|
||||
|
||||
Tiers are set in column B of the attendance sheet:
|
||||
- `A` — Adult, pays fees (per-month rate from `ADULT_FEE_MONTHLY_RATE`, fallback 700 CZK for 2+ sessions; 200 CZK for exactly 1)
|
||||
- `J` — Junior attending adult practices; their attendance is merged with the junior sheet
|
||||
- `X` — Excluded from junior fee calculation (coaches, etc.)
|
||||
|
||||
### Fee calculation
|
||||
|
||||
- Adults: 0 sessions → 0, 1 session → 200 CZK, 2+ sessions → monthly rate (default 700 CZK)
|
||||
- Juniors: 0 → 0, 1 → `"?"` (manual review required), 2+ → monthly rate (default 500 CZK)
|
||||
- Per-member per-month overrides live in the `exceptions` tab of the payments sheet (columns: Name, Period YYYY-MM, Amount, Note). Exceptions are keyed by `(normalize(name), normalize(period))`.
|
||||
|
||||
### Merged months
|
||||
|
||||
`ADULT_MERGED_MONTHS` / `JUNIOR_MERGED_MONTHS` in `attendance.py` map a source month to a target month (e.g., `"2025-12": "2026-01"` merges December into January billing). The target month accumulates attendance from both months.
|
||||
|
||||
### Caching
|
||||
|
||||
`get_cached_data()` in `app.py` checks the Drive API `modifiedTime` before each request and serves a JSON file from `tmp/` when the sheet hasn't changed. Cache is warmed up at startup (`warmup_cache()`). Flush via `/flush-cache` (POST) or `flush_cache()`.
|
||||
|
||||
### Payments sheet columns
|
||||
|
||||
`Date | Amount | manual fix | Person | Purpose | Inferred Amount | Sender | VS | Message | Bank ID | Sync ID`
|
||||
|
||||
`Person` and `Purpose` are written by `infer_payments.py` and can be manually corrected. `manual fix` column presence disables re-inference for that row. Multiple people or months are comma-separated in Person/Purpose.
|
||||
|
||||
### QR codes
|
||||
|
||||
`/qr?account=…&amount=…&message=…` generates a Czech QR Platba PNG (SPD format).
|
||||
|
||||
## Branching & merge requests
|
||||
|
||||
The remote is Gitea (`gitea.home.hrajfrisbee.cz/kacerr/fuj-management`).
|
||||
For **features**, do not commit to `main` directly. Use a branch + merge
|
||||
request flow:
|
||||
|
||||
1. **Create a branch off `main`** before starting work:
|
||||
- `feat/<slug>` for features (e.g. `feat/qr-code-overlay`)
|
||||
- `fix/<slug>` for bug-fix branches the user explicitly asks for
|
||||
- `<slug>` is short kebab-case
|
||||
2. **Commit on the branch** following the existing commit conventions
|
||||
(Co-Authored-By trailer, etc.).
|
||||
3. **Push the branch** to `origin` with `-u` so it tracks.
|
||||
4. **Open the MR with `tea`** rather than printing a compare URL:
|
||||
|
||||
```bash
|
||||
tea pr create \
|
||||
--title "<short title>" \
|
||||
--description "<body>" \
|
||||
--base main \
|
||||
--head <branch>
|
||||
```
|
||||
|
||||
`tea` is already authenticated against the Gitea instance; just run it.
|
||||
Print the resulting PR URL for the user. If `tea` is unavailable for
|
||||
some reason, fall back to printing the compare URL
|
||||
(`https://gitea.home.hrajfrisbee.cz/kacerr/fuj-management/compare/main...<branch>`)
|
||||
and let the user open the MR manually.
|
||||
5. **Do not merge or delete the branch** from the CLI — neither via `tea`,
|
||||
`gh`, nor `git push --delete`. The user does that in Gitea.
|
||||
|
||||
**Exceptions — when committing straight to `main` is fine:**
|
||||
- Small bug fixes / hotfixes the user describes as such.
|
||||
- Typo / comment / formatting tweaks.
|
||||
- Edits the user explicitly says to push to `main`.
|
||||
|
||||
When uncertain whether something is a feature or a small fix, ask before
|
||||
committing.
|
||||
|
||||
## Git Commits
|
||||
|
||||
When making git commits, always append yourself as co-author trailer to the end of the commit message to indicate AI assistance
|
||||
|
||||
## Changelog
|
||||
|
||||
Maintain a running changelog in `CHANGELOG.md` at the repo root. After every significant change, fix, or update — once the user confirms it works — append a new entry **at the top** of the file in this format:
|
||||
|
||||
```markdown
|
||||
## YYYY-MM-DD HH:MM TZ — short title
|
||||
|
||||
- One-line summary of what changed and why.
|
||||
- Key files touched (optional, only if useful for traceability).
|
||||
```
|
||||
|
||||
Get the timestamp with `date "+%Y-%m-%d %H:%M %Z"`. Skip trivial edits (typos, formatting, comment tweaks); only log changes a future reader would care about.
|
||||
|
||||
## Plans
|
||||
|
||||
When Claude Code's plan mode is used, save the plan file inside the repo at
|
||||
`docs/plans/YYYY-MM-DD-HHMM-<slug>.md` instead of the default `~/.claude/plans/`
|
||||
location. Get the timestamp with `date "+%Y-%m-%d-%H%M"` (matches the changelog
|
||||
convention). The `<slug>` should be a short kebab-case summary of the plan's topic.
|
||||
|
||||
Create the `docs/plans/` directory on first use. Plan files are committed to the
|
||||
repo so other contributors can review historical decisions.
|
||||
|
||||
84
Makefile
84
Makefile
@@ -1,10 +1,13 @@
|
||||
.PHONY: help fees match web web-debug image run sync sync-2026 test test-v docs
|
||||
.PHONY: help fees match web web-py web-debug web-go go-build go-test go-test-all go-parity go-run go-sync-debug go-lint capture-fixtures parity image run sync sync-2026 test test-v docs
|
||||
|
||||
export PYTHONPATH := scripts:$(PYTHONPATH)
|
||||
VENV := .venv
|
||||
PYTHON := $(VENV)/bin/python3
|
||||
CREDENTIALS := .secret/fuj-management-bot-credentials.json
|
||||
|
||||
GO_SRC := go
|
||||
GO_BIN := bin/fuj
|
||||
|
||||
$(PYTHON): .venv/.last_sync
|
||||
|
||||
.venv/.last_sync: pyproject.toml
|
||||
@@ -15,18 +18,28 @@ help:
|
||||
@echo "Available targets:"
|
||||
@echo " make fees - Calculate monthly fees from the attendance sheet"
|
||||
@echo " make match - Match Fio bank payments against expected attendance fees"
|
||||
@echo " make web - Start a dynamic web dashboard locally"
|
||||
@echo " make web-debug - Start a dynamic web dashboard locally in debug mode"
|
||||
@echo " make image - Build an OCI container image"
|
||||
@echo " make run - Run the built Docker image locally"
|
||||
@echo " make web - Start Python dashboard (alias for web-py, until M8)"
|
||||
@echo " make web-py - Start Python dashboard on :5001"
|
||||
@echo " make web-go - Build and start Go dashboard on :8080"
|
||||
@echo " make web-debug - Start Python dashboard in debug mode"
|
||||
@echo " make go-build - Build Go binary to bin/fuj"
|
||||
@echo " make go-test - Run Go unit tests"
|
||||
@echo " make go-parity - Run Go parity tests (requires -tags=parity fixture corpus)"
|
||||
@echo " make go-test-all - Run both unit and parity tests"
|
||||
@echo " make go-lint - Run golangci-lint on Go code"
|
||||
@echo " make go-sync-debug [DAYS=N] - Dry-run Go sync with Fio debug logs and txn table (default DAYS=30)"
|
||||
@echo " make capture-fixtures - Regenerate parity fixture corpus from live Python"
|
||||
@echo " make parity - Diff /api/* between web-py (:5001) and web-go (:8080); both must be running"
|
||||
@echo " make image - Build Python OCI container image"
|
||||
@echo " make run - Run the built Python Docker image locally"
|
||||
@echo " make sync - Sync Fio transactions to Google Sheets"
|
||||
@echo " make sync-2025 - Sync Fio transactions for Q4 2025 (Oct-Dec)"
|
||||
@echo " make sync-2026 - Sync Fio transactions for the whole year of 2026"
|
||||
@echo " make infer - Infer payment details (Person, Purpose, Amount) in the sheet"
|
||||
@echo " make reconcile - Show balance report using Google Sheets data"
|
||||
@echo " make venv - Sync virtual environment with pyproject.toml"
|
||||
@echo " make test - Run web application infrastructure tests"
|
||||
@echo " make test-v - Run tests with verbose output"
|
||||
@echo " make test - Run Python web application infrastructure tests"
|
||||
@echo " make test-v - Run Python tests with verbose output"
|
||||
@echo " make docs - Serve documentation in a browser"
|
||||
|
||||
venv:
|
||||
@@ -38,14 +51,67 @@ fees: $(PYTHON)
|
||||
match: $(PYTHON)
|
||||
$(PYTHON) scripts/match_payments.py
|
||||
|
||||
web: $(PYTHON)
|
||||
web: web-py
|
||||
|
||||
web-py: $(PYTHON)
|
||||
$(PYTHON) app.py
|
||||
|
||||
web-debug: $(PYTHON)
|
||||
FLASK_DEBUG=1 $(PYTHON) app.py
|
||||
|
||||
go-build:
|
||||
cd $(GO_SRC) && go build -trimpath \
|
||||
-ldflags "-X main.version=$$(git describe --tags --always 2>/dev/null || echo dev) \
|
||||
-X main.commit=$$(git rev-parse --short HEAD) \
|
||||
-X main.buildDate=$$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
|
||||
-o ../$(GO_BIN) ./cmd/fuj
|
||||
|
||||
go-test:
|
||||
cd $(GO_SRC) && go test -race ./...
|
||||
|
||||
go-parity:
|
||||
cd $(GO_SRC) && go test -tags=parity ./tests/parity/...
|
||||
|
||||
go-test-all: go-test go-parity
|
||||
|
||||
capture-fixtures: $(PYTHON)
|
||||
@echo "Capturing and scrubbing fixtures for all registered functions..."
|
||||
@for func in normalize parse_month_references calculate_fee calculate_junior_fee \
|
||||
parse_czk_amount generate_sync_id build_name_variants match_members \
|
||||
infer_transaction_details format_date reconcile; do \
|
||||
dir="go/tests/fixtures/$$([[ $$func == reconcile ]] && echo reconcile || echo pure/$$func)"; \
|
||||
mkdir -p "$$dir"; \
|
||||
PYTHONPATH=scripts:. $(PYTHON) scripts/capture_fixtures.py --func $$func --all \
|
||||
| while IFS= read -r line; do \
|
||||
case_id=$$(echo "$$line" | $(PYTHON) -c "import sys,json; print(json.load(sys.stdin)['case'])"); \
|
||||
echo "$$line" | $(PYTHON) scripts/scrub_fixtures.py > "$$dir/$${case_id}.json"; \
|
||||
done; \
|
||||
echo " $$func done"; \
|
||||
done
|
||||
@echo "capture-fixtures complete."
|
||||
|
||||
go-run: go-build
|
||||
./$(GO_BIN) $(ARGS)
|
||||
|
||||
DAYS ?= 30
|
||||
go-sync-debug: go-build
|
||||
LOG_LEVEL=DEBUG ./$(GO_BIN) sync -dry-run -print-fio-table -days $(DAYS)
|
||||
|
||||
go-lint:
|
||||
cd $(GO_SRC) && golangci-lint run ./...
|
||||
|
||||
web-go: go-build
|
||||
./$(GO_BIN) server
|
||||
|
||||
parity:
|
||||
cd $(GO_SRC) && go run ./cmd/parity $(ARGS)
|
||||
|
||||
image:
|
||||
docker build -t fuj-management:latest -f build/Dockerfile .
|
||||
docker build -t fuj-management:latest \
|
||||
--build-arg GIT_TAG=$$(git describe --tags --always 2>/dev/null || echo "untagged") \
|
||||
--build-arg GIT_COMMIT=$$(git rev-parse --short HEAD) \
|
||||
--build-arg BUILD_DATE=$$(date -u +%Y-%m-%dT%H:%M:%SZ) \
|
||||
-f build/Dockerfile .
|
||||
|
||||
run:
|
||||
docker run -it --rm -p 5001:5001 -v $(CURDIR)/.secret:/app/.secret:ro fuj-management:latest
|
||||
|
||||
777
app.py
777
app.py
@@ -7,7 +7,7 @@ import os
|
||||
import io
|
||||
import qrcode
|
||||
import logging
|
||||
from flask import Flask, render_template, g, send_file, request
|
||||
from flask import Flask, render_template, g, send_file, request, jsonify
|
||||
|
||||
# Configure logging, allowing override via LOG_LEVEL environment variable
|
||||
log_level = os.environ.get("LOG_LEVEL", "INFO").upper()
|
||||
@@ -21,9 +21,17 @@ from config import (
|
||||
ATTENDANCE_SHEET_ID, PAYMENTS_SHEET_ID, JUNIOR_SHEET_GID,
|
||||
BANK_ACCOUNT, CREDENTIALS_PATH,
|
||||
)
|
||||
from attendance import get_members_with_fees, get_junior_members_with_fees, ADULT_MERGED_MONTHS, JUNIOR_MERGED_MONTHS
|
||||
from match_payments import reconcile, fetch_sheet_data, fetch_exceptions, normalize
|
||||
from cache_utils import get_sheet_modified_time, read_cache, write_cache, _LAST_CHECKED
|
||||
from attendance import get_members_with_fees, get_junior_members_with_fees
|
||||
from match_payments import reconcile, fetch_sheet_data, fetch_exceptions
|
||||
from views import (
|
||||
build_adults_view_model,
|
||||
build_juniors_view_model,
|
||||
build_payments_view_model,
|
||||
adapt_junior_members,
|
||||
)
|
||||
from cache_utils import get_sheet_modified_time, read_cache, write_cache, _LAST_CHECKED, flush_cache
|
||||
from sync_fio_to_sheets import sync_to_sheets
|
||||
from infer_payments import infer_payments
|
||||
|
||||
def get_cached_data(cache_key, sheet_id, fetch_func, *args, serialize=None, deserialize=None, **kwargs):
|
||||
mod_time = get_sheet_modified_time(cache_key)
|
||||
@@ -36,25 +44,6 @@ def get_cached_data(cache_key, sheet_id, fetch_func, *args, serialize=None, dese
|
||||
write_cache(cache_key, mod_time, serialize(data) if serialize else data)
|
||||
return data
|
||||
|
||||
def get_month_labels(sorted_months, merged_months):
|
||||
labels = {}
|
||||
for m in sorted_months:
|
||||
dt = datetime.strptime(m, "%Y-%m")
|
||||
# Find which months were merged into m (e.g. 2026-01 is merged into 2026-02)
|
||||
merged_in = sorted([k for k, v in merged_months.items() if v == m])
|
||||
if merged_in:
|
||||
all_dts = [datetime.strptime(x, "%Y-%m") for x in sorted(merged_in + [m])]
|
||||
years = {d.year for d in all_dts}
|
||||
if len(years) > 1:
|
||||
parts = [d.strftime("%b %Y") for d in all_dts]
|
||||
labels[m] = "+".join(parts)
|
||||
else:
|
||||
parts = [d.strftime("%b") for d in all_dts]
|
||||
labels[m] = f"{'+'.join(parts)} {dt.strftime('%Y')}"
|
||||
else:
|
||||
labels[m] = dt.strftime("%b %Y")
|
||||
return labels
|
||||
|
||||
def warmup_cache():
|
||||
"""Pre-fetch all cached data so first request is fast."""
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -72,6 +61,23 @@ def warmup_cache():
|
||||
logger.info("Cache warmup complete.")
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
import json as _json
|
||||
_meta_path = Path(__file__).parent / "build_meta.json"
|
||||
BUILD_META = _json.loads(_meta_path.read_text()) if _meta_path.exists() else {
|
||||
"tag": "dev", "commit": "local", "build_date": ""
|
||||
}
|
||||
|
||||
|
||||
def _unwrap_view_model_for_api(vm: dict) -> dict:
|
||||
"""Expand pre-stringified JSON fields and rename to match Go API contract."""
|
||||
out = dict(vm)
|
||||
out["member_data"] = _json.loads(out.pop("member_data"))
|
||||
out["month_labels"] = _json.loads(out.pop("month_labels_json"))
|
||||
out["raw_payments"] = _json.loads(out.pop("raw_payments_json"))
|
||||
return out
|
||||
|
||||
|
||||
warmup_cache()
|
||||
|
||||
@app.before_request
|
||||
@@ -101,157 +107,121 @@ def inject_render_time():
|
||||
"total": f"{total:.3f}",
|
||||
"breakdown": " | ".join(breakdown)
|
||||
}
|
||||
return dict(get_render_time=get_render_time)
|
||||
return dict(get_render_time=get_render_time, build_meta=BUILD_META)
|
||||
|
||||
@app.route("/")
|
||||
def index():
|
||||
# Redirect root to /adults for convenience while there are no other apps
|
||||
return '<meta http-equiv="refresh" content="0; url=/adults" />'
|
||||
|
||||
@app.route("/fees")
|
||||
def fees():
|
||||
@app.route("/flush-cache", methods=["GET", "POST"])
|
||||
def flush_cache_endpoint():
|
||||
if request.method == "GET":
|
||||
return render_template("flush-cache.html")
|
||||
deleted = flush_cache()
|
||||
return render_template("flush-cache.html", flushed=True, deleted=deleted)
|
||||
|
||||
@app.route("/sync-bank")
|
||||
def sync_bank():
|
||||
import contextlib
|
||||
output = io.StringIO()
|
||||
success = True
|
||||
try:
|
||||
with contextlib.redirect_stdout(output), contextlib.redirect_stderr(output):
|
||||
# sync_to_sheets: equivalent of make sync-2026
|
||||
output.write("=== Syncing Fio transactions (2026) ===\n")
|
||||
sync_to_sheets(
|
||||
spreadsheet_id=PAYMENTS_SHEET_ID,
|
||||
credentials_path=CREDENTIALS_PATH,
|
||||
date_from_str="2026-01-01",
|
||||
date_to_str="2026-12-31",
|
||||
sort_by_date=True,
|
||||
)
|
||||
output.write("\n=== Inferring payment details ===\n")
|
||||
infer_payments(PAYMENTS_SHEET_ID, CREDENTIALS_PATH)
|
||||
output.write("\n=== Flushing cache ===\n")
|
||||
deleted = flush_cache()
|
||||
output.write(f"Deleted {deleted} cache files.\n")
|
||||
output.write("\n=== Done ===\n")
|
||||
except Exception as e:
|
||||
import traceback
|
||||
output.write(f"\n!!! Error: {e}\n")
|
||||
output.write(traceback.format_exc())
|
||||
success = False
|
||||
return render_template("sync.html", output=output.getvalue(), success=success)
|
||||
|
||||
@app.route("/version")
|
||||
def version():
|
||||
return BUILD_META
|
||||
|
||||
@app.route("/api/version")
|
||||
def api_version():
|
||||
return jsonify(BUILD_META)
|
||||
|
||||
@app.route("/api/adults")
|
||||
def api_adults():
|
||||
attendance_url = f"https://docs.google.com/spreadsheets/d/{ATTENDANCE_SHEET_ID}/edit"
|
||||
payments_url = f"https://docs.google.com/spreadsheets/d/{PAYMENTS_SHEET_ID}/edit"
|
||||
|
||||
members_data = get_cached_data("attendance_regular", ATTENDANCE_SHEET_ID, get_members_with_fees)
|
||||
record_step("fetch_members")
|
||||
if not members_data:
|
||||
return "No data."
|
||||
return jsonify({"error": "no data"}), 503
|
||||
members, sorted_months = members_data
|
||||
|
||||
# Filter to adults only for display
|
||||
results = [(name, fees) for name, tier, fees in members if tier == "A"]
|
||||
|
||||
# Format month labels
|
||||
month_labels = get_month_labels(sorted_months, ADULT_MERGED_MONTHS)
|
||||
|
||||
monthly_totals = {m: 0 for m in sorted_months}
|
||||
|
||||
# Get exceptions for formatting
|
||||
credentials_path = CREDENTIALS_PATH
|
||||
transactions = get_cached_data("payments_transactions", PAYMENTS_SHEET_ID, fetch_sheet_data, PAYMENTS_SHEET_ID, CREDENTIALS_PATH)
|
||||
exceptions = get_cached_data(
|
||||
"exceptions_dict", PAYMENTS_SHEET_ID, fetch_exceptions,
|
||||
PAYMENTS_SHEET_ID, credentials_path,
|
||||
PAYMENTS_SHEET_ID, CREDENTIALS_PATH,
|
||||
serialize=lambda d: [[list(k), v] for k, v in d.items()],
|
||||
deserialize=lambda c: {tuple(k): v for k, v in c},
|
||||
)
|
||||
record_step("fetch_exceptions")
|
||||
|
||||
formatted_results = []
|
||||
for name, month_fees in results:
|
||||
row = {"name": name, "months": []}
|
||||
norm_name = normalize(name)
|
||||
for m in sorted_months:
|
||||
fee, count = month_fees.get(m, (0, 0))
|
||||
|
||||
# Check for exception
|
||||
norm_period = normalize(m)
|
||||
ex_data = exceptions.get((norm_name, norm_period))
|
||||
override_amount = ex_data["amount"] if ex_data else None
|
||||
|
||||
if override_amount is not None and override_amount != fee:
|
||||
cell = f"{override_amount} ({fee}) CZK ({count})" if count > 0 else f"{override_amount} ({fee}) CZK"
|
||||
is_overridden = True
|
||||
else:
|
||||
if isinstance(fee, int):
|
||||
monthly_totals[m] += fee
|
||||
cell = f"{fee} CZK ({count})" if count > 0 else "-"
|
||||
is_overridden = False
|
||||
row["months"].append({"cell": cell, "overridden": is_overridden})
|
||||
formatted_results.append(row)
|
||||
|
||||
record_step("process_data")
|
||||
|
||||
return render_template(
|
||||
"fees.html",
|
||||
months=[month_labels[m] for m in sorted_months],
|
||||
results=formatted_results,
|
||||
totals=[f"{monthly_totals[m]} CZK" for m in sorted_months],
|
||||
attendance_url=attendance_url,
|
||||
payments_url=payments_url
|
||||
result = reconcile(members, sorted_months, transactions, exceptions)
|
||||
vm = build_adults_view_model(
|
||||
members, sorted_months, result, transactions,
|
||||
datetime.now().strftime("%Y-%m"),
|
||||
attendance_url=attendance_url, payments_url=payments_url, bank_account=BANK_ACCOUNT,
|
||||
)
|
||||
return jsonify(_unwrap_view_model_for_api(vm))
|
||||
|
||||
@app.route("/fees-juniors")
|
||||
def fees_juniors():
|
||||
@app.route("/api/juniors")
|
||||
def api_juniors():
|
||||
attendance_url = f"https://docs.google.com/spreadsheets/d/{ATTENDANCE_SHEET_ID}/edit#gid={JUNIOR_SHEET_GID}"
|
||||
payments_url = f"https://docs.google.com/spreadsheets/d/{PAYMENTS_SHEET_ID}/edit"
|
||||
|
||||
members_data = get_cached_data("attendance_juniors", ATTENDANCE_SHEET_ID, get_junior_members_with_fees)
|
||||
record_step("fetch_junior_members")
|
||||
if not members_data:
|
||||
return "No data."
|
||||
members, sorted_months = members_data
|
||||
|
||||
# Sort members by name
|
||||
results = sorted([(name, fees) for name, tier, fees in members], key=lambda x: x[0])
|
||||
|
||||
# Format month labels
|
||||
month_labels = get_month_labels(sorted_months, JUNIOR_MERGED_MONTHS)
|
||||
|
||||
monthly_totals = {m: 0 for m in sorted_months}
|
||||
|
||||
# Get exceptions for formatting (reusing payments sheet)
|
||||
credentials_path = CREDENTIALS_PATH
|
||||
junior_members_data = get_cached_data("attendance_juniors", ATTENDANCE_SHEET_ID, get_junior_members_with_fees)
|
||||
if not junior_members_data:
|
||||
return jsonify({"error": "no data"}), 503
|
||||
junior_members, sorted_months = junior_members_data
|
||||
transactions = get_cached_data("payments_transactions", PAYMENTS_SHEET_ID, fetch_sheet_data, PAYMENTS_SHEET_ID, CREDENTIALS_PATH)
|
||||
exceptions = get_cached_data(
|
||||
"exceptions_dict", PAYMENTS_SHEET_ID, fetch_exceptions,
|
||||
PAYMENTS_SHEET_ID, credentials_path,
|
||||
PAYMENTS_SHEET_ID, CREDENTIALS_PATH,
|
||||
serialize=lambda d: [[list(k), v] for k, v in d.items()],
|
||||
deserialize=lambda c: {tuple(k): v for k, v in c},
|
||||
)
|
||||
record_step("fetch_exceptions")
|
||||
|
||||
formatted_results = []
|
||||
for name, month_fees in results:
|
||||
row = {"name": name, "months": []}
|
||||
norm_name = normalize(name)
|
||||
for m in sorted_months:
|
||||
fee_data = month_fees.get(m, (0, 0, 0, 0))
|
||||
if len(fee_data) == 4:
|
||||
fee, total_count, adult_count, junior_count = fee_data
|
||||
else:
|
||||
fee, total_count = fee_data
|
||||
adult_count, junior_count = 0, 0
|
||||
|
||||
# Check for exception
|
||||
norm_period = normalize(m)
|
||||
ex_data = exceptions.get((norm_name, norm_period))
|
||||
override_amount = ex_data["amount"] if ex_data else None
|
||||
|
||||
if ex_data is None and isinstance(fee, int):
|
||||
monthly_totals[m] += fee
|
||||
|
||||
# Formulate the count string display
|
||||
if adult_count > 0 and junior_count > 0:
|
||||
count_str = f"{total_count} ({adult_count}A+{junior_count}J)"
|
||||
elif adult_count > 0:
|
||||
count_str = f"{total_count} (A)"
|
||||
elif junior_count > 0:
|
||||
count_str = f"{total_count} (J)"
|
||||
else:
|
||||
count_str = f"{total_count}"
|
||||
|
||||
if override_amount is not None and override_amount != fee:
|
||||
cell = f"{override_amount} ({fee}) CZK / {count_str}" if total_count > 0 else f"{override_amount} ({fee}) CZK"
|
||||
is_overridden = True
|
||||
else:
|
||||
if fee == "?":
|
||||
cell = f"? / {count_str}" if total_count > 0 else "-"
|
||||
else:
|
||||
cell = f"{fee} CZK / {count_str}" if total_count > 0 else "-"
|
||||
is_overridden = False
|
||||
row["months"].append({"cell": cell, "overridden": is_overridden})
|
||||
formatted_results.append(row)
|
||||
|
||||
record_step("process_data")
|
||||
|
||||
return render_template(
|
||||
"fees-juniors.html",
|
||||
months=[month_labels[m] for m in sorted_months],
|
||||
results=formatted_results,
|
||||
totals=[f"{t} CZK" if isinstance(t, int) else t for t in monthly_totals.values()],
|
||||
attendance_url=attendance_url,
|
||||
payments_url=payments_url
|
||||
adapted_members = adapt_junior_members(junior_members)
|
||||
result = reconcile(adapted_members, sorted_months, transactions, exceptions)
|
||||
vm = build_juniors_view_model(
|
||||
junior_members, adapted_members, sorted_months, result, transactions,
|
||||
datetime.now().strftime("%Y-%m"),
|
||||
attendance_url=attendance_url, payments_url=payments_url, bank_account=BANK_ACCOUNT,
|
||||
)
|
||||
return jsonify(_unwrap_view_model_for_api(vm))
|
||||
|
||||
@app.route("/api/payments")
|
||||
def api_payments():
|
||||
attendance_url = f"https://docs.google.com/spreadsheets/d/{ATTENDANCE_SHEET_ID}/edit"
|
||||
payments_url = f"https://docs.google.com/spreadsheets/d/{PAYMENTS_SHEET_ID}/edit"
|
||||
transactions = get_cached_data("payments_transactions", PAYMENTS_SHEET_ID, fetch_sheet_data, PAYMENTS_SHEET_ID, CREDENTIALS_PATH)
|
||||
adults_data = get_cached_data("attendance_regular", ATTENDANCE_SHEET_ID, get_members_with_fees)
|
||||
juniors_data = get_cached_data("attendance_juniors", ATTENDANCE_SHEET_ID, get_junior_members_with_fees)
|
||||
member_names = []
|
||||
if adults_data:
|
||||
member_names.extend(name for name, _, _ in adults_data[0])
|
||||
if juniors_data:
|
||||
member_names.extend(name for name, _, _ in juniors_data[0])
|
||||
vm = build_payments_view_model(
|
||||
transactions, member_names,
|
||||
attendance_url=attendance_url, payments_url=payments_url,
|
||||
)
|
||||
return jsonify(vm)
|
||||
|
||||
@app.route("/adults")
|
||||
def adults_view():
|
||||
@@ -277,221 +247,20 @@ def adults_view():
|
||||
result = reconcile(members, sorted_months, transactions, exceptions)
|
||||
record_step("reconcile")
|
||||
|
||||
month_labels = get_month_labels(sorted_months, ADULT_MERGED_MONTHS)
|
||||
adult_names = sorted([name for name, tier, _ in members if tier == "A"])
|
||||
|
||||
monthly_totals = {m: {"expected": 0, "paid": 0} for m in sorted_months}
|
||||
formatted_results = []
|
||||
for name in adult_names:
|
||||
data = result["members"][name]
|
||||
row = {"name": name, "months": [], "balance": data["total_balance"], "unpaid_periods": ""}
|
||||
unpaid_months = []
|
||||
for m in sorted_months:
|
||||
mdata = data["months"].get(m, {"expected": 0, "original_expected": 0, "attendance_count": 0, "paid": 0, "exception": None})
|
||||
expected = mdata.get("expected", 0)
|
||||
original_expected = mdata.get("original_expected", 0)
|
||||
count = mdata.get("attendance_count", 0)
|
||||
paid = int(mdata.get("paid", 0))
|
||||
exception_info = mdata.get("exception", None)
|
||||
|
||||
monthly_totals[m]["expected"] += expected
|
||||
monthly_totals[m]["paid"] += paid
|
||||
|
||||
override_amount = exception_info["amount"] if exception_info else None
|
||||
|
||||
if override_amount is not None and override_amount != original_expected:
|
||||
is_overridden = True
|
||||
fee_display = f"{override_amount} ({original_expected}) CZK ({count})" if count > 0 else f"{override_amount} ({original_expected}) CZK"
|
||||
else:
|
||||
is_overridden = False
|
||||
fee_display = f"{expected} CZK ({count})" if count > 0 else f"{expected} CZK"
|
||||
|
||||
status = "empty"
|
||||
cell_text = "-"
|
||||
amount_to_pay = 0
|
||||
|
||||
if expected > 0:
|
||||
amount_to_pay = max(0, expected - paid)
|
||||
if paid >= expected:
|
||||
status = "ok"
|
||||
cell_text = f"{paid}/{fee_display}"
|
||||
elif paid > 0:
|
||||
status = "partial"
|
||||
cell_text = f"{paid}/{fee_display}"
|
||||
unpaid_months.append(month_labels[m])
|
||||
else:
|
||||
status = "unpaid"
|
||||
cell_text = f"0/{fee_display}"
|
||||
unpaid_months.append(month_labels[m])
|
||||
elif paid > 0:
|
||||
status = "surplus"
|
||||
cell_text = f"PAID {paid}"
|
||||
else:
|
||||
cell_text = "-"
|
||||
amount_to_pay = 0
|
||||
|
||||
if expected > 0 or paid > 0:
|
||||
tooltip = f"Received: {paid}, Expected: {expected}"
|
||||
else:
|
||||
tooltip = ""
|
||||
|
||||
row["months"].append({
|
||||
"text": cell_text,
|
||||
"overridden": is_overridden,
|
||||
"status": status,
|
||||
"amount": amount_to_pay,
|
||||
"month": month_labels[m],
|
||||
"tooltip": tooltip
|
||||
})
|
||||
|
||||
row["unpaid_periods"] = ", ".join(unpaid_months) if unpaid_months else ("Older debt" if data["total_balance"] < 0 else "")
|
||||
row["balance"] = data["total_balance"]
|
||||
formatted_results.append(row)
|
||||
|
||||
formatted_totals = []
|
||||
for m in sorted_months:
|
||||
t = monthly_totals[m]
|
||||
status = "empty"
|
||||
if t["expected"] > 0 or t["paid"] > 0:
|
||||
if t["paid"] == t["expected"]:
|
||||
status = "ok"
|
||||
elif t["paid"] < t["expected"]:
|
||||
status = "unpaid"
|
||||
else:
|
||||
status = "surplus"
|
||||
|
||||
formatted_totals.append({
|
||||
"text": f"{t['paid']} / {t['expected']} CZK",
|
||||
"status": status
|
||||
})
|
||||
|
||||
credits = sorted([{"name": n, "amount": a["total_balance"]} for n, a in result["members"].items() if a["total_balance"] > 0 and n in adult_names], key=lambda x: x["name"])
|
||||
debts = sorted([{"name": n, "amount": abs(a["total_balance"])} for n, a in result["members"].items() if a["total_balance"] < 0 and n in adult_names], key=lambda x: x["name"])
|
||||
unmatched = result["unmatched"]
|
||||
import json
|
||||
|
||||
record_step("process_data")
|
||||
|
||||
return render_template(
|
||||
"adults.html",
|
||||
months=[month_labels[m] for m in sorted_months],
|
||||
raw_months=sorted_months,
|
||||
results=formatted_results,
|
||||
totals=formatted_totals,
|
||||
member_data=json.dumps(result["members"]),
|
||||
month_labels_json=json.dumps(month_labels),
|
||||
credits=credits,
|
||||
debts=debts,
|
||||
unmatched=unmatched,
|
||||
vm = build_adults_view_model(
|
||||
members, sorted_months, result, transactions,
|
||||
datetime.now().strftime("%Y-%m"),
|
||||
attendance_url=attendance_url,
|
||||
payments_url=payments_url,
|
||||
bank_account=BANK_ACCOUNT
|
||||
bank_account=BANK_ACCOUNT,
|
||||
)
|
||||
|
||||
@app.route("/reconcile")
|
||||
def reconcile_view():
|
||||
attendance_url = f"https://docs.google.com/spreadsheets/d/{ATTENDANCE_SHEET_ID}/edit"
|
||||
payments_url = f"https://docs.google.com/spreadsheets/d/{PAYMENTS_SHEET_ID}/edit"
|
||||
|
||||
# Use hardcoded credentials path for now, consistent with other scripts
|
||||
credentials_path = CREDENTIALS_PATH
|
||||
|
||||
members_data = get_cached_data("attendance_regular", ATTENDANCE_SHEET_ID, get_members_with_fees)
|
||||
record_step("fetch_members")
|
||||
if not members_data:
|
||||
return "No data."
|
||||
members, sorted_months = members_data
|
||||
|
||||
transactions = get_cached_data("payments_transactions", PAYMENTS_SHEET_ID, fetch_sheet_data, PAYMENTS_SHEET_ID, credentials_path)
|
||||
record_step("fetch_payments")
|
||||
exceptions = get_cached_data(
|
||||
"exceptions_dict", PAYMENTS_SHEET_ID, fetch_exceptions,
|
||||
PAYMENTS_SHEET_ID, credentials_path,
|
||||
serialize=lambda d: [[list(k), v] for k, v in d.items()],
|
||||
deserialize=lambda c: {tuple(k): v for k, v in c},
|
||||
)
|
||||
record_step("fetch_exceptions")
|
||||
result = reconcile(members, sorted_months, transactions, exceptions)
|
||||
record_step("reconcile")
|
||||
|
||||
# Format month labels
|
||||
month_labels = get_month_labels(sorted_months, ADULT_MERGED_MONTHS)
|
||||
|
||||
# Filter to adults for the main table
|
||||
adult_names = sorted([name for name, tier, _ in members if tier == "A"])
|
||||
|
||||
formatted_results = []
|
||||
for name in adult_names:
|
||||
data = result["members"][name]
|
||||
row = {"name": name, "months": [], "balance": data["total_balance"], "unpaid_periods": ""}
|
||||
unpaid_months = []
|
||||
for m in sorted_months:
|
||||
mdata = data["months"].get(m, {"expected": 0, "original_expected": 0, "paid": 0})
|
||||
expected = mdata["expected"]
|
||||
paid = int(mdata["paid"])
|
||||
|
||||
status = "empty"
|
||||
cell_text = "-"
|
||||
amount_to_pay = 0
|
||||
|
||||
if expected > 0:
|
||||
if paid >= expected:
|
||||
status = "ok"
|
||||
cell_text = "OK"
|
||||
elif paid > 0:
|
||||
status = "partial"
|
||||
cell_text = f"{paid}/{expected}"
|
||||
amount_to_pay = expected - paid
|
||||
unpaid_months.append(month_labels[m])
|
||||
else:
|
||||
status = "unpaid"
|
||||
cell_text = f"UNPAID {expected}"
|
||||
amount_to_pay = expected
|
||||
unpaid_months.append(month_labels[m])
|
||||
elif paid > 0:
|
||||
status = "surplus"
|
||||
cell_text = f"PAID {paid}"
|
||||
|
||||
row["months"].append({
|
||||
"text": cell_text,
|
||||
"status": status,
|
||||
"amount": amount_to_pay,
|
||||
"month": month_labels[m]
|
||||
})
|
||||
|
||||
row["unpaid_periods"] = ", ".join(unpaid_months) if unpaid_months else ("Older debt" if data["total_balance"] < 0 else "")
|
||||
row["balance"] = data["total_balance"] # Updated to use total_balance
|
||||
formatted_results.append(row)
|
||||
|
||||
# Format credits and debts
|
||||
credits = sorted([{"name": n, "amount": a["total_balance"]} for n, a in result["members"].items() if a["total_balance"] > 0 and n in adult_names], key=lambda x: x["name"])
|
||||
debts = sorted([{"name": n, "amount": abs(a["total_balance"])} for n, a in result["members"].items() if a["total_balance"] < 0 and n in adult_names], key=lambda x: x["name"])
|
||||
# Format unmatched
|
||||
unmatched = result["unmatched"]
|
||||
import json
|
||||
|
||||
record_step("process_data")
|
||||
|
||||
return render_template(
|
||||
"reconcile.html",
|
||||
months=[month_labels[m] for m in sorted_months],
|
||||
raw_months=sorted_months,
|
||||
results=formatted_results,
|
||||
member_data=json.dumps(result["members"]),
|
||||
month_labels_json=json.dumps(month_labels),
|
||||
credits=credits,
|
||||
debts=debts,
|
||||
unmatched=unmatched,
|
||||
attendance_url=attendance_url,
|
||||
payments_url=payments_url,
|
||||
bank_account=BANK_ACCOUNT
|
||||
)
|
||||
return render_template("adults.html", **vm)
|
||||
|
||||
@app.route("/juniors")
|
||||
def juniors_view():
|
||||
attendance_url = f"https://docs.google.com/spreadsheets/d/{ATTENDANCE_SHEET_ID}/edit#gid={JUNIOR_SHEET_GID}"
|
||||
payments_url = f"https://docs.google.com/spreadsheets/d/{PAYMENTS_SHEET_ID}/edit"
|
||||
|
||||
credentials_path = CREDENTIALS_PATH
|
||||
|
||||
junior_members_data = get_cached_data("attendance_juniors", ATTENDANCE_SHEET_ID, get_junior_members_with_fees)
|
||||
@@ -510,268 +279,19 @@ def juniors_view():
|
||||
)
|
||||
record_step("fetch_exceptions")
|
||||
|
||||
# Adapt junior tuple format (name, tier, {month: (fee, total_count, adult_count, junior_count)})
|
||||
# to what match_payments expects: (name, tier, {month: (expected_fee, attendance_count)})
|
||||
adapted_members = []
|
||||
for name, tier, fees_dict in junior_members:
|
||||
adapted_fees = {}
|
||||
for m, fee_data in fees_dict.items():
|
||||
if len(fee_data) == 4:
|
||||
fee, total_count, _, _ = fee_data
|
||||
adapted_fees[m] = (fee, total_count)
|
||||
else:
|
||||
fee, count = fee_data
|
||||
adapted_fees[m] = (fee, count)
|
||||
adapted_members.append((name, tier, adapted_fees))
|
||||
|
||||
adapted_members = adapt_junior_members(junior_members)
|
||||
result = reconcile(adapted_members, sorted_months, transactions, exceptions)
|
||||
record_step("reconcile")
|
||||
|
||||
# Format month labels
|
||||
month_labels = get_month_labels(sorted_months, JUNIOR_MERGED_MONTHS)
|
||||
junior_names = sorted([name for name, tier, _ in adapted_members])
|
||||
junior_members_dict = {name: fees_dict for name, _, fees_dict in junior_members}
|
||||
|
||||
monthly_totals = {m: {"expected": 0, "paid": 0} for m in sorted_months}
|
||||
formatted_results = []
|
||||
for name in junior_names:
|
||||
data = result["members"][name]
|
||||
row = {"name": name, "months": [], "balance": data["total_balance"], "unpaid_periods": ""}
|
||||
unpaid_months = []
|
||||
for m in sorted_months:
|
||||
mdata = data["months"].get(m, {"expected": 0, "original_expected": 0, "attendance_count": 0, "paid": 0, "exception": None})
|
||||
expected = mdata.get("expected", 0)
|
||||
original_expected = mdata.get("original_expected", 0)
|
||||
count = mdata.get("attendance_count", 0)
|
||||
paid = int(mdata.get("paid", 0))
|
||||
exception_info = mdata.get("exception", None)
|
||||
|
||||
if expected != "?" and isinstance(expected, int):
|
||||
monthly_totals[m]["expected"] += expected
|
||||
monthly_totals[m]["paid"] += paid
|
||||
|
||||
orig_fee_data = junior_members_dict.get(name, {}).get(m)
|
||||
adult_count = 0
|
||||
junior_count = 0
|
||||
if orig_fee_data and len(orig_fee_data) == 4:
|
||||
_, _, adult_count, junior_count = orig_fee_data
|
||||
|
||||
breakdown = ""
|
||||
if adult_count > 0 and junior_count > 0:
|
||||
breakdown = f":{junior_count}J,{adult_count}A"
|
||||
elif junior_count > 0:
|
||||
breakdown = f":{junior_count}J"
|
||||
elif adult_count > 0:
|
||||
breakdown = f":{adult_count}A"
|
||||
|
||||
count_str = f" ({count}{breakdown})" if count > 0 else ""
|
||||
|
||||
override_amount = exception_info["amount"] if exception_info else None
|
||||
|
||||
if override_amount is not None and override_amount != original_expected:
|
||||
is_overridden = True
|
||||
fee_display = f"{override_amount} ({original_expected}) CZK{count_str}"
|
||||
else:
|
||||
is_overridden = False
|
||||
fee_display = f"{expected} CZK{count_str}"
|
||||
|
||||
status = "empty"
|
||||
cell_text = "-"
|
||||
amount_to_pay = 0
|
||||
|
||||
if expected == "?" or (isinstance(expected, int) and expected > 0):
|
||||
if expected == "?":
|
||||
status = "empty"
|
||||
cell_text = "?"
|
||||
elif paid >= expected:
|
||||
status = "ok"
|
||||
cell_text = f"{paid}/{fee_display}"
|
||||
elif paid > 0:
|
||||
status = "partial"
|
||||
cell_text = f"{paid}/{fee_display}"
|
||||
amount_to_pay = expected - paid
|
||||
unpaid_months.append(month_labels[m])
|
||||
else:
|
||||
status = "unpaid"
|
||||
cell_text = f"0/{fee_display}"
|
||||
amount_to_pay = expected
|
||||
unpaid_months.append(month_labels[m])
|
||||
elif paid > 0:
|
||||
status = "surplus"
|
||||
cell_text = f"PAID {paid}"
|
||||
|
||||
if (isinstance(expected, int) and expected > 0) or paid > 0:
|
||||
tooltip = f"Received: {paid}, Expected: {expected}"
|
||||
else:
|
||||
tooltip = ""
|
||||
|
||||
row["months"].append({
|
||||
"text": cell_text,
|
||||
"overridden": is_overridden,
|
||||
"status": status,
|
||||
"amount": amount_to_pay,
|
||||
"month": month_labels[m],
|
||||
"tooltip": tooltip
|
||||
})
|
||||
|
||||
row["unpaid_periods"] = ", ".join(unpaid_months) if unpaid_months else ("Older debt" if data["total_balance"] < 0 else "")
|
||||
row["balance"] = data["total_balance"]
|
||||
formatted_results.append(row)
|
||||
|
||||
formatted_totals = []
|
||||
for m in sorted_months:
|
||||
t = monthly_totals[m]
|
||||
status = "empty"
|
||||
if t["expected"] > 0 or t["paid"] > 0:
|
||||
if t["paid"] == t["expected"]:
|
||||
status = "ok"
|
||||
elif t["paid"] < t["expected"]:
|
||||
status = "unpaid"
|
||||
else:
|
||||
status = "surplus"
|
||||
|
||||
formatted_totals.append({
|
||||
"text": f"{t['paid']} / {t['expected']} CZK",
|
||||
"status": status
|
||||
})
|
||||
|
||||
# Format credits and debts
|
||||
credits = sorted([{"name": n, "amount": a["total_balance"]} for n, a in result["members"].items() if a["total_balance"] > 0], key=lambda x: x["name"])
|
||||
debts = sorted([{"name": n, "amount": abs(a["total_balance"])} for n, a in result["members"].items() if a["total_balance"] < 0], key=lambda x: x["name"])
|
||||
unmatched = result["unmatched"]
|
||||
import json
|
||||
|
||||
record_step("process_data")
|
||||
|
||||
return render_template(
|
||||
"juniors.html",
|
||||
months=[month_labels[m] for m in sorted_months],
|
||||
raw_months=sorted_months,
|
||||
results=formatted_results,
|
||||
totals=formatted_totals,
|
||||
member_data=json.dumps(result["members"]),
|
||||
month_labels_json=json.dumps(month_labels),
|
||||
credits=credits,
|
||||
debts=debts,
|
||||
unmatched=unmatched,
|
||||
vm = build_juniors_view_model(
|
||||
junior_members, adapted_members, sorted_months, result, transactions,
|
||||
datetime.now().strftime("%Y-%m"),
|
||||
attendance_url=attendance_url,
|
||||
payments_url=payments_url,
|
||||
bank_account=BANK_ACCOUNT
|
||||
bank_account=BANK_ACCOUNT,
|
||||
)
|
||||
|
||||
@app.route("/reconcile-juniors")
|
||||
def reconcile_juniors_view():
|
||||
attendance_url = f"https://docs.google.com/spreadsheets/d/{ATTENDANCE_SHEET_ID}/edit#gid={JUNIOR_SHEET_GID}"
|
||||
payments_url = f"https://docs.google.com/spreadsheets/d/{PAYMENTS_SHEET_ID}/edit"
|
||||
|
||||
credentials_path = CREDENTIALS_PATH
|
||||
|
||||
junior_members_data = get_cached_data("attendance_juniors", ATTENDANCE_SHEET_ID, get_junior_members_with_fees)
|
||||
record_step("fetch_junior_members")
|
||||
if not junior_members_data:
|
||||
return "No data."
|
||||
junior_members, sorted_months = junior_members_data
|
||||
|
||||
transactions = get_cached_data("payments_transactions", PAYMENTS_SHEET_ID, fetch_sheet_data, PAYMENTS_SHEET_ID, credentials_path)
|
||||
record_step("fetch_payments")
|
||||
exceptions = get_cached_data(
|
||||
"exceptions_dict", PAYMENTS_SHEET_ID, fetch_exceptions,
|
||||
PAYMENTS_SHEET_ID, credentials_path,
|
||||
serialize=lambda d: [[list(k), v] for k, v in d.items()],
|
||||
deserialize=lambda c: {tuple(k): v for k, v in c},
|
||||
)
|
||||
record_step("fetch_exceptions")
|
||||
|
||||
# Adapt junior tuple format (name, tier, {month: (fee, total_count, adult_count, junior_count)})
|
||||
# to what match_payments expects: (name, tier, {month: (expected_fee, attendance_count)})
|
||||
adapted_members = []
|
||||
for name, tier, fees_dict in junior_members:
|
||||
adapted_fees = {}
|
||||
for m, fee_data in fees_dict.items():
|
||||
if len(fee_data) == 4:
|
||||
fee, total_count, _, _ = fee_data
|
||||
adapted_fees[m] = (fee, total_count)
|
||||
else:
|
||||
fee, count = fee_data
|
||||
adapted_fees[m] = (fee, count)
|
||||
adapted_members.append((name, tier, adapted_fees))
|
||||
|
||||
result = reconcile(adapted_members, sorted_months, transactions, exceptions)
|
||||
record_step("reconcile")
|
||||
|
||||
# Format month labels
|
||||
month_labels = get_month_labels(sorted_months, JUNIOR_MERGED_MONTHS)
|
||||
|
||||
# Filter to juniors for the main table
|
||||
junior_names = sorted([name for name, tier, _ in adapted_members])
|
||||
|
||||
formatted_results = []
|
||||
for name in junior_names:
|
||||
data = result["members"][name]
|
||||
row = {"name": name, "months": [], "balance": data["total_balance"], "unpaid_periods": ""}
|
||||
unpaid_months = []
|
||||
for m in sorted_months:
|
||||
mdata = data["months"].get(m, {"expected": 0, "original_expected": 0, "paid": 0})
|
||||
expected = mdata["expected"]
|
||||
paid = int(mdata["paid"])
|
||||
|
||||
status = "empty"
|
||||
cell_text = "-"
|
||||
amount_to_pay = 0
|
||||
|
||||
if expected == "?" or (isinstance(expected, int) and expected > 0):
|
||||
if expected == "?":
|
||||
status = "empty"
|
||||
cell_text = "?"
|
||||
elif paid >= expected:
|
||||
status = "ok"
|
||||
cell_text = "OK"
|
||||
elif paid > 0:
|
||||
status = "partial"
|
||||
cell_text = f"{paid}/{expected}"
|
||||
amount_to_pay = expected - paid
|
||||
unpaid_months.append(month_labels[m])
|
||||
else:
|
||||
status = "unpaid"
|
||||
cell_text = f"UNPAID {expected}"
|
||||
amount_to_pay = expected
|
||||
unpaid_months.append(month_labels[m])
|
||||
elif paid > 0:
|
||||
status = "surplus"
|
||||
cell_text = f"PAID {paid}"
|
||||
|
||||
row["months"].append({
|
||||
"text": cell_text,
|
||||
"status": status,
|
||||
"amount": amount_to_pay,
|
||||
"month": month_labels[m]
|
||||
})
|
||||
|
||||
row["unpaid_periods"] = ", ".join(unpaid_months) if unpaid_months else ("Older debt" if data["total_balance"] < 0 else "")
|
||||
row["balance"] = data["total_balance"]
|
||||
formatted_results.append(row)
|
||||
|
||||
# Format credits and debts
|
||||
credits = sorted([{"name": n, "amount": a["total_balance"]} for n, a in result["members"].items() if a["total_balance"] > 0], key=lambda x: x["name"])
|
||||
debts = sorted([{"name": n, "amount": abs(a["total_balance"])} for n, a in result["members"].items() if a["total_balance"] < 0], key=lambda x: x["name"])
|
||||
import json
|
||||
|
||||
record_step("process_data")
|
||||
|
||||
return render_template(
|
||||
"reconcile-juniors.html",
|
||||
months=[month_labels[m] for m in sorted_months],
|
||||
raw_months=sorted_months,
|
||||
results=formatted_results,
|
||||
member_data=json.dumps(result["members"]),
|
||||
month_labels_json=json.dumps(month_labels),
|
||||
credits=credits,
|
||||
debts=debts,
|
||||
unmatched=[],
|
||||
attendance_url=attendance_url,
|
||||
payments_url=payments_url,
|
||||
bank_account=BANK_ACCOUNT
|
||||
)
|
||||
return render_template("juniors.html", **vm)
|
||||
|
||||
@app.route("/payments")
|
||||
def payments():
|
||||
@@ -782,36 +302,21 @@ def payments():
|
||||
transactions = get_cached_data("payments_transactions", PAYMENTS_SHEET_ID, fetch_sheet_data, PAYMENTS_SHEET_ID, credentials_path)
|
||||
record_step("fetch_payments")
|
||||
|
||||
# Group transactions by person
|
||||
grouped = {}
|
||||
for tx in transactions:
|
||||
person = str(tx.get("person", "")).strip()
|
||||
if not person:
|
||||
person = "Unmatched / Unknown"
|
||||
adults_data = get_cached_data("attendance_regular", ATTENDANCE_SHEET_ID, get_members_with_fees)
|
||||
juniors_data = get_cached_data("attendance_juniors", ATTENDANCE_SHEET_ID, get_junior_members_with_fees)
|
||||
member_names = []
|
||||
if adults_data:
|
||||
member_names.extend(name for name, _, _ in adults_data[0])
|
||||
if juniors_data:
|
||||
member_names.extend(name for name, _, _ in juniors_data[0])
|
||||
|
||||
# Handle multiple people (comma separated)
|
||||
people = [p.strip() for p in person.split(",") if p.strip()]
|
||||
for p in people:
|
||||
# Strip markers
|
||||
clean_p = re.sub(r"\[\?\]\s*", "", p)
|
||||
if clean_p not in grouped:
|
||||
grouped[clean_p] = []
|
||||
grouped[clean_p].append(tx)
|
||||
|
||||
# Sort people and their transactions
|
||||
sorted_people = sorted(grouped.keys())
|
||||
for p in sorted_people:
|
||||
# Sort by date descending
|
||||
grouped[p].sort(key=lambda x: str(x.get("date", "")), reverse=True)
|
||||
|
||||
record_step("process_data")
|
||||
return render_template(
|
||||
"payments.html",
|
||||
grouped_payments=grouped,
|
||||
sorted_people=sorted_people,
|
||||
vm = build_payments_view_model(
|
||||
transactions, member_names,
|
||||
attendance_url=attendance_url,
|
||||
payments_url=payments_url
|
||||
payments_url=payments_url,
|
||||
)
|
||||
record_step("process_data")
|
||||
return render_template("payments.html", **vm)
|
||||
|
||||
@app.route("/qr")
|
||||
def qr_code():
|
||||
|
||||
@@ -24,6 +24,17 @@ COPY templates/ ./templates/
|
||||
COPY build/entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /entrypoint.sh
|
||||
|
||||
ARG GIT_TAG=unknown
|
||||
ARG GIT_COMMIT=unknown
|
||||
ARG BUILD_DATE=unknown
|
||||
|
||||
LABEL org.opencontainers.image.version="${GIT_TAG}" \
|
||||
org.opencontainers.image.revision="${GIT_COMMIT}" \
|
||||
org.opencontainers.image.created="${BUILD_DATE}" \
|
||||
org.opencontainers.image.title="fuj-management"
|
||||
|
||||
RUN echo "{\"tag\": \"${GIT_TAG}\", \"commit\": \"${GIT_COMMIT}\", \"build_date\": \"${BUILD_DATE}\"}" > /app/build_meta.json
|
||||
|
||||
EXPOSE 5001
|
||||
|
||||
HEALTHCHECK --interval=60s --timeout=5s --start-period=5s \
|
||||
|
||||
52
docs/operation-manual.md
Normal file
52
docs/operation-manual.md
Normal file
@@ -0,0 +1,52 @@
|
||||
# Operation Manual
|
||||
|
||||
## Adding a Monthly Fee Override
|
||||
|
||||
Use this when the club decides to charge a different flat fee for a specific month — for example, a reduced fee during a short or holiday month.
|
||||
|
||||
There are two independent dictionaries in [scripts/attendance.py](../scripts/attendance.py), one for adults and one for juniors. Edit whichever tiers need an override.
|
||||
|
||||
### Adults
|
||||
|
||||
Add an entry to `ADULT_FEE_MONTHLY_RATE` (line ~15):
|
||||
|
||||
```python
|
||||
ADULT_FEE_MONTHLY_RATE = {
|
||||
"2026-03": 350 # reduced fee for March 2026
|
||||
}
|
||||
```
|
||||
|
||||
The key is `YYYY-MM`, the value is the fee in CZK. This replaces `ADULT_FEE_DEFAULT` (750 CZK) for members who attended 2+ practices that month. The single-practice fee (`ADULT_FEE_SINGLE`, 200 CZK) is unaffected.
|
||||
|
||||
### Juniors
|
||||
|
||||
Add an entry to `JUNIOR_MONTHLY_RATE` (line ~20):
|
||||
|
||||
```python
|
||||
JUNIOR_MONTHLY_RATE = {
|
||||
"2025-09": 250, # reduced fee for September 2025
|
||||
"2026-03": 250 # reduced fee for March 2026
|
||||
}
|
||||
```
|
||||
|
||||
The key is `YYYY-MM`, the value is the fee in CZK. This replaces `JUNIOR_FEE_DEFAULT` (500 CZK) for members who attended 2+ practices that month.
|
||||
|
||||
### Example: March 2026
|
||||
|
||||
Both tiers reduced to 350 CZK (adults) and 250 CZK (juniors):
|
||||
|
||||
```python
|
||||
ADULT_FEE_MONTHLY_RATE = {
|
||||
"2026-03": 350
|
||||
}
|
||||
|
||||
JUNIOR_MONTHLY_RATE = {
|
||||
"2026-03": 250
|
||||
}
|
||||
```
|
||||
|
||||
### Notes
|
||||
|
||||
- Overrides apply to all members of the given tier — use the **exceptions sheet** in Google Sheets for per-member overrides instead.
|
||||
- After changing these values, restart the web dashboard (`make web`) for the change to take effect.
|
||||
- The override only affects the calculated/expected fee. It does not modify any already-recorded payments in the bank sheet.
|
||||
@@ -0,0 +1,52 @@
|
||||
# Plan: Document plan-file location convention in `CLAUDE.md`
|
||||
|
||||
## Context
|
||||
|
||||
The user wants all plan files (created during Claude Code's plan mode) to live
|
||||
inside the project at `docs/plans/`, with a creation timestamp in the filename.
|
||||
This keeps planning artifacts version-controlled alongside the code, makes it
|
||||
easy to see when each plan was drafted, and — critically — needs to be
|
||||
discoverable by other contributors who use Claude Code on this repo. So the
|
||||
convention belongs in `CLAUDE.md`, not in private agent memory.
|
||||
|
||||
## Approach
|
||||
|
||||
1. **Add a new section to `CLAUDE.md`** (placed near the existing "Changelog"
|
||||
section, since both are about persisted artifacts that Claude maintains):
|
||||
|
||||
```markdown
|
||||
## Plans
|
||||
|
||||
When Claude Code's plan mode is used, save the plan file inside the repo at
|
||||
`docs/plans/YYYY-MM-DD-HHMM-<slug>.md` instead of the default
|
||||
`~/.claude/plans/` location. Get the timestamp with
|
||||
`date "+%Y-%m-%d-%H%M"` (matches the changelog convention). The `<slug>`
|
||||
should be a short kebab-case summary of the plan's topic.
|
||||
|
||||
Create the `docs/plans/` directory on first use. Plan files are committed
|
||||
to the repo so other contributors can review historical decisions.
|
||||
```
|
||||
|
||||
2. **Create the `docs/plans/` directory** with a `.gitkeep` (or just let it
|
||||
appear when the first plan is moved in) so the path exists.
|
||||
|
||||
3. **Move this current plan** into the new location once plan mode exits:
|
||||
`docs/plans/2026-05-03-1200-document-plan-location-convention.md`
|
||||
(timestamp will be re-generated with the actual `date` output).
|
||||
|
||||
4. **No memory entry needed** — the rule lives in `CLAUDE.md` and is loaded
|
||||
automatically into every Claude Code session in this repo.
|
||||
|
||||
## Files touched
|
||||
|
||||
- [CLAUDE.md](CLAUDE.md) — add the new "## Plans" section.
|
||||
- New directory: [docs/plans/](docs/plans/) — created on first use.
|
||||
- Move this plan file from `~/.claude/plans/...` into `docs/plans/` with the
|
||||
proper timestamped filename.
|
||||
|
||||
## Verification
|
||||
|
||||
- `grep -A 5 "## Plans" CLAUDE.md` shows the new section.
|
||||
- `ls docs/plans/` lists this plan file with a `YYYY-MM-DD-HHMM-` prefix.
|
||||
- Next time plan mode is entered in this repo, the new plan is written to
|
||||
`docs/plans/` with a fresh timestamp (verify by re-entering plan mode).
|
||||
160
docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md
Normal file
160
docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md
Normal file
@@ -0,0 +1,160 @@
|
||||
# Go Rewrite — Progress Tracker
|
||||
|
||||
Companion to [2026-05-03-2349-go-backend-rewrite.md](2026-05-03-2349-go-backend-rewrite.md).
|
||||
|
||||
**Current milestone:** M5 — JSON-only `/api/...` routes ✅
|
||||
**Started:** 2026-05-04
|
||||
**Last updated:** 2026-05-07 (M5.4)
|
||||
|
||||
## How to use
|
||||
|
||||
- Tick a checkbox when the task's PR/commit lands. Append the SHA in the same
|
||||
line: `[x] **M1.1** ... — `abc1234``.
|
||||
- One task = one focused commit or PR. If a task balloons, split it and add
|
||||
sub-tasks below the parent.
|
||||
- Note decisions, surprises, or blockers under "Notes & decisions" at the
|
||||
bottom — that's where future-you (or a contributor) will look first.
|
||||
- Don't reorder milestones. Within a milestone, tasks can be done in any
|
||||
order unless explicitly noted.
|
||||
|
||||
---
|
||||
|
||||
## M1 — Skeleton + tooling
|
||||
|
||||
Goal: `make web-go` serves a hello page on :8080 in parallel with `make web-py` on :5001. Lint clean.
|
||||
|
||||
- [x] **M1.1** Create `go/` tree skeleton + `go.mod` initialized to latest stable Go
|
||||
- [x] **M1.2** Add `cmd/fuj/main.go` with subcommand dispatcher — stdlib `flag` + `os.Args[1]` switch
|
||||
- [x] **M1.3** Wire `fuj server` subcommand: `net/http` ServeMux on `:8080`, plaintext hello page
|
||||
- [x] **M1.4** Add Makefile targets: `go-build`, `go-test`, `go-run`, `go-lint`
|
||||
- [x] **M1.5** Rename existing `make web` → `make web-py`; added `make web-go`; kept `make web` as alias
|
||||
- [x] **M1.6** Add `go/.golangci.yml` (govet, staticcheck, errcheck, gofumpt, unused) + `make go-lint` clean
|
||||
- [x] **M1.7** Write `go/build/Dockerfile` (multi-stage `golang:1.26` → `alpine:3`); parallel `build-go` job in Gitea CI
|
||||
- [x] **M1.8** Add `internal/config` package mirroring `scripts/config.py` (same env var names + defaults)
|
||||
- [x] **M1.9** Add `internal/logging` (slog, level from config) + `middleware/timer.go` (method/path/status/ms)
|
||||
- [x] **M1.10** Gate passed: `make go-build`, `make go-lint`, `make go-test`, `curl :8080` all green; CHANGELOG entry added
|
||||
|
||||
**Gate:** ✅ `make go-build` succeeds, `curl localhost:8080` returns hello page, `make go-lint` clean.
|
||||
|
||||
---
|
||||
|
||||
## M2 — Pure-domain helpers (port leaf-first)
|
||||
|
||||
Goal: every pure function from the Python backend exists in Go with a parity test against captured fixtures (M3 produces fixtures in parallel — order is M2.1 → M3.1/M3.2 → M3.3+ alongside M2.2+).
|
||||
|
||||
Each task: port the function, write Go unit tests for fresh cases, hook into the Tier-1 parity runner.
|
||||
|
||||
- [x] **M2.1** `domain/czech.Normalize` — port [czech_utils.py](scripts/czech_utils.py) `normalize` (NFKD + combining-mark strip + lowercase) — `20ade6d`
|
||||
- [x] **M2.2** `domain/czech.ParseMonthReferences` — port `parse_month_references` (45 month declensions, range wrap, year inference) — `0a8017f`
|
||||
- [x] **M2.3** `domain/fees.CalculateFee` — port [attendance.py](scripts/attendance.py) `calculate_fee` (constants table) — `0fc3b6d`
|
||||
- [x] **M2.4** `domain/fees.CalculateJuniorFee` — port `calculate_junior_fee` with `Expected{Value int; Unknown bool}` for the `"?"` sentinel — `0fc3b6d`
|
||||
- [x] **M2.5** `domain/money.ParseCZK` — port [infer_payments.py](scripts/infer_payments.py) `parse_czk_amount` (Czech locale: comma decimal, dot/space thousand separators) — `d24d205`
|
||||
- [x] **M2.6** `domain/synch.GenerateSyncID` — port [sync_fio_to_sheets.py](scripts/sync_fio_to_sheets.py) `generate_sync_id` (SHA-256, byte-stable hash; verify float string format against real sheet rows)
|
||||
- [x] **M2.7** `domain/matching.BuildNameVariants` + `MatchMembers` — port `_build_name_variants` and `match_members` from [match_payments.py](scripts/match_payments.py) (auto vs review confidence, common-surname filter) — `e596f00`
|
||||
- [x] **M2.8** `domain/matching.InferTransactionDetails` — port `infer_transaction_details` (composes name + month parsing) — `e596f00`
|
||||
- [x] **M2.9** `domain/matching.FormatDate` — port `format_date` (handles Google Sheets serial-day numbers since 1899-12-30) — `e596f00`
|
||||
- [x] **M2.10** `domain/reconcile.Reconcile` — port `reconcile` (three-phase allocation: greedy / proportional with float-remainder absorption / even-split fallback). The single most load-bearing function; budget extra time. — `c53bf5a`
|
||||
- [x] **M2.11** `fuj fees` subcommand wired up via `domain/fees` + (M4-stub) attendance loader — fail gracefully on missing IO until M4 lands — `56aa230`
|
||||
- [x] **M2.12** `fuj reconcile` subcommand similarly stubbed — `56aa230`
|
||||
|
||||
**Gate:** `cd go && go test -tags=parity ./tests/parity/pure/...` green for every fixture in `tests/fixtures/pure/`.
|
||||
|
||||
---
|
||||
|
||||
## M3 — Fixture capture + characterization framework
|
||||
|
||||
Goal: deterministic, PII-free fixture corpus that drives parity tests. Runs in parallel with M2 (M3.1/M3.2 unblocks M2.1).
|
||||
|
||||
- [x] **M3.1** `scripts/capture_fixtures.py` — pure-function output dumper. Reads inputs from stdin / argv, prints `{"input":..., "output":...}` JSON — `57518a8`
|
||||
- [x] **M3.2** `scripts/scrub_fixtures.py` — replaces names with `Member_<8hex>` (deterministic per name); scrambles sender/account/VS/bank_id with stable bijection; preserves dates, amounts, exception keys — `57518a8`
|
||||
- [x] **M3.3** Capture pure-fn fixtures for M2.1–M2.9 (run helper + scrubber, commit to `tests/fixtures/pure/<func>/<case>.json`) — `57518a8`
|
||||
- [x] **M3.4** Capture ~10 reconcile fixtures spanning every code path: greedy, proportional (float remainder), even-split, out-of-window credit, exception override, `other:` purpose, junior `"?"`, multi-person comma-split, multi-month range, unmatched. Commit to `tests/fixtures/reconcile/` — `57518a8`
|
||||
- [x] **M3.5** Hook fixtures into Tier-1 test runner with `-tags=parity` build constraint — `57518a8`
|
||||
- [x] **M3.6** Document fixture-refresh workflow in `tests/fixtures/README.md` (what to do when sheet schema changes) — `57518a8`
|
||||
|
||||
**Gate:** ✅ `tests/fixtures/` populated (98 files); `make go-parity` green; `make go-lint` (parity tag) clean; raw `tmp/*.json` confirmed gitignored. Merged as `57518a8`.
|
||||
|
||||
---
|
||||
|
||||
## M4 — IO layer behind interfaces
|
||||
|
||||
Goal: every external IO (Sheets, Drive, Fio, file cache) accessed through a narrow Go interface with both a real and a fake implementation.
|
||||
|
||||
- [x] **M4.1** Design IO interfaces (`SheetsClient`, `DriveClient`, `FioClient`, `FileCache`) + in-memory fakes seeded from M3 fixtures
|
||||
- [x] **M4.2** `internal/io/sheets` — Google client (read + append + batchUpdate); fake with call-capture
|
||||
- [x] **M4.3** `internal/io/drive` — Drive `modifiedTime` client + fake
|
||||
- [x] **M4.4** `internal/io/fio` — API JSON impl (token-based); parses by hardcoded `column0..column22` indices matching [fio_utils.py](scripts/fio_utils.py)
|
||||
- [x] **M4.5** `internal/io/fio` — transparent-page HTML scraper using `golang.org/x/net/html` token visitor; targets the **second** `<table class="table">`
|
||||
- [x] **M4.6** `internal/io/cache` — FileCache with `modifiedTime` gating + two TTL knobs + atomic writes (`os.Rename`)
|
||||
- [x] **M4.7** `services/banksync.SyncToSheets` + `fuj sync` subcommand
|
||||
- [x] **M4.8** `services/banksync.InferPayments` + `fuj infer [--dry-run]` subcommand; `NewSources` wires all IO into fees+reconcile
|
||||
|
||||
**Gate:** ✅ Fakes-only unit tests; `make go-test` + `make go-lint` both green. Live smoke test deferred to first real sync run.
|
||||
|
||||
---
|
||||
|
||||
## M5 — JSON-only `/api/...` routes
|
||||
|
||||
Goal: byte-equal JSON between Python and Go for every route. This is the parity contract.
|
||||
|
||||
- [x] **M5.1** Hand-author Go structs for `/api/adults`, `/api/juniors`, `/api/payments`, `/api/version` with explicit `json:` tags matching Python keys; emit JSON Schemas via `github.com/invopop/jsonschema` to `tests/fixtures/api-schema/` — `f253e3f`
|
||||
- [x] **M5.2** Implement Go handlers for `/api/*` routes composing `services/*` results into the JSON structs — `7d48e8f`
|
||||
- [x] **M5.3** Add Python `/api/X` shadow endpoints in [app.py](app.py): `jsonify(view_model_dict)` — no transformation — `40e4a9e`
|
||||
- [x] **M5.4** Build `cmd/parity/main.go`: hits both backends' `/api/X`, normalizes allowlist (`render_time.total`, `build_meta`), prints `cmp.Diff`. Add `make parity` target
|
||||
|
||||
**Gate:** For each route, `make parity` reports zero non-allowlisted diffs across the M3 fixture corpus.
|
||||
|
||||
---
|
||||
|
||||
## M6 — Go-native HTML frontend
|
||||
|
||||
Goal: feature-equivalent UX on the Go side, designed cleanly. Not a Jinja port.
|
||||
|
||||
- [ ] **M6.1** Template skeleton: base layout, nav (Adults/Juniors/Payments/Sync/Flush), terminal-green-on-black theme; `embed.FS` for `templates/` + `static/`
|
||||
- [ ] **M6.2** `/adults` page: table, name filter input, month range filter, totals row, credits/debts/unmatched sections, Pay buttons that link to `/qr`
|
||||
- [ ] **M6.3** `/juniors` page: same structure + per-month J/A attendance breakdown + `"?"` sentinel rendering
|
||||
- [ ] **M6.4** `/payments` page: grouped-by-person ledger view
|
||||
- [ ] **M6.5** Modal JS module (`static/js/member-detail.js`): fetches `/api/adults` (or juniors), renders status/exceptions/transactions on row click; keyboard nav (Esc, ↑/↓)
|
||||
- [ ] **M6.6** `/qr`, `/sync-bank`, `/flush-cache`, `/version` pages
|
||||
- [ ] **M6.7** Wire `embed.FS` into handlers; verify single-binary deployment includes all assets
|
||||
|
||||
**Gate:** Browser smoke on :8080: all pages render, name+month filters work, modal opens with correct data, QR loads, sync/flush work end-to-end.
|
||||
|
||||
---
|
||||
|
||||
## M7 — Parallel-running watch period
|
||||
|
||||
Goal: prove parity over real time before flipping the default.
|
||||
|
||||
- [ ] **M7.1** Add Go service to `docker-compose.yml` on different port (alongside Python container)
|
||||
- [ ] **M7.2** Set up `parity-nightly.yml` Gitea workflow: boot both, replay fixed transaction script, fail on diff
|
||||
- [ ] **M7.3** Run `make parity` daily for 7–14 days, log any diffs; investigate and fix root cause (don't just allowlist)
|
||||
- [ ] **M7.4** Manual feature parity check: walk through every UI feature on both sides, sign off in Notes section
|
||||
|
||||
**Gate:** Zero non-allowlisted JSON diffs over 7 consecutive days, including a sync-bank execution + flush + attendance update; user sign-off on UI feature parity.
|
||||
|
||||
---
|
||||
|
||||
## M8 — Cutover + Python retirement
|
||||
|
||||
Goal: Go is the one true backend.
|
||||
|
||||
- [ ] **M8.1** Update bookmarks, README, CLAUDE.md to point at Go (`make web` aliases to `make web-go`)
|
||||
- [ ] **M8.2** Run Go-only for 2 weeks including a month-end settlement; keep Python container available but unrouted
|
||||
- [ ] **M8.3** Manual reconciliation review: produce a balance report on `python-final` and on Go for the same period; sign off they match
|
||||
- [ ] **M8.4** Tag final Python image as `python-final` in registry; remove Python service from `docker-compose.yml`
|
||||
- [ ] **M8.5** Delete [app.py](app.py), [scripts/](scripts/), Python `Dockerfile`, [tests/](tests/), `pyproject.toml`, `uv.lock`
|
||||
- [ ] **M8.6** Update [CLAUDE.md](CLAUDE.md) to reflect Go-only state (commands, architecture, key modules); CHANGELOG entry
|
||||
|
||||
**Gate:** Two consecutive months of Go-only operation with end-of-month settlement complete; zero rollbacks.
|
||||
|
||||
---
|
||||
|
||||
## Notes & decisions
|
||||
|
||||
(Add entries as you go. Format: `YYYY-MM-DD — short note`.)
|
||||
|
||||
- 2026-05-07 — `/api/version` excluded from parity diff by design. Each binary's tag/commit/build_date is identity, not a data contract — diffing it would always flag a diff between independently built backends. Route remains reachable via `make parity ARGS="-route /api/version"` for manual inspection.
|
||||
- 2026-05-04 — Plan approved. Versioning policy: latest stable for Go and all libs at the time M1 starts. Frontends explicitly allowed to diverge between Python and Go; only the JSON API contract is parity-locked. No reverse proxy — both backends run on different ports via `make web-py` / `make web-go`.
|
||||
- 2026-05-07 — M4 complete. Chose fakes-only unit tests (no live integration tests) and CSV-via-public-URL for attendance (no Sheets API auth required for read-only). golangci-lint gofumpt extra-rules differ slightly from standalone gofumpt; used `golangci-lint run --fix --enable-only gofumpt` to auto-resolve formatting.
|
||||
- 2026-05-04 — M1 complete. Dockerfile base changed from `distroless/static:nonroot` → `alpine:3` for debuggability (can tighten later). CLI dispatcher uses stdlib `flag`; module path `fuj-management/go`. golangci-lint v1 embedded gofumpt merges all imports into one group (no stdlib/local split) — accepted as the project style.
|
||||
424
docs/plans/2026-05-03-2349-go-backend-rewrite.md
Normal file
424
docs/plans/2026-05-03-2349-go-backend-rewrite.md
Normal file
@@ -0,0 +1,424 @@
|
||||
# Plan: Full Go rewrite of the Python/Flask backend
|
||||
|
||||
## Context
|
||||
|
||||
The current Flask app ([app.py](app.py) + [scripts/](scripts/), ~2400 LOC of
|
||||
Python) handles attendance-based fee calculation, Fio bank sync, payment
|
||||
reconciliation, and a server-rendered dashboard. The user wants a full
|
||||
rewrite in Go with two goals:
|
||||
|
||||
1. **Quality Go code** as the primary outcome — idiomatic stdlib-first
|
||||
design, strong typing, proper layering. The Python codebase grew
|
||||
organically and mixes domain logic, IO, and HTTP concerns.
|
||||
2. **Feature-parity certainty** — no behavioural drift between the Python
|
||||
and Go versions on anything that touches money. Reconciliation is real
|
||||
money; silent divergence is unacceptable.
|
||||
|
||||
**Switchable runtime**: both backends run on different TCP ports, started
|
||||
independently via Makefile targets (`make web-py` on :5001, `make web-go` on
|
||||
:8080). The user opens whichever they want in a browser. No reverse proxy,
|
||||
no traffic-splitting, no shared frontend constraint — just two services
|
||||
that read the same Google Sheets and the same `tmp/` cache.
|
||||
|
||||
**Frontends are allowed to diverge.** The Go web layer is designed cleanly
|
||||
in its own right rather than as a byte-compatible Jinja port. Both backends
|
||||
expose a JSON API (`/api/...`) with an identical contract — that's what
|
||||
parity testing locks down. Rendered HTML and inline JS can be different.
|
||||
|
||||
## Versioning policy
|
||||
|
||||
- **Go**: latest stable release at project start. Pin in `go.mod` via the
|
||||
`go` directive (e.g. `go 1.X`) and use the matching `golang:1.X` builder
|
||||
image. Bump on each new minor as it lands stable.
|
||||
- **Go libraries**: latest stable for every dependency in `go.mod`; run
|
||||
`go get -u ./... && go mod tidy` at the start and quarterly thereafter.
|
||||
- **Python deps** (during the parallel-run period): keep
|
||||
[pyproject.toml](pyproject.toml) on its current versions to avoid
|
||||
destabilizing the parity baseline; bump only after Python retires.
|
||||
- **Base images**: `golang:latest-stable` builder → `gcr.io/distroless/static:latest`
|
||||
runtime, both pinned by digest in CI for reproducibility.
|
||||
- **CI runners**: latest stable Linux image on Gitea Actions.
|
||||
|
||||
The plan does not hardcode specific version numbers below — implementation
|
||||
picks current-stable at the time M1 starts.
|
||||
|
||||
## Approach summary
|
||||
|
||||
- **Three-layer Go architecture**: pure domain (no IO) → IO clients (behind
|
||||
interfaces, easily faked) → HTTP/services (composition).
|
||||
- **Capture-then-port**: dump current Python outputs as JSON fixtures, port
|
||||
Go function-by-function, assert byte-equality with `cmp.Diff`.
|
||||
- **JSON contract is the spec, not the templates.** Each Python route gets
|
||||
an `/api/X` shadow that returns the dict already passed to the template.
|
||||
Go defines typed structs matching that shape; both sides validate against
|
||||
generated JSON Schema.
|
||||
- **Money is integer CZK**: existing fees are integer CZK (750/200/500);
|
||||
keep it that way to avoid float drift in reconcile allocation. Where
|
||||
Sheets returns floats, parse and round at the boundary.
|
||||
- **Frontend rewrite, not port**: Go uses `html/template` with cleanly
|
||||
organized templates and JS extracted into static files served via
|
||||
`embed.FS`. Same UX (filterable table, member-detail modal, QR launcher)
|
||||
but designed natively, no Jinja-port baggage.
|
||||
|
||||
## Go project layout
|
||||
|
||||
`go/` lives at the repo root alongside `scripts/` and `templates/` so both
|
||||
backends share the same git history during migration.
|
||||
|
||||
```
|
||||
go/
|
||||
cmd/
|
||||
fuj/main.go # single binary, subcommands: server | fees | sync | infer | reconcile
|
||||
parity/main.go # diff tool: hits both backends' /api/X, prints JSON diff
|
||||
internal/
|
||||
domain/ # pure, no IO, no net/*
|
||||
czech/ # normalize, parse_month_references
|
||||
fees/ # calculate_fee, calculate_junior_fee, "?" sentinel type
|
||||
money/ # parse_czk_amount, format helpers
|
||||
reconcile/ # reconcile() + Ledger, MemberResult types
|
||||
matching/ # _build_name_variants, match_members, infer_transaction_details
|
||||
synch/ # generate_sync_id (pure hash)
|
||||
io/ # IO behind interfaces, all impls have an in-memory fake
|
||||
sheets/ # SheetsClient + Google impl + fake
|
||||
drive/ # DriveClient for modifiedTime
|
||||
fio/ # FioClient: API JSON impl + transparent-page HTML scraper
|
||||
cache/ # FileCache with modifiedTime gating + two TTL knobs
|
||||
services/ # composition layer; pure + IO, no HTTP
|
||||
attendance/ # GetMembersWithFees, GetJuniorMembersWithFees
|
||||
payments/ # FetchTransactions, FetchExceptions, BuildView
|
||||
banksync/ # SyncToSheets, InferPayments (write ops)
|
||||
web/
|
||||
handlers/ # one file per route family
|
||||
view/ # HTML view-model structs (per route)
|
||||
api/ # JSON view-model structs (the parity-locked contract)
|
||||
templates/ # *.tmpl, embed.FS — designed natively, not a Jinja port
|
||||
static/ # js/*.js, css/*.css served via embed.FS
|
||||
middleware/ # request timer, recovery, slog
|
||||
config/ # mirrors scripts/config.py (env loading)
|
||||
qr/ # SPD string builder + PNG via go-qrcode
|
||||
tests/
|
||||
fixtures/ # JSON fixtures captured from Python (PII-scrubbed)
|
||||
parity/ # Go-side characterization tests (replay fixtures)
|
||||
build/Dockerfile # multi-stage: latest-stable golang builder → distroless static
|
||||
go.mod
|
||||
```
|
||||
|
||||
## Library choices
|
||||
|
||||
All on latest stable as per the versioning policy above.
|
||||
|
||||
| Concern | Pick | Rationale |
|
||||
|---|---|---|
|
||||
| HTTP routing | `net/http` ServeMux | 8 static routes; no need for chi/gin given modern stdlib pattern matching |
|
||||
| Templates | `html/template` | Auto-escaping; native Go feel |
|
||||
| Static assets | `embed.FS` | Single binary, no loose files |
|
||||
| Sheets/Drive | `google.golang.org/api/{sheets/v4,drive/v3}` + `option` | Official client; service-account auth via `option.WithCredentialsFile` |
|
||||
| OAuth | `golang.org/x/oauth2/google` (token only; drop installed-app flow + pickle) | Production already uses service accounts |
|
||||
| QR PNG | `github.com/skip2/go-qrcode` | Mature, byte-stable PNG output |
|
||||
| NFKD | `golang.org/x/text/unicode/norm` + `unicode.IsMark` | Direct equivalent of `unicodedata.normalize("NFKD", ...)` |
|
||||
| HTML scrape | `golang.org/x/net/html` token visitor | Counts `<table class="table">` to target the second one |
|
||||
| CSV | `encoding/csv` (stdlib) | Match for Python `csv.reader` |
|
||||
| Logging | `log/slog` (stdlib) | Honors `LOG_LEVEL` env |
|
||||
| Diff/testing | `testing` + `github.com/google/go-cmp/cmp` | Readable `cmp.Diff` for parity assertions |
|
||||
| Lint | `golangci-lint` (govet, staticcheck, errcheck, gofumpt, unused) | Standard quality gate |
|
||||
|
||||
## Migration sequencing — eight milestones with hard gates
|
||||
|
||||
**M1 — Skeleton + tooling.** Create `go/` tree, `go.mod` (latest stable
|
||||
Go), Makefile targets (`go-build`, `go-test`, `go-run`, `web-go`),
|
||||
`golangci-lint` config. `cmd/fuj server` prints a hello + version and
|
||||
listens on :8080.
|
||||
*Gate:* `make go-build` succeeds; `make web-go` serves a "hello" page on
|
||||
:8080 in parallel with `make web` on :5001; lint clean.
|
||||
|
||||
**M2 — Pure-domain helpers, port leaf-first.** Order:
|
||||
[czech_utils.py](scripts/czech_utils.py) `normalize` → `parse_month_references` →
|
||||
[attendance.py](scripts/attendance.py) `calculate_fee`/`calculate_junior_fee` →
|
||||
[infer_payments.py](scripts/infer_payments.py) `parse_czk_amount` →
|
||||
[sync_fio_to_sheets.py](scripts/sync_fio_to_sheets.py) `generate_sync_id` →
|
||||
[match_payments.py](scripts/match_payments.py) helpers (`_build_name_variants`,
|
||||
`match_members`, `infer_transaction_details`, `format_date`) → `reconcile`.
|
||||
Each gets a Go unit test plus a parity test driven by JSON fixtures from M3.
|
||||
Also: `fuj fees` and `fuj reconcile` subcommands wired up (pure-domain CLIs).
|
||||
*Gate:* All ported helpers pass parity tests.
|
||||
|
||||
**M3 — Fixture capture + characterization framework.** Build
|
||||
`scripts/capture_fixtures.py` (Python helper that prints function results as
|
||||
JSON to stdout — user pipes to disk) and `scripts/scrub_fixtures.py`
|
||||
(replaces member names with deterministic pseudonyms `Member_<8hex>`,
|
||||
scrambles sender/account/VS/bank_id while preserving structural
|
||||
relationships, dates, amounts, exception keys). Capture ~10 reconcile
|
||||
fixtures spanning every code path: greedy, proportional with float
|
||||
remainder, even-split fallback, out-of-window credit, exception override,
|
||||
`other:` purpose, junior `"?"`, comma-separated multi-person, multi-month
|
||||
range, unmatched.
|
||||
*Gate:* `tests/fixtures/` populated and committed; M2 parity tests green.
|
||||
|
||||
**M4 — IO layer behind interfaces.** Implement Sheets/Drive/Fio clients
|
||||
matching Python return shapes. Drop the OAuth+pickle path entirely (service
|
||||
account only). All clients have in-memory fakes for tests. Wire `fuj sync`
|
||||
and `fuj infer` subcommands.
|
||||
*Gate:* `go test -tags=integration ./internal/io/...` round-trips against a
|
||||
test sheet (separate from prod); default-tag tests use fakes.
|
||||
|
||||
**M5 — JSON-only `/api/...` routes.** Add 8 Go route handlers that return
|
||||
JSON. Add symmetric `/api/X` shadow endpoints in [app.py](app.py) that
|
||||
`jsonify` the existing view-model dict (no transformation).
|
||||
*Gate:* For each route, `cmd/parity` asserts
|
||||
`cmp.Diff(python.json, go.json) == ""` modulo allowlist
|
||||
(`render_time.total`, `build_meta`).
|
||||
|
||||
**M6 — Go-native HTML frontend.** Design Go templates cleanly (not a Jinja
|
||||
port). Extract JS from inline into `internal/web/static/js/*.js` served via
|
||||
`embed.FS`. Vanilla JS, no framework — same UX as Python (sortable table,
|
||||
member-detail modal, name filter, month range filter, QR launcher) but
|
||||
organized as proper modules. Templates render the JSON API response into
|
||||
HTML; frontend JS fetches additional data from `/api/X` for the modal
|
||||
rather than embedding `member_data` in `<script>`.
|
||||
*Gate:* Browser smoke test of all routes on :8080 covers: name filter,
|
||||
month filter, modal opens with correct months/transactions/exceptions, QR
|
||||
modal renders, navigation between adults/juniors/payments works.
|
||||
|
||||
**M7 — Parallel-running watch period.** Both `make web-py` and `make web-go`
|
||||
running locally (and in production via two containers on different ports).
|
||||
Daily/manual `cmd/parity` runs catch any JSON drift. The user verifies the
|
||||
Go UI matches what they expect feature-by-feature against the Python UI.
|
||||
Run 1–2 weeks.
|
||||
*Gate:* Zero non-allowlisted JSON diffs over 7 consecutive days, including
|
||||
a sync-bank execution, a flush, and an attendance update. User sign-off
|
||||
that the Go UI is feature-complete.
|
||||
|
||||
**M8 — Cutover + Python retirement.** Switch the bookmarked URL / docs to
|
||||
the Go port. Keep Python container running but unrouted (or stopped) for
|
||||
1 week as rollback. Then delete [app.py](app.py), [scripts/](scripts/),
|
||||
the Python `Dockerfile`, and the Python tests. Update
|
||||
[CLAUDE.md](CLAUDE.md) to reflect the Go-only state.
|
||||
*Gate:* Two consecutive months of Go-only operation including end-of-month
|
||||
settlement.
|
||||
|
||||
## CLI port (decided: port as Go subcommands)
|
||||
|
||||
Single Go binary `fuj` with subcommands replacing the existing Makefile
|
||||
targets. Each reuses the domain layer directly:
|
||||
|
||||
| Old | New | Backed by | Milestone |
|
||||
|---|---|---|---|
|
||||
| `make fees` | `fuj fees` | `domain/fees` + `services/attendance` | M2 |
|
||||
| `make reconcile` | `fuj reconcile` | `domain/reconcile` | M2 |
|
||||
| `make sync-2026` | `fuj sync --year=2026` | `services/banksync.SyncToSheets` | M4 |
|
||||
| `make infer` | `fuj infer [--dry-run]` | `services/banksync.InferPayments` | M4 |
|
||||
| `make web` (py) | stays as Python `make web-py` until M8 | — | — |
|
||||
| `make web-go` | `fuj server` | `web/handlers` | M1 |
|
||||
|
||||
Makefile targets get rewritten to invoke `./bin/fuj <subcommand>` once each
|
||||
is ported. The Python `make` targets for already-ported commands stay as
|
||||
`make X-py` aliases until M8, so you can run either side for cross-checks.
|
||||
|
||||
## JSON API contract strategy
|
||||
|
||||
**Go-defines, Python-conforms** with a 1-step bootstrap:
|
||||
|
||||
1. Run Python locally and dump `result["members"]`, `formatted_results`,
|
||||
`monthly_totals`, etc., to JSON. This is the spec.
|
||||
2. Hand-author Go structs with explicit `json:` tags matching exact Python
|
||||
keys (`total_balance`, `original_expected`, `attendance_count` — no
|
||||
reliance on default lowercasing).
|
||||
3. Generate `tests/fixtures/api-schema/*.schema.json` from the Go structs
|
||||
using `github.com/invopop/jsonschema`. Commit them.
|
||||
4. Add a Python-side schema validator running in CI against the new
|
||||
`/api/X` responses.
|
||||
|
||||
**Two known-tricky shapes:**
|
||||
|
||||
- Junior `expected: int | "?"` →
|
||||
```go
|
||||
type Expected struct{ Value int; Unknown bool }
|
||||
// MarshalJSON emits 42 or "?"
|
||||
```
|
||||
Same for `original_expected`.
|
||||
- Tuple dict keys `(normalize(name), normalize(period))` for exceptions —
|
||||
internal only, never crosses JSON. Use
|
||||
`map[ExceptionKey]Exception` with `ExceptionKey struct{ Name, Period string }`.
|
||||
|
||||
## Characterization test harness — two tiers
|
||||
|
||||
(HTML rendering parity dropped: frontends are intentionally different.)
|
||||
|
||||
**Tier 1 — Pure-function parity** (fast, every commit). Fixtures at
|
||||
`tests/fixtures/pure/<func>/<case>.json` containing `{input, output}`,
|
||||
captured once via `scripts/capture_fixtures.py`. Go test reads each, calls
|
||||
the ported function, asserts deep equality with `cmp.Diff`. Functions in
|
||||
scope: `normalize`, `parse_month_references`, `parse_czk_amount`,
|
||||
`parse_czech_amount`, `parse_czech_date`, `format_date`,
|
||||
`_build_name_variants`, `match_members`, `infer_transaction_details`,
|
||||
`generate_sync_id`, `calculate_fee`, `calculate_junior_fee`, `reconcile`.
|
||||
|
||||
**Tier 2 — JSON API parity** (medium, on PR + nightly). `cmd/parity/main.go`
|
||||
hits both `:5001/api/X` and `:8080/api/X` with a fixture-seeded `tmp/`
|
||||
cache, normalizes volatile fields (`render_time`, build metadata), asserts
|
||||
byte-equality. Cache freezing: pre-populate `tmp/*_cache.json` from
|
||||
scrubbed snapshots so both backends read identical data.
|
||||
|
||||
**PII scrubbing** is mandatory ([CLAUDE.md](CLAUDE.md): "Member data must
|
||||
never be committed"). `scripts/scrub_fixtures.py` produces deterministic
|
||||
pseudonyms preserving uniqueness and structural relationships. Only
|
||||
scrubbed fixtures land in `tests/fixtures/`; raw `tmp/*.json` stays
|
||||
gitignored.
|
||||
|
||||
## Side-by-side runtime
|
||||
|
||||
Two services on different ports, started independently. No reverse proxy.
|
||||
|
||||
```
|
||||
make web-py # Python on :5001 (existing target, perhaps renamed from `make web`)
|
||||
make web-go # Go on :8080
|
||||
```
|
||||
|
||||
Both read the same Google Sheets and write to the same `tmp/` cache
|
||||
directory. The user opens `localhost:5001` or `localhost:8080` directly to
|
||||
A/B compare.
|
||||
|
||||
**Cache directory coordination**: both backends use `tmp/`. Go writes via
|
||||
`os.WriteFile` to `tmp/<key>_cache.json.tmp` then `os.Rename` (atomic on
|
||||
Linux). Python's writes are pre-existing-non-atomic; accept until Python
|
||||
retires.
|
||||
|
||||
**Sync coordination**: `/sync-bank` is non-idempotent under concurrency.
|
||||
Both backends `flock` on `tmp/sync.lock`; Go uses `syscall.Flock`. (In
|
||||
practice the user is unlikely to trigger sync from both UIs at once, but
|
||||
the lock is cheap insurance.)
|
||||
|
||||
**Production deployment**: keep the existing Python container; add a Go
|
||||
container in `docker-compose.yml` exposed on a different port. After M8,
|
||||
remove the Python service.
|
||||
|
||||
## CI/CD
|
||||
|
||||
Currently zero test CI ([.gitea/workflows/build.yaml](.gitea/workflows/build.yaml)
|
||||
only does `docker build`/`push`). Add `/.gitea/workflows/test.yml`:
|
||||
|
||||
```yaml
|
||||
jobs:
|
||||
python-tests: # fix M3 broken-test references first
|
||||
- uv sync && pytest tests/
|
||||
go-tests:
|
||||
- cd go && go test -race ./...
|
||||
- cd go && golangci-lint run
|
||||
parity-pure: # Tier 1
|
||||
- cd go && go test -tags=parity ./tests/parity/...
|
||||
```
|
||||
|
||||
Branch protection: `python-tests`, `go-tests`, `parity-pure` block merge.
|
||||
Tier-2 parity runs nightly via `parity-nightly.yml` (boots both servers
|
||||
via docker-compose with seeded caches, replays a fixed transaction script,
|
||||
fails on any non-allowlisted diff).
|
||||
|
||||
A new Go `build/Dockerfile` (multi-stage: latest-stable `golang` builder →
|
||||
`gcr.io/distroless/static:latest`, both pinned by digest) mirrors the
|
||||
existing Python build job and produces a single static binary image.
|
||||
|
||||
## Risk register (top 4)
|
||||
|
||||
(Template auto-escape divergence dropped: irrelevant when frontends differ.)
|
||||
|
||||
1. **Sync ID hash drift** — HIGH/HIGH. Python builds the SHA-256 input by
|
||||
`str()`-ing each field then `.lower()`-ing the joined string;
|
||||
`str(750.0) == "750.0"`, `str(750) == "750"`. If Sheets API returns
|
||||
floats in Python but Go unmarshals as int, `750` vs `750.0` → different
|
||||
hash → duplicate rows. *Mitigation:* dedicated parity test with ~50
|
||||
real-row fixtures; if Go can't reproduce Python's float string format,
|
||||
normalize at the boundary (round to 2 decimals, format with explicit
|
||||
precision).
|
||||
2. **Float allocation in `reconcile()` proportional phase** — HIGH/MEDIUM.
|
||||
Python's "last month absorbs remainder" depends on dict iteration order;
|
||||
Go map iteration is randomized. *Mitigation:* always iterate
|
||||
`sorted_months` explicitly in Go, never the map. Lock the distribution
|
||||
with a parity test on (300, 300, 150) months × 751-CZK payment.
|
||||
3. **NFKD edge cases** — MEDIUM/MEDIUM. Python `unicodedata` and Go
|
||||
`golang.org/x/text` use the same algorithm but can differ on niche
|
||||
compatibility decompositions if `x/text` is older than CPython's tables.
|
||||
*Mitigation:* parity test with every distinct character ever observed in
|
||||
member names; pin `x/text` version explicitly.
|
||||
4. **Czech month parser semantics** — MEDIUM/MEDIUM. Wrap-around year
|
||||
inference (`if start_m > end_m and m >= start_m: year = default_year - 1`)
|
||||
plus the "month >= 10 → previous year" heuristic are easy to mis-port.
|
||||
*Mitigation:* port table and algorithm verbatim line-for-line; parity
|
||||
test with ~30 real `message`-field fixture strings.
|
||||
|
||||
## Cutover plan
|
||||
|
||||
Simpler without a proxy in the middle:
|
||||
|
||||
1. After M7's 7-day clean window + user sign-off, treat Go as primary.
|
||||
Update bookmarks, docs, `make web` to point at Go.
|
||||
2. Keep `make web-py` available for 1-week rollback. Run both containers
|
||||
in production but only point users at the Go one.
|
||||
3. Watch 2 weeks including a month-end settlement on Go-only.
|
||||
4. Decommission Python: remove from `docker-compose.yml`, delete
|
||||
[app.py](app.py) and [scripts/](scripts/), update
|
||||
[CLAUDE.md](CLAUDE.md). Keep image tagged `python-final` in registry as
|
||||
a 6-month rollback option.
|
||||
|
||||
**Retirement criteria:** zero parity-diff incidents in last 30 days, zero
|
||||
rollbacks, two month-end settlements completed Go-only, manual
|
||||
reconciliation review against `python-final` signed off.
|
||||
|
||||
## Critical files
|
||||
|
||||
- [scripts/match_payments.py](scripts/match_payments.py) — `reconcile()` is
|
||||
the single most load-bearing function (~200 lines of allocation logic)
|
||||
that must port byte-equivalently.
|
||||
- [scripts/czech_utils.py](scripts/czech_utils.py) — `normalize` and
|
||||
`parse_month_references` underpin every member/month match across the
|
||||
system. 45 Czech month declensions, range wrap-around, year inference.
|
||||
- [app.py](app.py) — defines the 8-route HTTP surface and view-model
|
||||
shapes. The spec for the Go web layer's JSON API.
|
||||
- [scripts/sync_fio_to_sheets.py](scripts/sync_fio_to_sheets.py) —
|
||||
`generate_sync_id` defines the dedup contract against existing rows in
|
||||
the live sheet. Any drift creates duplicates.
|
||||
- [scripts/attendance.py](scripts/attendance.py) — fee math + merged-month
|
||||
logic + junior `"?"` sentinel.
|
||||
- [scripts/cache_utils.py](scripts/cache_utils.py) — Drive `modifiedTime`
|
||||
gating + two-TTL fallback that must be reproduced for shared-cache
|
||||
safety.
|
||||
- [templates/adults.html](templates/adults.html) — read for the JSON shape
|
||||
the existing inline JS consumes (`member_data`); the Go frontend doesn't
|
||||
have to mirror the template, but the JSON contract derived from this
|
||||
page's data injection is the parity spec.
|
||||
|
||||
## Verification
|
||||
|
||||
End-to-end checks per milestone:
|
||||
|
||||
- **M1**: `make go-build && ./bin/fuj server --help` prints subcommand
|
||||
list. `make web-go` serves :8080 in parallel with `make web-py` on :5001.
|
||||
- **M2-M3**: `cd go && go test -tags=parity ./tests/parity/pure/...` green.
|
||||
Spot-check: feed a known Czech-message string through both
|
||||
`parse_month_references` implementations, diff outputs.
|
||||
- **M4**: `go test -tags=integration ./internal/io/sheets/...` round-trips
|
||||
against a test sheet (separate from prod).
|
||||
- **M5**: `curl localhost:5001/api/adults | jq -S . > py.json && curl
|
||||
localhost:8080/api/adults | jq -S . > go.json && diff py.json go.json` —
|
||||
empty diff modulo allowlist.
|
||||
- **M6**: Browser open `localhost:8080/adults`, click a member row, modal
|
||||
opens with all months / transactions / exceptions correctly populated.
|
||||
Same on `/juniors`. Click a Pay button → QR loads. Name filter and month
|
||||
range filter work.
|
||||
- **M7**: Run `cd go && ./bin/parity --base http://localhost:5001
|
||||
--candidate http://localhost:8080 --routes adults,juniors,payments`
|
||||
daily for 7 days, zero non-allowlisted diffs. User confirms Go UI is
|
||||
feature-complete vs Python UI side-by-side.
|
||||
- **M8**: `make web-py` removed from Makefile; `make web` points at Go;
|
||||
manual end-of-month settlement on Go matches the prior month's
|
||||
Python-produced report.
|
||||
|
||||
## Open questions / forks the user can override at review
|
||||
|
||||
- **Frontend JS organization in M6**: default is vanilla JS in separate
|
||||
files via `embed.FS`. If the user wants HTMX, Alpine.js, or a small
|
||||
framework, raise it before M6.
|
||||
- **CI host**: Gitea Actions assumed (matches existing
|
||||
[.gitea/workflows/build.yaml](.gitea/workflows/build.yaml)).
|
||||
- **Test sheet for M4 integration tests**: would need provisioning.
|
||||
Confirm whether to use a copy of the production sheet (PII!) or a
|
||||
synthetic one seeded by the fixture-capture process.
|
||||
233
docs/plans/2026-05-04-1115-go-rewrite-m1-kickoff.md
Normal file
233
docs/plans/2026-05-04-1115-go-rewrite-m1-kickoff.md
Normal file
@@ -0,0 +1,233 @@
|
||||
# Plan: Go rewrite — M1 kickoff (skeleton + tooling)
|
||||
|
||||
Companion to [2026-05-03-2349-go-backend-rewrite.md](2026-05-03-2349-go-backend-rewrite.md)
|
||||
and the progress tracker
|
||||
[2026-05-03-2349-go-backend-rewrite-progress.md](2026-05-03-2349-go-backend-rewrite-progress.md).
|
||||
|
||||
## Context
|
||||
|
||||
The master plan for a full Go rewrite of the Flask backend is approved
|
||||
(2026-05-04). No Go code exists yet — this plan executes **M1** end-to-end:
|
||||
a working `go/` skeleton, a `fuj` binary with a `server` subcommand serving
|
||||
a hello page on `:8080`, lint config, Makefile + CI integration, and an
|
||||
`internal/config` package mirroring [scripts/config.py](scripts/config.py).
|
||||
|
||||
After M1, both backends run side-by-side locally (`make web-py` on `:5001`,
|
||||
`make web-go` on `:8080`) — that side-by-side capability is what unblocks
|
||||
M2's parity testing and every later milestone.
|
||||
|
||||
## Locked-in decisions
|
||||
|
||||
| # | Decision | Choice |
|
||||
|---|---|---|
|
||||
| 1 | CLI dispatcher | stdlib `flag` + `os.Args[1]` switch (no cobra) |
|
||||
| 2 | Go module path | `fuj-management/go` |
|
||||
| 3 | Go version | `1.26` (latest stable; user toolchain is `go1.26.1`) |
|
||||
| 4 | M1 scope | all 10 progress-tracker sub-tasks in one session |
|
||||
| 5 | Lint | `golangci-lint` with govet, staticcheck, errcheck, gofumpt, unused |
|
||||
| 6 | Logging | `log/slog` text handler, level from `LOG_LEVEL` env |
|
||||
| 7 | HTTP | `net/http.ServeMux` (Go 1.22+ pattern matching) |
|
||||
| 8 | Container base | `golang:1.26` builder → `gcr.io/distroless/static:nonroot` runtime |
|
||||
| 9 | CI | extend [.gitea/workflows/build.yaml](.gitea/workflows/build.yaml) with a `go-build` job parallel to existing Python `build` job; tag suffix `-go` |
|
||||
|
||||
## Files to create
|
||||
|
||||
```
|
||||
go/
|
||||
go.mod # module fuj-management/go, go 1.26
|
||||
go.sum # empty / generated
|
||||
.golangci.yml # govet, staticcheck, errcheck, gofumpt, unused
|
||||
cmd/fuj/main.go # subcommand dispatcher + version vars
|
||||
internal/
|
||||
config/config.go # env loader mirroring scripts/config.py
|
||||
logging/logger.go # slog setup honoring LOG_LEVEL
|
||||
web/
|
||||
server.go # `fuj server` handler: ServeMux on :8080, hello page
|
||||
middleware/timer.go # request-timer middleware (parity with Python `get_render_time`)
|
||||
build/
|
||||
Dockerfile # multi-stage golang:1.26 → distroless/static
|
||||
```
|
||||
|
||||
No `embed.FS`, no templates, no static assets in M1 — the hello page is
|
||||
inline HTML in `server.go`. Templates land in M6.
|
||||
|
||||
## Files to edit
|
||||
|
||||
- [Makefile](Makefile) — add Go targets, rename `web` → `web-py`, keep
|
||||
`web` as transitional alias to `web-py` until M8.
|
||||
- [.gitignore](.gitignore) — add `bin/` and `go/.cache/` (if any).
|
||||
- [.gitea/workflows/build.yaml](.gitea/workflows/build.yaml) — add
|
||||
`go-build` job that builds and pushes `<tag>-go` image.
|
||||
- [CHANGELOG.md](CHANGELOG.md) — top-of-file entry per CLAUDE.md convention.
|
||||
- [docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md](docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md)
|
||||
— tick M1.1–M1.10 with commit SHAs as they land.
|
||||
|
||||
## Execution sequence
|
||||
|
||||
Order is tight: each step keeps the tree compilable and lint-clean.
|
||||
|
||||
1. **Skeleton (M1.1)** — `mkdir -p go/{cmd/fuj,internal/{config,logging,web/middleware},build}` and `cd go && go mod init fuj-management/go`. Pin `go 1.26` in `go.mod`.
|
||||
|
||||
2. **Config + logger (M1.8, M1.9)** — write `internal/config/config.go` mirroring [scripts/config.py](scripts/config.py): exported constants for `AttendanceSheetID`, `PaymentsSheetID`, `JuniorSheetGID`, env-driven `CredentialsPath`, `BankAccount`, `CacheTTL`, `CacheAPICheckTTL`, `LogLevel`, `FioAPIToken`. Write `internal/logging/logger.go` with a `New() *slog.Logger` honoring `LOG_LEVEL` (`DEBUG|INFO|WARN|ERROR`).
|
||||
|
||||
3. **Web middleware + handler (M1.3)** — `internal/web/middleware/timer.go` logs `method path status ms` for every request. `internal/web/server.go` exposes `Run(ctx, addr) error`: `http.ServeMux` with `GET /` returning a minimal HTML hello page that includes `version`, `commit`, and `buildDate` (linker-injected via `-X main.version=…`).
|
||||
|
||||
4. **Subcommand dispatcher (M1.2)** — `cmd/fuj/main.go`:
|
||||
- Package-level `var version, commit, buildDate string` for `-ldflags -X` injection.
|
||||
- `os.Args[1]` switch over `server | version | fees | reconcile | sync | infer | help`. M1 implements `server` and `version`; the rest print `<cmd>: not implemented yet (lands in M2/M4)` and exit 2.
|
||||
- Each subcommand parses its own `flag.NewFlagSet`. `server` flags: `--addr` (default `:8080`).
|
||||
|
||||
5. **Lint config (M1.6)** — `go/.golangci.yml` enabling `govet`, `staticcheck`, `errcheck`, `gofumpt`, `unused`. Run `golangci-lint run ./...` to confirm clean.
|
||||
|
||||
6. **Makefile (M1.4, M1.5)** — add:
|
||||
```make
|
||||
GO_BIN := bin/fuj
|
||||
GO_SRC := go
|
||||
|
||||
go-build:
|
||||
cd $(GO_SRC) && go build -trimpath \
|
||||
-ldflags "-X main.version=$$(git describe --tags --always 2>/dev/null || echo dev) \
|
||||
-X main.commit=$$(git rev-parse --short HEAD) \
|
||||
-X main.buildDate=$$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
|
||||
-o ../$(GO_BIN) ./cmd/fuj
|
||||
|
||||
go-test:
|
||||
cd $(GO_SRC) && go test -race ./...
|
||||
|
||||
go-run: go-build
|
||||
./$(GO_BIN) $(ARGS)
|
||||
|
||||
go-lint:
|
||||
cd $(GO_SRC) && golangci-lint run ./...
|
||||
|
||||
web-go: go-build
|
||||
./$(GO_BIN) server --addr :8080
|
||||
```
|
||||
Rename existing `web:` target to `web-py:` and add `web: web-py` as alias.
|
||||
|
||||
7. **Dockerfile + CI (M1.7)** — `go/build/Dockerfile`:
|
||||
```dockerfile
|
||||
FROM golang:1.26 AS build
|
||||
WORKDIR /src
|
||||
COPY go/go.mod go/go.sum ./
|
||||
RUN go mod download
|
||||
COPY go/ ./
|
||||
ARG GIT_TAG=unknown
|
||||
ARG GIT_COMMIT=unknown
|
||||
ARG BUILD_DATE=unknown
|
||||
RUN CGO_ENABLED=0 go build -trimpath \
|
||||
-ldflags "-s -w -X main.version=${GIT_TAG} -X main.commit=${GIT_COMMIT} -X main.buildDate=${BUILD_DATE}" \
|
||||
-o /out/fuj ./cmd/fuj
|
||||
|
||||
FROM gcr.io/distroless/static:nonroot
|
||||
COPY --from=build /out/fuj /usr/local/bin/fuj
|
||||
EXPOSE 8080
|
||||
USER nonroot:nonroot
|
||||
ENTRYPOINT ["/usr/local/bin/fuj","server"]
|
||||
```
|
||||
In [.gitea/workflows/build.yaml](.gitea/workflows/build.yaml), add a parallel job:
|
||||
```yaml
|
||||
build-go:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: docker login ...
|
||||
- run: |
|
||||
docker build -f go/build/Dockerfile \
|
||||
--build-arg GIT_TAG=$TAG \
|
||||
--build-arg GIT_COMMIT=${{ github.sha }} \
|
||||
--build-arg BUILD_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) \
|
||||
-t gitea.home.hrajfrisbee.cz/${{ github.repository }}:$TAG-go .
|
||||
docker push gitea.home.hrajfrisbee.cz/${{ github.repository }}:$TAG-go
|
||||
```
|
||||
|
||||
8. **Smoke verify (M1.10)** — see Verification section below; then append a CHANGELOG entry and tick M1 boxes in the progress tracker with commit SHAs.
|
||||
|
||||
## Reuse / parity with Python side
|
||||
|
||||
- `internal/config` mirrors [scripts/config.py](scripts/config.py) **exactly** — same env var names, same defaults. No new env knobs in M1.
|
||||
- Request-timer middleware records elapsed milliseconds; this is the Go-side
|
||||
equivalent of the Python `get_render_time` helper that supplies
|
||||
`render_time.total` to templates. Allowlisted as volatile in the future
|
||||
parity diff (M5).
|
||||
- Constants `AttendanceSheetID`, `PaymentsSheetID`, `JuniorSheetGID` are
|
||||
copied verbatim from [scripts/config.py](scripts/config.py); they don't
|
||||
get used until M4 but live in `internal/config` from day one.
|
||||
|
||||
## Verification
|
||||
|
||||
Run from repo root after all changes are in place:
|
||||
|
||||
```bash
|
||||
# 1. Builds clean
|
||||
make go-build && test -x bin/fuj
|
||||
|
||||
# 2. Lint clean
|
||||
make go-lint
|
||||
|
||||
# 3. Subcommand dispatcher works
|
||||
./bin/fuj help
|
||||
./bin/fuj version # prints version/commit/buildDate
|
||||
./bin/fuj fees # prints "not implemented yet" and exits 2
|
||||
|
||||
# 4. Server runs and hello page is served
|
||||
make web-go &
|
||||
GO_PID=$!
|
||||
sleep 1
|
||||
curl -sf http://localhost:8080/ | grep -q "fuj"
|
||||
kill $GO_PID
|
||||
|
||||
# 5. Side-by-side: both backends up
|
||||
make web-py & # :5001
|
||||
PY_PID=$!
|
||||
make web-go & # :8080
|
||||
GO_PID=$!
|
||||
sleep 2
|
||||
curl -sf http://localhost:5001/ >/dev/null && echo "py OK"
|
||||
curl -sf http://localhost:8080/ >/dev/null && echo "go OK"
|
||||
kill $PY_PID $GO_PID
|
||||
|
||||
# 6. Race-free unit tests pass (none yet beyond a smoke test, but harness works)
|
||||
make go-test
|
||||
|
||||
# 7. Docker image builds locally
|
||||
docker build -f go/build/Dockerfile -t fuj-go:dev .
|
||||
docker run --rm -p 8080:8080 fuj-go:dev &
|
||||
sleep 1
|
||||
curl -sf http://localhost:8080/ >/dev/null && echo "container OK"
|
||||
docker stop $(docker ps -lq)
|
||||
```
|
||||
|
||||
All seven steps must succeed. Then update the progress tracker and
|
||||
CHANGELOG.
|
||||
|
||||
## Out of scope for M1 (deferred to later milestones)
|
||||
|
||||
- Domain logic — `czech.Normalize`, fees, reconcile, etc. → **M2**.
|
||||
- Fixture capture and parity tests → **M3**.
|
||||
- Sheets/Drive/Fio clients and `internal/io/*` → **M4**.
|
||||
- `/api/*` JSON routes and `cmd/parity` → **M5**.
|
||||
- HTML templates, static assets, `embed.FS` → **M6**.
|
||||
- Removing the Python backend → **M8**.
|
||||
|
||||
## Open items / forks the user can override at review
|
||||
|
||||
- **CI tag suffix**: `<tag>-go` proposed. Alternative: separate image
|
||||
repository (`fuj-management-go:<tag>`). The suffix keeps things in one
|
||||
registry path; speak up if separate repos are preferred.
|
||||
- **Distroless variant**: `nonroot` chosen for least privilege. If the
|
||||
existing Python container runs as root and the user expects parity,
|
||||
switch to `gcr.io/distroless/static` (root). Doesn't affect M1
|
||||
functionality.
|
||||
- **Hello page content**: minimal HTML mentioning `fuj`, version, commit,
|
||||
build date, link list to future routes. Speak up if you want a different
|
||||
shape — it gets thrown away in M6 anyway.
|
||||
|
||||
## Critical files
|
||||
|
||||
- [docs/plans/2026-05-03-2349-go-backend-rewrite.md](docs/plans/2026-05-03-2349-go-backend-rewrite.md) — master plan (approved 2026-05-04)
|
||||
- [docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md](docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md) — task tracker; tick M1.1–M1.10 here
|
||||
- [Makefile](Makefile) — current target structure (renaming `web` → `web-py`)
|
||||
- [scripts/config.py](scripts/config.py) — source of truth for env vars / IDs that `internal/config` mirrors
|
||||
- [build/Dockerfile](build/Dockerfile) — Python container (unchanged); the new Go Dockerfile lives at `go/build/Dockerfile`
|
||||
- [.gitea/workflows/build.yaml](.gitea/workflows/build.yaml) — extended with parallel `build-go` job
|
||||
81
docs/plans/2026-05-04-2249-payment-name-match-exact.md
Normal file
81
docs/plans/2026-05-04-2249-payment-name-match-exact.md
Normal file
@@ -0,0 +1,81 @@
|
||||
# Exact full-name match for payment inference
|
||||
|
||||
## Context
|
||||
|
||||
A bank payment with the message `Henrietta Ottová (Heny): 04/2026` is being inferred to **two** members: the correct `Henrietta Ottová` *and* the unrelated `Tomáš Němeček (Tov)`. As a result, `reconcile()` splits the amount 50/50 between them, producing wrong balances.
|
||||
|
||||
**Root cause** (`scripts/match_payments.py:51-115`): `match_members` runs four substring checks via raw Python `in`, with no word boundaries. Tomáš's nickname `Tov` normalizes to `tov`, which is literally a substring of `ottova`. Check #3 (`match_payments.py:79-85`) treats bare nickname presence as an `auto`-confidence match, so Tomáš is appended even though no part of his name is actually in the message. There is also no short-circuit when a member's full canonical name appears verbatim — every other member is still scored against the same haystack.
|
||||
|
||||
**Goal:** when a member's full canonical name (diacritics-insensitive) appears in the message as whole words, return only the full-name hit(s) and skip nickname/partial scoring entirely. Additionally, harden the remaining checks with word boundaries so future substring collisions (any nickname or short name part that happens to live inside another member's surname) can't reproduce this class of bug.
|
||||
|
||||
## Approach
|
||||
|
||||
Single-file change in [scripts/match_payments.py](scripts/match_payments.py). Two coordinated edits to `match_members` (`match_payments.py:51-115`):
|
||||
|
||||
### 1. Add an exact-canonical-name short-circuit (new, before the existing loop)
|
||||
|
||||
After computing `normalized_text`, do a first pass that collects every member whose `normalized_base` (the full name minus the parenthesized nickname, normalized) appears in the haystack as **whole words**. If at least one is found, return *only* those as `auto` matches and skip the rest of the function.
|
||||
|
||||
Implementation sketch (inserted between [match_payments.py:58](scripts/match_payments.py#L58) and [match_payments.py:61](scripts/match_payments.py#L61)):
|
||||
|
||||
```python
|
||||
exact_matches = []
|
||||
for name in member_names:
|
||||
variants = _build_name_variants(name)
|
||||
full_name = variants[0] if variants else ""
|
||||
if full_name and re.search(rf"\b{re.escape(full_name)}\b", normalized_text):
|
||||
exact_matches.append((name, "auto"))
|
||||
if exact_matches:
|
||||
return exact_matches
|
||||
```
|
||||
|
||||
This satisfies the user's primary ask: when the message literally contains the canonical name, that wins outright. Multi-member messages still work — every full-name occurrence is collected.
|
||||
|
||||
### 2. Replace remaining `in normalized_text` checks with `\b…\b` regex
|
||||
|
||||
For the three checks that survive the short-circuit (and the `review`-tier partials), swap raw `in` for whole-word regex so `tov` cannot match inside `ottova`, `dan` cannot match inside `bohdan`, etc. Affected lines:
|
||||
|
||||
- [match_payments.py:73](scripts/match_payments.py#L73) — first+last name both present
|
||||
- [match_payments.py:82](scripts/match_payments.py#L82) — nickname presence
|
||||
- [match_payments.py:94](scripts/match_payments.py#L94) — last-name partial (`review`)
|
||||
- [match_payments.py:99](scripts/match_payments.py#L99) — first-name partial (`review`)
|
||||
- [match_payments.py:104](scripts/match_payments.py#L104) — single-name member partial
|
||||
|
||||
Helper to keep the call sites tidy:
|
||||
|
||||
```python
|
||||
def _word_in(needle: str, haystack: str) -> bool:
|
||||
return bool(re.search(rf"\b{re.escape(needle)}\b", haystack))
|
||||
```
|
||||
|
||||
Check #1 (line 67) becomes redundant once the short-circuit is in place, but leave it untouched as a defensive fallback in case `_build_name_variants` ever returns a `full_name` shorter than the 3-char filter would allow. (No code change there.)
|
||||
|
||||
### 3. Why this is sufficient
|
||||
|
||||
- The reported message `Henrietta Ottová (Heny): 04/2026` hits the new short-circuit on `henrietta ottova`, returns `[("Henrietta Ottová", "auto")]`, and never even evaluates Tomáš.
|
||||
- Bare-nickname messages (e.g. `Heny 04/2026`) skip the short-circuit (no full name present) and fall into the existing nickname check — now word-bounded, so `tov` no longer collides with `ottova` even there.
|
||||
- Combined-payment messages listing two full names continue to work: both are collected by the short-circuit.
|
||||
|
||||
### Files to modify
|
||||
|
||||
- [scripts/match_payments.py](scripts/match_payments.py) — only `match_members` (lines 51-115). Add `_word_in` helper just above it.
|
||||
|
||||
### Files to read for confidence (no edits)
|
||||
|
||||
- [scripts/czech_utils.py](scripts/czech_utils.py) — confirm `normalize()` semantics (NFKD strip + lowercase). Already understood; relevant because `re.escape` on already-normalized lowercase ASCII is safe.
|
||||
- [scripts/infer_payments.py](scripts/infer_payments.py) — confirm it just consumes the `match_members` output verbatim and writes comma-joined names. No change needed; the upstream fix propagates.
|
||||
- [scripts/match_payments.py:336-362](scripts/match_payments.py#L336-L362) — `reconcile()` only re-runs inference when `Person` is empty, so existing wrong rows in the sheet must be cleared by hand or via the `manual fix`/blank-cell workflow before re-running `make infer`.
|
||||
|
||||
## Verification
|
||||
|
||||
1. **Unit test** — add `tests/test_match_members.py` (new file, mirroring `tests/test_reconcile_exceptions.py` style). Cases:
|
||||
- `match_members("Henrietta Ottová (Heny): 04/2026", ["Henrietta Ottová", "Tomáš Němeček (Tov)"])` → `[("Henrietta Ottová", "auto")]` only.
|
||||
- `match_members("Heny 04/2026", ["Tomáš Němeček (Tov)", "Henrietta Ottová"])` → no match for Tomáš (the substring trap is closed); whatever the legitimate behavior for "Heny" is, document it.
|
||||
- Combined payment: `match_members("Henrietta Ottová a Tomáš Němeček 04/2026", ["Henrietta Ottová", "Tomáš Němeček (Tov)"])` → both as `auto`.
|
||||
- Sanity: `match_members("VS 1234 Tomáš Němeček", [...])` still returns Tomáš.
|
||||
|
||||
2. **Run the suite**: `make test`.
|
||||
|
||||
3. **End-to-end**: clear the buggy row's `Person`/`Purpose` cells in the payments sheet, then `make infer`, then `make reconcile`. Confirm the payment now allocates fully to Henrietta and balance reflects it.
|
||||
|
||||
4. **Changelog**: per [CLAUDE.md](CLAUDE.md), append an entry to [CHANGELOG.md](CHANGELOG.md) once the user confirms the fix works in production. Format: `## 2026-05-04 HH:MM TZ — fix: payment inference exact-match short-circuit`.
|
||||
@@ -0,0 +1,99 @@
|
||||
# Member modal — raw payments debug list
|
||||
|
||||
## Context
|
||||
|
||||
When a payer's bank message doesn't follow our convention, [`infer_payments.py`](scripts/infer_payments.py) may map the transfer to the wrong period (or none), and today the member detail modal hides this — it only shows the post-allocation, per-month splits produced by [`reconcile()`](scripts/match_payments.py:295). To diagnose these cases the user needs to see the **original sheet rows** that were attributed to a member: full `Amount`, `Inferred Amount`, `Person`, `Purpose`, `Sender`, `Message`, `Bank ID`, `manual fix`. The list should be hidden by default and revealed by a small toggle, since it is only relevant during debugging.
|
||||
|
||||
## Approach
|
||||
|
||||
Reuse the grouping logic that already exists in the [`/payments` route](app.py:540-553): group raw `tx` dicts by parsed `Person`, expose that mapping to the modal, and render it on demand under a new collapsible section.
|
||||
|
||||
### 1. Backend — group raw txs by member
|
||||
|
||||
In [`app.py`](app.py):
|
||||
|
||||
- Factor the existing per-person grouping in [`payments()`](app.py:530-568) into a small helper near the top of the file:
|
||||
```python
|
||||
def group_payments_by_person(transactions):
|
||||
grouped = {}
|
||||
for tx in transactions:
|
||||
person = str(tx.get("person", "")).strip()
|
||||
if not person:
|
||||
continue # unmatched rows are not tied to a member
|
||||
for p in person.split(","):
|
||||
p = re.sub(r"\[\?\]\s*", "", p).strip()
|
||||
if not p:
|
||||
continue
|
||||
grouped.setdefault(p, []).append(tx)
|
||||
for rows in grouped.values():
|
||||
rows.sort(key=lambda t: str(t.get("date", "")), reverse=True)
|
||||
return grouped
|
||||
```
|
||||
Call it from [`payments()`](app.py:530), [`adults_view()`](app.py:160) and [`juniors_view()`](app.py:326) — the existing `payments()` body collapses to one line.
|
||||
|
||||
- In `adults_view()` and `juniors_view()`, after `transactions = get_cached_data(...)`, build `raw_payments_by_person = group_payments_by_person(transactions)` and pass it to `render_template` as `raw_payments_json=json.dumps(raw_payments_by_person)`.
|
||||
|
||||
- Note: rows where `Person` is empty are skipped on purpose — those have no member to attach to and are already shown by the dashboard's `Unmatched` block.
|
||||
|
||||
### 2. Templates — add a collapsible raw section to the modal
|
||||
|
||||
In [`templates/adults.html`](templates/adults.html) and [`templates/juniors.html`](templates/juniors.html), make the same structural and JS changes (the modal markup is mirrored in both files — `adults.html:677-682` and `juniors.html:658-663`).
|
||||
|
||||
- Inject the new dataset alongside the existing `memberData`:
|
||||
```html
|
||||
const rawPaymentsByPerson = {{ raw_payments_json| safe }};
|
||||
```
|
||||
(next to [`adults.html:696`](templates/adults.html#L696)).
|
||||
|
||||
- Add a new section directly **after** the Payment History block:
|
||||
```html
|
||||
<div class="modal-section">
|
||||
<div class="modal-section-title">
|
||||
Raw Payments
|
||||
<a href="#" id="rawPaymentsToggle" class="raw-toggle"
|
||||
onclick="toggleRawPayments(event)">[show]</a>
|
||||
</div>
|
||||
<div id="modalRawList" class="tx-list" style="display: none;">
|
||||
<!-- Filled by JS -->
|
||||
</div>
|
||||
</div>
|
||||
```
|
||||
Add a small CSS rule for `.raw-toggle` (muted color, smaller font, `margin-left: 8px`) — a few lines next to the existing `.modal-section-title` style. Don't restyle the whole modal.
|
||||
|
||||
- In `showMemberDetails(name)`:
|
||||
- Reset the toggle to `[show]` and the `#modalRawList` to `display: none` on every open (so the state doesn't leak between members).
|
||||
- Populate `#modalRawList` from `rawPaymentsByPerson[name] || []`. For each row render: `Date | Purpose` on the meta line, `Amount CZK` (with `Inferred: X CZK` annotation when `inferred_amount` differs from `amount`), `Sender`, `Person` (full string — useful when split between multiple people), `Message`, and a small footer with `Bank ID` and a `[manual fix]` marker if `manual_fix` is truthy. Reuse the existing `tx-item` / `tx-meta` / `tx-main` / `tx-msg` styles to match the rest of the modal.
|
||||
- When the list is empty, render `<div style="color: #444; font-style: italic; padding: 10px 0;">No raw payments tied to this member.</div>` (same idiom used at [`adults.html:813`](templates/adults.html#L813)).
|
||||
|
||||
- Add the toggle handler near `closeModal`:
|
||||
```js
|
||||
function toggleRawPayments(ev) {
|
||||
ev.preventDefault();
|
||||
const list = document.getElementById('modalRawList');
|
||||
const link = document.getElementById('rawPaymentsToggle');
|
||||
const hidden = list.style.display === 'none';
|
||||
list.style.display = hidden ? 'block' : 'none';
|
||||
link.textContent = hidden ? '[hide]' : '[show]';
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Why not extend `reconcile()` instead
|
||||
|
||||
`reconcile()` already collapses each row into per-month allocated shares and drops `purpose`, `inferred_amount`, `bank_id`, `manual_fix`, and the gross `amount` ([trace](scripts/match_payments.py:436-469)). Carrying the raw `tx` through `reconcile()` would inflate the contract for every consumer when only the modal needs it. Grouping the already-fetched `transactions` list at the route level is one extra dict per request and reuses the cached payments data — no new sheet reads.
|
||||
|
||||
## Critical files
|
||||
|
||||
- [app.py](app.py) — add `group_payments_by_person()` helper; call it in `adults_view()`, `juniors_view()`, and `payments()`; pass `raw_payments_json` to the two dashboard templates.
|
||||
- [templates/adults.html](templates/adults.html) — modal section + JS + tiny CSS for the toggle link.
|
||||
- [templates/juniors.html](templates/juniors.html) — same changes as adults.html.
|
||||
|
||||
## Verification
|
||||
|
||||
1. `make web-debug` and open `http://localhost:5001/adults`.
|
||||
2. Pick a member known to have multiple payments (use the existing `/payments` page as a cross-reference).
|
||||
3. Click `[i]` → modal opens, raw list is hidden, link shows `[show]`. Click the link → list appears with the raw rows; click again → hides, link returns to `[show]`.
|
||||
4. Switch to another member via keyboard (ArrowDown) — the toggle resets to hidden and the list updates to the new member's rows (no leaking).
|
||||
5. Compare the raw rows in the modal against the `/payments` page grouping for the same person — same set of rows, same `Date`/`Amount`/`Message`.
|
||||
6. Pick a row with a non-conformant message (e.g. one where `Person` was inferred to multiple people) — confirm `Person` shows the full comma-separated string and `Inferred Amount` is visible when it differs from `Amount`.
|
||||
7. Repeat the click-through on `/juniors` to confirm parity.
|
||||
8. `make test` — no backend behavior change is expected, but run to catch template/route smoke breakage.
|
||||
@@ -0,0 +1,135 @@
|
||||
# Tolerate diacritic / case / whitespace mismatches between `Person` column and member names
|
||||
|
||||
## Context
|
||||
|
||||
For "Mária Maco" there is a payment row in the payments sheet with `Purpose = 2026-04`, but the modal for that member shows neither a paid 2026-04 cell **nor** a row in payment history. Both symptoms collapse to a single root cause in [`reconcile()`](scripts/match_payments.py#L295), confirmed by reading the code:
|
||||
|
||||
- [`scripts/match_payments.py:404`](scripts/match_payments.py#L404) — `if member_name not in ledger:` is a **byte-exact** comparison. `member_name` is the `Person` cell from the payments sheet with only `.strip()` and `[?]` markers removed ([:349-353](scripts/match_payments.py#L349-L353)). `ledger` keys are the canonical names from the attendance sheet. There is no diacritic, case, or whitespace normalization on this path. (`czech_utils.normalize` is imported and used for the `exceptions` lookup at [:282-283 / :321-322](scripts/match_payments.py#L282-L322), but **not** for member-name matching.)
|
||||
- When a row falls through that check, it is appended to `unmatched` and never reaches `ledger[member_name][m]['paid']` or `['transactions']`. The dashboard's per-month "paid" cell stays unpaid, and because the modal's payment history is built from `data.months[m].transactions` ([`templates/adults.html:772-776`](templates/adults.html#L772-L776)), the row also disappears from the modal's history list.
|
||||
- The new "Raw Payments" debug section ([`templates/adults.html:861`](templates/adults.html#L861)) uses `rawPaymentsByPerson[name]`. Its keys come from [`group_payments_by_person()` in `app.py:60-73`](app.py#L60-L73), which also stores the **literal** `Person` string (only `.strip()` and `[?]` stripped). So if the attendance-sheet name and the `Person` cell differ at the byte level, that section also returns an empty list — which is why the user does not see the row anywhere in the modal.
|
||||
|
||||
The most likely cause for "Mária Maco" specifically: the `Person` cell was typed (or pasted) without the `á` diacritic — `Maria Maco` vs `Mária Maco`. Other plausible variants the current code silently drops: case differences (`mária maco`), trailing/embedded extra whitespace, and NBSP characters.
|
||||
|
||||
The fix is to make the matching tolerant via the existing [`czech_utils.normalize()`](scripts/czech_utils.py#L22-L25) helper (NFKD + lowercase), with a small whitespace-collapse on top, and apply the same canonicalization in `group_payments_by_person()` so the modal's raw-payments lookup uses the canonical attendance-sheet name as the key.
|
||||
|
||||
## Approach
|
||||
|
||||
### 1. `scripts/match_payments.py` — tolerant `Person` → `ledger` resolution in `reconcile()`
|
||||
|
||||
- Add a small private helper at module scope:
|
||||
|
||||
```python
|
||||
def _canonical_key(name: str) -> str:
|
||||
return re.sub(r"\s+", " ", normalize(name)).strip()
|
||||
```
|
||||
|
||||
Uses the existing `normalize()` from `czech_utils` ([:22-25](scripts/czech_utils.py#L22-L25)) and additionally collapses whitespace runs to a single space so `"Mária Maco"` and `"Mária Maco"` both reduce to `"maria maco"`.
|
||||
|
||||
- Inside [`reconcile()`](scripts/match_payments.py#L295), right after `member_names` is computed ([:308](scripts/match_payments.py#L308)), build a lookup dict once:
|
||||
|
||||
```python
|
||||
canonical_by_key: dict[str, str] = {}
|
||||
for name in member_names:
|
||||
key = _canonical_key(name)
|
||||
canonical_by_key.setdefault(key, name) # first wins; ambiguity handled below
|
||||
```
|
||||
|
||||
- Replace the byte-exact check at [:404](scripts/match_payments.py#L404). Resolve each `member_name` from `matched_members` to the canonical attendance-sheet name before any ledger / credits access:
|
||||
|
||||
```python
|
||||
for raw_member_name, confidence in matched_members:
|
||||
member_name = canonical_by_key.get(_canonical_key(raw_member_name))
|
||||
if member_name is None:
|
||||
logger.warning(
|
||||
"Payment matched to unknown member %r (tx: %s, %s) — adding to unmatched",
|
||||
raw_member_name, tx.get("date", "?"), tx.get("message", "?"),
|
||||
)
|
||||
unmatched.append(tx)
|
||||
continue
|
||||
if member_name != raw_member_name:
|
||||
logger.info(
|
||||
"Person cell %r resolved to canonical member %r — consider fixing the sheet",
|
||||
raw_member_name, member_name,
|
||||
)
|
||||
# ... rest of the loop body unchanged: ledger[member_name], credits[member_name], …
|
||||
```
|
||||
|
||||
The `logger.info` line lets the user see (in `make web-debug` logs) which sheet rows have a non-canonical `Person` value, so they can clean them up at their own pace — without breaking allocation in the meantime.
|
||||
|
||||
- Leave the rest of the function untouched. Once `member_name` is the canonical name, every downstream key (`ledger[member_name]`, `credits[member_name]`, `other_ledger[member_name]`, the `tx["person"]` echo into `transactions`) is already correct.
|
||||
|
||||
### 2. `app.py` — canonicalize the raw-payments grouping key
|
||||
|
||||
- The current [`group_payments_by_person()`](app.py#L60-L73) cannot canonicalize on its own because it does not know the attendance-sheet member list. Extend its signature to accept the member list and reuse `_canonical_key`:
|
||||
|
||||
```python
|
||||
from match_payments import _canonical_key # or re-export via a tiny public name
|
||||
|
||||
def group_payments_by_person(transactions, member_names=None):
|
||||
canonical_by_key = (
|
||||
{_canonical_key(n): n for n in member_names} if member_names else {}
|
||||
)
|
||||
grouped = {}
|
||||
for tx in transactions:
|
||||
person = str(tx.get("person", "")).strip()
|
||||
if not person:
|
||||
continue
|
||||
for p in person.split(","):
|
||||
p = re.sub(r"\[\?\]\s*", "", p).strip()
|
||||
if not p:
|
||||
continue
|
||||
key = canonical_by_key.get(_canonical_key(p), p) # fallback: keep raw
|
||||
grouped.setdefault(key, []).append(tx)
|
||||
for rows in grouped.values():
|
||||
rows.sort(key=lambda t: str(t.get("date", "")), reverse=True)
|
||||
return grouped
|
||||
```
|
||||
|
||||
- Update the three call sites to pass `member_names`:
|
||||
- `adults_view()` around [`app.py:333`](app.py#L333) — `members` is already in scope; pass `[name for name, _, _ in members]`.
|
||||
- `juniors_view()` around [`app.py:539`](app.py#L539) — same.
|
||||
- `payments()` around [`app.py:549`](app.py#L549) — same; needs the adult+junior member names so the `/payments` per-person grouping is consistent.
|
||||
|
||||
- Naming: `_canonical_key` starts with an underscore inside `match_payments.py`. To avoid leaking a private symbol, expose it as `canonical_member_key` (no underscore) in `match_payments.py` and import that name from `app.py`.
|
||||
|
||||
### 3. Why not also touch `infer_payments.py`
|
||||
|
||||
`infer_payments.py` already writes canonical attendance-sheet names into the `Person` column (it picks from `member_names`). The bug only manifests when the cell was filled in **manually** by a human (typed without diacritics, different case) or was written by an older inference that has since drifted from a renamed attendance row. Making `reconcile()` tolerant fixes the symptom for both cases without changing inference. The `logger.info` line is sufficient signal for the user to clean up the sheet on their own schedule.
|
||||
|
||||
### 4. Tests
|
||||
|
||||
**4a. Delete obsolete route tests in [tests/test_app.py](tests/test_app.py).** Four tests target Flask routes that no longer exist (the old fee/reconcile pages were merged into `/adults` and `/juniors`); they currently fail with 404. Their coverage is already provided by `test_adults_route`, `test_juniors_route`, and `test_payments_route`. Delete:
|
||||
|
||||
- `test_fees_route` ([tests/test_app.py:22-35](tests/test_app.py#L22-L35)) — hits `/fees`
|
||||
- `test_fees_juniors_route` ([tests/test_app.py:37-55](tests/test_app.py#L37-L55)) — hits `/fees-juniors`
|
||||
- `test_reconcile_route` ([tests/test_app.py:57-81](tests/test_app.py#L57-L81)) — hits `/reconcile`; also asserts a literal `OK` string the merged dashboard no longer renders
|
||||
- `test_reconcile_juniors_route` ([tests/test_app.py:101-131](tests/test_app.py#L101-L131)) — hits `/reconcile-juniors`; same `OK` assertion mismatch
|
||||
|
||||
The two tests that reference junior-only formatting (`? / 1 (J)` and `500 CZK / 4 (1A+3J)`) are testing a retired template, not the live `/juniors` page — no need to migrate those assertions; the live `/juniors` format is already covered by `test_juniors_route`.
|
||||
|
||||
**4b. Add `tests/test_match_payments.py`** (new file) covering the resolution helper and `reconcile()` end-to-end for the canonicalization fix:
|
||||
|
||||
- `_canonical_key("Mária Maco") == _canonical_key("maria maco")`
|
||||
- `reconcile()` with member `"Mária Maco"` and a tx `{person: "Maria Maco", purpose: "2026-04", amount: 750, ...}` produces:
|
||||
- `result['members']['Mária Maco']['months']['2026-04']['paid'] == 750`
|
||||
- the tx appears in `result['members']['Mária Maco']['months']['2026-04']['transactions']`
|
||||
- `result['unmatched']` is empty
|
||||
- `reconcile()` with `Person = "Někdo Neznámý"` (no match in members) still routes to `unmatched`.
|
||||
|
||||
## Critical files
|
||||
|
||||
- [scripts/match_payments.py](scripts/match_payments.py) — add `canonical_member_key()` helper; build `canonical_by_key` once in `reconcile()`; resolve `raw_member_name` → `member_name` before ledger access at [:404](scripts/match_payments.py#L404).
|
||||
- [app.py](app.py) — extend `group_payments_by_person()` to accept `member_names` and key the grouped dict by canonical attendance-sheet name; update three call sites.
|
||||
- [tests/test_app.py](tests/test_app.py) — delete the four obsolete route tests listed in §4a.
|
||||
- [tests/test_match_payments.py](tests/test_match_payments.py) — add the cases above (create the file if missing).
|
||||
- [docs/plans/](docs/plans/) — per project [CLAUDE.md](CLAUDE.md), move this plan file to `docs/plans/2026-05-05-1640-payment-person-name-canonicalization.md` once execution starts (the plan-mode harness writes to `~/.claude/plans/` by default).
|
||||
|
||||
## Verification
|
||||
|
||||
1. **Reproduce first.** Before touching code, open `/adults`, click `[i]` next to "Mária Maco", and confirm both: 2026-04 is unpaid and the payment is missing from history. Inspect the actual `Person` cell value in the payments sheet for the 2026-04 row — confirm it differs from `"Mária Maco"` (likely missing the `á`). Record the exact string for the test case.
|
||||
2. `make test` — new tests pass; existing tests still green.
|
||||
3. `make web-debug` and reload `/adults`. The 2026-04 cell for "Mária Maco" turns green (`cell-ok`); the modal's payment history shows the row; the "Raw Payments" section also shows the row. Server log emits `Person cell 'Maria Maco' resolved to canonical member 'Mária Maco' — consider fixing the sheet`.
|
||||
4. Cross-check `/payments` — the row appears under the `Mária Maco` group (canonical key), not under a separate `Maria Maco` group.
|
||||
5. Spot-check one member with the conventionally-correct `Person` value (e.g. one of the recent payers visible on the dashboard) — paid cells and history are unchanged, no spurious resolution log line.
|
||||
6. Confirm a payment with a genuinely unknown `Person` (typo of a non-member) still ends up in the dashboard's `Unmatched` block and emits the existing `Payment matched to unknown member …` warning.
|
||||
7. Append a `CHANGELOG.md` entry per [CLAUDE.md](CLAUDE.md) once the user confirms the fix works.
|
||||
83
docs/plans/2026-05-05-2144-branch-per-feature-workflow.md
Normal file
83
docs/plans/2026-05-05-2144-branch-per-feature-workflow.md
Normal file
@@ -0,0 +1,83 @@
|
||||
# Branch-per-feature + Gitea MR workflow
|
||||
|
||||
## Context
|
||||
|
||||
Until now, Claude has been committing feature work directly to `main`
|
||||
(see recent history: `feat: Lower adult monthly fee…`, `feat: Go rewrite M1…`,
|
||||
all on `main`). The user wants to switch to a branch-per-feature flow with
|
||||
review via a Gitea merge request, so that:
|
||||
|
||||
- Feature work is reviewable as a self-contained diff before it lands.
|
||||
- `main` stays releasable.
|
||||
- The change history shows reviewed merges, not unsupervised pushes.
|
||||
|
||||
The remote is Gitea (`https://gitea.home.hrajfrisbee.cz/kacerr/fuj-management.git`),
|
||||
which supports the standard pull/merge-request flow.
|
||||
|
||||
This plan only modifies `CLAUDE.md`. No code changes.
|
||||
|
||||
## Scope clarification (from user)
|
||||
|
||||
- **MR creation method:** Claude pushes the branch and prints the Gitea
|
||||
"compare" URL. The user opens / merges the MR in the browser. No `tea` CLI,
|
||||
no API calls.
|
||||
- **When the flow applies:** Features only. Small bug fixes and hotfixes can
|
||||
still be committed straight to `main`. Claude decides feature-vs-fix based
|
||||
on scope; when uncertain, ask.
|
||||
- **Branch naming:** `feat/<slug>` for features, `fix/<slug>` for the
|
||||
occasional bug-fix branch the user explicitly requests. `<slug>` is
|
||||
kebab-case, short, descriptive.
|
||||
|
||||
## Change
|
||||
|
||||
Add a new top-level section to `CLAUDE.md` titled **"Branching & merge requests"**,
|
||||
placed immediately before the existing `## Git Commits` section so the workflow
|
||||
context appears before the commit-message convention.
|
||||
|
||||
### Proposed section content
|
||||
|
||||
```markdown
|
||||
## Branching & merge requests
|
||||
|
||||
The remote is Gitea (`gitea.home.hrajfrisbee.cz/kacerr/fuj-management`).
|
||||
For **features**, do not commit to `main` directly. Use a branch + merge
|
||||
request flow:
|
||||
|
||||
1. **Create a branch off `main`** before starting work:
|
||||
- `feat/<slug>` for features (e.g. `feat/qr-code-overlay`)
|
||||
- `fix/<slug>` for bug-fix branches the user explicitly asks for
|
||||
- `<slug>` is short kebab-case
|
||||
2. **Commit on the branch** following the existing commit conventions
|
||||
(Co-Authored-By trailer, etc.).
|
||||
3. **Push the branch** to `origin` with `-u` so it tracks.
|
||||
4. **Print the Gitea compare URL** so the user can open the MR in the
|
||||
browser:
|
||||
`https://gitea.home.hrajfrisbee.cz/kacerr/fuj-management/compare/main...<branch>`
|
||||
Do **not** use `tea`, `gh`, or call the Gitea API — the user opens and
|
||||
merges the MR themselves.
|
||||
5. **Do not merge or delete the branch** from the CLI. The user does that
|
||||
in Gitea.
|
||||
|
||||
**Exceptions — when committing straight to `main` is fine:**
|
||||
- Small bug fixes / hotfixes the user describes as such.
|
||||
- Typo / comment / formatting tweaks.
|
||||
- Edits the user explicitly says to push to `main`.
|
||||
|
||||
When uncertain whether something is "feature" or "small fix", ask before
|
||||
committing.
|
||||
```
|
||||
|
||||
## Files to modify
|
||||
|
||||
- [CLAUDE.md](CLAUDE.md) — insert the new `## Branching & merge requests`
|
||||
section just above the existing `## Git Commits` section (around line 95).
|
||||
|
||||
## Verification
|
||||
|
||||
- Re-read `CLAUDE.md` and confirm the new section is well-placed and the
|
||||
existing structure (`## Git Commits`, `## Changelog`, `## Plans`) is intact.
|
||||
- `git diff CLAUDE.md` should show only an additive change.
|
||||
- No code, tests, or runtime behavior changes — nothing else to test.
|
||||
- Behavior verification happens on the **next** feature request: Claude
|
||||
should create a `feat/<slug>` branch, commit there, push, and print the
|
||||
compare URL instead of committing on `main`.
|
||||
154
docs/plans/2026-05-05-2204-go-rewrite-m2-1-czech-normalize.md
Normal file
154
docs/plans/2026-05-05-2204-go-rewrite-m2-1-czech-normalize.md
Normal file
@@ -0,0 +1,154 @@
|
||||
# Plan: Go rewrite — M2.1 `domain/czech.Normalize`
|
||||
|
||||
## Context
|
||||
|
||||
The Go rewrite finished M1 (skeleton, tooling, hello server) in commit
|
||||
`cf0f176` on 2026-05-04. The next milestone, **M2 — Pure-domain helpers**,
|
||||
is current per [progress tracker](2026-05-03-2349-go-backend-rewrite-progress.md)
|
||||
but has no work landed yet (all 12 sub-tasks unchecked).
|
||||
|
||||
This plan covers only the **first** M2 task: porting Python's
|
||||
`normalize` from [scripts/czech_utils.py](../../scripts/czech_utils.py)
|
||||
to Go as `internal/domain/czech.Normalize`. It is the lowest-level helper
|
||||
in the domain — `parse_month_references`, `_build_name_variants`,
|
||||
`match_members`, exception keys, and `reconcile` all transitively depend
|
||||
on it. Getting it byte-equivalent first removes a class of "why does my
|
||||
match not fire" failures from every later M2 task.
|
||||
|
||||
**Decision (confirmed in plan-mode Q):** start with hand-written Go unit
|
||||
tests for fresh Czech edge cases. Defer parity-fixture wiring until
|
||||
M3.1/M3.2 land (separate task); add the parity test for `Normalize`
|
||||
retroactively at that point.
|
||||
|
||||
## Scope
|
||||
|
||||
- New package `go/internal/domain/czech/` with `Normalize` and unit tests.
|
||||
- Add `golang.org/x/text` dependency to `go/go.mod` (currently zero deps).
|
||||
- **Out of scope:** `ParseMonthReferences` (M2.2), fixture tooling
|
||||
(M3.1/M3.2), CLI subcommand wiring (M2.11/M2.12), parity test runner.
|
||||
|
||||
## Recommended approach
|
||||
|
||||
### Python contract to match
|
||||
|
||||
```python
|
||||
def normalize(text: str) -> str:
|
||||
nfkd = unicodedata.normalize("NFKD", text)
|
||||
return "".join(c for c in nfkd if not unicodedata.combining(c)).lower()
|
||||
```
|
||||
|
||||
Three semantic operations:
|
||||
1. NFKD decompose
|
||||
2. Drop characters where `unicodedata.combining(c)` is non-zero
|
||||
3. Lowercase
|
||||
|
||||
### Go implementation
|
||||
|
||||
`go/internal/domain/czech/normalize.go`:
|
||||
|
||||
```go
|
||||
package czech
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode"
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
||||
func Normalize(s string) string {
|
||||
decomposed := norm.NFKD.String(s)
|
||||
var b strings.Builder
|
||||
b.Grow(len(decomposed))
|
||||
for _, r := range decomposed {
|
||||
if unicode.In(r, unicode.Mn) {
|
||||
continue
|
||||
}
|
||||
b.WriteRune(r)
|
||||
}
|
||||
return strings.ToLower(b.String())
|
||||
}
|
||||
```
|
||||
|
||||
**Two precision points worth flagging:**
|
||||
|
||||
1. **`unicode.Mn` not `unicode.IsMark`.** The plan's library-choices
|
||||
table mentions `unicode.IsMark`, but that covers Mn + Mc + Me. Python
|
||||
`unicodedata.combining()` returns 0 for Mc/Me (their canonical
|
||||
combining class is 0), so it effectively filters only Mn. Use
|
||||
`unicode.In(r, unicode.Mn)` for byte-equivalence with Python. Cite
|
||||
this in a one-line code comment; it's the kind of thing a future
|
||||
reader will second-guess.
|
||||
2. **`strings.ToLower` vs Go's locale-aware tools.** Python's `.lower()`
|
||||
on already-decomposed Latin is straight ASCII lowercase for Czech.
|
||||
Stdlib `strings.ToLower` matches; do not pull in `golang.org/x/text/cases`.
|
||||
|
||||
### Tests
|
||||
|
||||
`go/internal/domain/czech/normalize_test.go` — table-driven, covers:
|
||||
|
||||
- ASCII passthrough: `"Honza" → "honza"`
|
||||
- Czech lowercase diacritics: `"žluťoučký" → "zlutoucky"`
|
||||
- Mixed case + diacritics: `"Příliš" → "prilis"`
|
||||
- Czech caron + ring: `"Dvořák" → "dvorak"`, `"Růžena" → "ruzena"`
|
||||
- Hard letters: `"Čeněk" → "cenek"`, `"Kačer" → "kacer"`
|
||||
- Empty string: `"" → ""`
|
||||
- Already-normalized: `"prilis" → "prilis"` (idempotence)
|
||||
- Pre-composed vs decomposed input both produce the same output (NFC
|
||||
`"é"` and `"é"` both → `"e"`)
|
||||
- Whitespace preserved: `"Jan Novák" → "jan novak"`
|
||||
|
||||
Run a one-shot cross-check against the live Python implementation for
|
||||
each test input before locking the table:
|
||||
```
|
||||
PYTHONPATH=scripts:. python -c \
|
||||
'from czech_utils import normalize; print(repr(normalize("Dvořák")))'
|
||||
```
|
||||
This is the manual stand-in for the M3 parity fixtures.
|
||||
|
||||
### Wire-up
|
||||
|
||||
- `go get golang.org/x/text@latest` (run from `go/`); `go mod tidy`.
|
||||
- No CLI changes — `cmd/fuj` already stubs `fees`/`reconcile` with
|
||||
exit code 2; no need to touch dispatcher for this task. `Normalize`
|
||||
is consumed by other domain code, not by users directly.
|
||||
|
||||
## Critical files
|
||||
|
||||
- New: [go/internal/domain/czech/normalize.go](../../go/internal/domain/czech/normalize.go)
|
||||
- New: [go/internal/domain/czech/normalize_test.go](../../go/internal/domain/czech/normalize_test.go)
|
||||
- Modified: [go/go.mod](../../go/go.mod), `go/go.sum` (new)
|
||||
- Reference (read-only): [scripts/czech_utils.py](../../scripts/czech_utils.py) — the porting source
|
||||
- Reference (read-only): [docs/plans/2026-05-03-2349-go-backend-rewrite.md](2026-05-03-2349-go-backend-rewrite.md) — risk #3 (NFKD edge cases)
|
||||
|
||||
## Verification
|
||||
|
||||
End-to-end checks before marking M2.1 done:
|
||||
|
||||
1. `cd go && go build ./...` — clean compile.
|
||||
2. `cd go && go test ./internal/domain/czech/...` — all table cases green.
|
||||
3. `cd go && go test -race ./...` — race-clean.
|
||||
4. `cd go && golangci-lint run` (or `make go-lint` from repo root) — clean.
|
||||
5. **Spot parity** (manual, will be automated in M3): for each Go test
|
||||
input, run the Python `normalize` via `PYTHONPATH=scripts:. python -c
|
||||
'...'` and confirm bytes match. Capture the diff in the commit
|
||||
message if anything surprises.
|
||||
6. `make go-build && make go-test && make go-lint` from repo root — proves
|
||||
the existing M1 gate still passes.
|
||||
|
||||
## Branching & follow-up
|
||||
|
||||
Per [CLAUDE.md](../../CLAUDE.md), this is feature work → branch + Gitea MR:
|
||||
|
||||
- Branch: `feat/m2-1-czech-normalize` off `main`.
|
||||
- Single commit, Co-Authored-By trailer.
|
||||
- Push with `-u`, print compare URL
|
||||
`https://gitea.home.hrajfrisbee.cz/kacerr/fuj-management/compare/main...feat/m2-1-czech-normalize`
|
||||
- User opens/merges the MR.
|
||||
- After merge: tick `M2.1` in the progress tracker with the commit SHA;
|
||||
add a one-line CHANGELOG entry; record any porting surprise in the
|
||||
tracker's "Notes & decisions" section (e.g. the `Mn`-vs-`IsMark`
|
||||
precision point if it bears noting).
|
||||
|
||||
Next task after this lands is **M2.2 `ParseMonthReferences`** — the
|
||||
larger, edge-case-heavier sibling. Whether to start it before or after
|
||||
M3.1/M3.2 is a separate decision the user can make then.
|
||||
@@ -0,0 +1,205 @@
|
||||
# Plan: Go rewrite — M2.2 `domain/czech.ParseMonthReferences`
|
||||
|
||||
## Context
|
||||
|
||||
M2.1 (`domain/czech.Normalize`) merged via PR #4 (`d9a61b3`) on
|
||||
2026-05-05. Per the [progress tracker](2026-05-03-2349-go-backend-rewrite-progress.md),
|
||||
**M2.2** is next: port `parse_month_references` from
|
||||
[scripts/czech_utils.py](../../scripts/czech_utils.py) to Go as
|
||||
`internal/domain/czech.ParseMonthReferences`.
|
||||
|
||||
This function is the second-most-load-bearing pure helper after
|
||||
`reconcile`: every payment-message → month inference goes through it.
|
||||
Risk #4 in the [parent plan](2026-05-03-2349-go-backend-rewrite.md)
|
||||
specifically calls out its semantics — wrap-around year inference and
|
||||
the `m >= 10 → previous year` standalone heuristic — as easy to mis-port.
|
||||
|
||||
This plan locks the test table against the live Python implementation
|
||||
*before* coding, so the Go port has a verified parity baseline even
|
||||
before the M3.1/M3.2 fixture infrastructure exists.
|
||||
|
||||
## Scope
|
||||
|
||||
- New file `go/internal/domain/czech/parse_month_references.go` in the
|
||||
existing `czech` package (alongside [normalize.go](../../go/internal/domain/czech/normalize.go)).
|
||||
- New file `go/internal/domain/czech/parse_month_references_test.go`
|
||||
with the test table below.
|
||||
- **Out of scope:** parity-fixture wiring (M3.1/M3.2); CLI hook-up
|
||||
(M2.11/M2.12); any consumer call-sites.
|
||||
- **No new dependencies** — stdlib `regexp`, `sort`, `strconv`, `strings`
|
||||
plus the existing `czech.Normalize` cover everything.
|
||||
|
||||
## Recommended approach
|
||||
|
||||
### Python contract to mirror
|
||||
|
||||
Three regex passes, all run on `normalize(text)`:
|
||||
|
||||
1. `([\d+]+)\s*/\s*(\d{2,4})` — captures `"11+12/2025"`, `"01/26"`, `"1/26"`.
|
||||
Split the months part on `+`, keep digit-only tokens, validate `1..12`.
|
||||
Year < 100 → year + 2000.
|
||||
2. `(\d{1,2})\s*\.\s*(\d{4})` — captures `"12.2025"`. **4-digit year only**
|
||||
(so `"1.26"` does not match).
|
||||
3. Czech month names. First the **range** sub-pass:
|
||||
`(name)\s*-\s*(name)` finds pairs; walk start→end with `m % 12 + 1`,
|
||||
stopping when `m == end_m`. Wrap rule: if `start_m > end_m`, months
|
||||
`>= start_m` are `defaultYear - 1`, the rest are `defaultYear`. Both
|
||||
matched names go into a `foundInRanges` set.
|
||||
Then the **standalone** sub-pass: `\b(name)\b`, skipping any name in
|
||||
`foundInRanges`. For each remaining match, `m >= 10 → defaultYear - 1`,
|
||||
else `defaultYear`.
|
||||
|
||||
Output: sorted, deduplicated `[]string` of `"YYYY-MM"`.
|
||||
|
||||
### Go signature
|
||||
|
||||
```go
|
||||
package czech
|
||||
|
||||
// ParseMonthReferences extracts YYYY-MM month references from Czech
|
||||
// free text. defaultYear seeds two heuristics: standalone month names
|
||||
// with m >= 10 are treated as defaultYear-1 (out-of-year backfill), and
|
||||
// wrap-around ranges (e.g. listopad-leden) place months >= start in
|
||||
// defaultYear-1.
|
||||
func ParseMonthReferences(text string, defaultYear int) []string
|
||||
```
|
||||
|
||||
Required `defaultYear` (no default value — Go convention).
|
||||
|
||||
### Implementation sketch
|
||||
|
||||
```go
|
||||
var czechMonths = map[string]int{
|
||||
"leden": 1, "ledna": 1, "lednu": 1,
|
||||
"unor": 2, "unora": 2, "unoru": 2,
|
||||
"brezen": 3, "brezna": 3, "breznu": 3,
|
||||
"duben": 4, "dubna": 4, "dubnu": 4,
|
||||
"kveten": 5, "kvetna": 5, "kvetnu": 5,
|
||||
"cerven": 6, "cervna": 6, "cervnu": 6,
|
||||
"cervenec": 7, "cervnce": 7, "cervenci": 7,
|
||||
"srpen": 8, "srpna": 8, "srpnu": 8,
|
||||
"zari": 9,
|
||||
"rijen": 10, "rijna": 10, "rijnu": 10,
|
||||
"listopad": 11, "listopadu": 11,
|
||||
"prosinec": 12, "prosince": 12, "prosinci": 12,
|
||||
}
|
||||
|
||||
// Sorted by descending length at init, so longer alternatives win in
|
||||
// the regex (e.g. "cervenec" beats "cerven"). Mirrors Python's
|
||||
// sorted(..., key=len, reverse=True).
|
||||
var monthNameAlt = buildMonthNameAlt()
|
||||
|
||||
var (
|
||||
numericRe = regexp.MustCompile(`([\d+]+)\s*/\s*(\d{2,4})`)
|
||||
dotRe = regexp.MustCompile(`(\d{1,2})\s*\.\s*(\d{4})`)
|
||||
rangeRe = regexp.MustCompile(`(` + monthNameAlt + `)\s*-\s*(` + monthNameAlt + `)`)
|
||||
standRe = regexp.MustCompile(`\b(` + monthNameAlt + `)\b`)
|
||||
)
|
||||
```
|
||||
|
||||
Three Go-specific gotchas worth a code comment:
|
||||
|
||||
1. **RE2 alternation is leftmost-first**, same as Python `re`. Sorting
|
||||
month names by descending length is therefore necessary (otherwise
|
||||
`"cervenec"` matches as `"cerven"` + leftover `"ec"`). Mirror the
|
||||
Python sort exactly.
|
||||
2. **Map iteration is randomized in Go.** Build the alternation list
|
||||
from a sorted slice of keys, not by iterating the map.
|
||||
3. **`\d` and `\b`** in Go RE2 are ASCII-only, which matches the
|
||||
effective behavior on `Normalize`'d input (NFKD already collapsed
|
||||
any Unicode digits/letters that would matter; standalone Devanagari
|
||||
digits in member messages aren't a real-world concern).
|
||||
|
||||
The walk loop uses a bounded counter (max 12 iterations) defensively in
|
||||
Go; Python's `while True` is fine because every range terminates within
|
||||
12 hops, but a future reader appreciates the bound.
|
||||
|
||||
### Test table (verified against live Python — `default_year=2026`)
|
||||
|
||||
Locked outputs from `PYTHONPATH=scripts:. python -c 'from czech_utils
|
||||
import parse_month_references; print(parse_month_references(<input>, 2026))'`
|
||||
on 2026-05-05.
|
||||
|
||||
| # | Input | Expected | Path exercised |
|
||||
|---|---|---|---|
|
||||
| 1 | `""` | `[]` | empty |
|
||||
| 2 | `"11+12/2025"` | `["2025-11", "2025-12"]` | numeric, plus-split |
|
||||
| 3 | `"1/2026"` | `["2026-01"]` | numeric, single |
|
||||
| 4 | `"01/26"` | `["2026-01"]` | 2-digit year normalization |
|
||||
| 5 | `"11+12/25"` | `["2025-11", "2025-12"]` | plus-split + 2-digit year |
|
||||
| 6 | `"12+1+2/2026"` | `["2026-01", "2026-02", "2026-12"]` | sorting |
|
||||
| 7 | `"12.2025"` | `["2025-12"]` | dot pattern |
|
||||
| 8 | `"1.26"` | `[]` | dot pattern requires 4-digit year |
|
||||
| 9 | `"leden"` | `["2026-01"]` | standalone, m<10 |
|
||||
| 10 | `"prosinec"` | `["2025-12"]` | standalone, m≥10 → previous year |
|
||||
| 11 | `"prosince"` | `["2025-12"]` | declension |
|
||||
| 12 | `"lednu"` | `["2026-01"]` | declension |
|
||||
| 13 | `"rijen"` | `["2025-10"]` | m≥10 boundary (10 itself) |
|
||||
| 14 | `"zari"` | `["2026-09"]` | m<10 just below boundary |
|
||||
| 15 | `"listopad-leden"` | `["2025-11", "2025-12", "2026-01"]` | wrap range Nov→Jan |
|
||||
| 16 | `"rijen-leden"` | `["2025-10", "2025-11", "2025-12", "2026-01"]` | wrap from October |
|
||||
| 17 | `"unor-kveten"` | `["2026-02", "2026-03", "2026-04", "2026-05"]` | non-wrap range |
|
||||
| 18 | `"leden-leden"` | `["2026-01"]` | degenerate range |
|
||||
| 19 | `"unor-listopad"` | `["2026-02", ..., "2026-11"]` (10 entries) | range spans m≥10 — heuristic does NOT fire (range exclusion) |
|
||||
| 20 | `"cervenec-srpen"` | `["2026-07", "2026-08"]` | longest-match alt (`cervenec` not `cerven`+`ec`) |
|
||||
| 21 | `"listopad-leden, prosinec"` | `["2025-11", "2025-12", "2026-01"]` | range + standalone, dedup |
|
||||
| 22 | `"prosinec leden"` | `["2025-12", "2026-01"]` | two standalones, no range |
|
||||
| 23 | `"11+12/2025, leden-brezen"` | `["2025-11", "2025-12", "2026-01", "2026-02", "2026-03"]` | numeric + range mix |
|
||||
| 24 | `"11+12/25 a listopad"` | `["2025-11", "2025-12"]` | dedup across passes |
|
||||
| 25 | `"prosince/2025"` | `["2025-12"]` | numeric pattern fails (no digits before `/`); standalone fires |
|
||||
| 26 | `"listopad-prosinec/2025"` | `["2026-11", "2026-12"]` | range wins; numeric pattern fails |
|
||||
| 27 | `"01.2026 / 02.2026"` | `["2026-01", "2026-02"]` | dot pattern only; numeric matches `(2026, 02)` but month 2026 is out of range |
|
||||
| 28 | `"/12/2025"` | `["2025-12"]` | numeric matches at second `/` |
|
||||
| 29 | `"PROSINEC"` | `["2025-12"]` | normalize lowercases |
|
||||
| 30 | `"Žluťoučký prosinec"` | `["2025-12"]` | normalize strips diacritics |
|
||||
| 31 | `"Únor - květen"` | `["2026-02", ..., "2026-05"]` | range tolerates spaces around `-`, diacritics survive normalize |
|
||||
| 32 | `"platba 11/2025 a leden"` | `["2025-11", "2026-01"]` | mixed natural-language |
|
||||
| 33 | `"December"` | `[]` | English month names not recognized |
|
||||
| 34 | `"11+12/2025 11+12/2025"` | `["2025-11", "2025-12"]` | dedup of repeated input |
|
||||
| 35 | `"leden 2026"` | `["2026-01"]` | trailing year is ignored unless dot/slash separator present |
|
||||
|
||||
35 cases is enough to lock semantics; the M3.x corpus will pile on
|
||||
real-message fixtures later.
|
||||
|
||||
### Wire-up
|
||||
|
||||
- No `go.mod` changes (stdlib only).
|
||||
- No CLI changes.
|
||||
- `Normalize` is in the same package, so call it directly.
|
||||
|
||||
## Critical files
|
||||
|
||||
- New: [go/internal/domain/czech/parse_month_references.go](../../go/internal/domain/czech/parse_month_references.go)
|
||||
- New: [go/internal/domain/czech/parse_month_references_test.go](../../go/internal/domain/czech/parse_month_references_test.go)
|
||||
- Reference (read-only): [scripts/czech_utils.py](../../scripts/czech_utils.py) — the porting source
|
||||
- Reference (read-only): [docs/plans/2026-05-03-2349-go-backend-rewrite.md](2026-05-03-2349-go-backend-rewrite.md) — risk #4
|
||||
- Reuses: [go/internal/domain/czech/normalize.go](../../go/internal/domain/czech/normalize.go) — `Normalize` is called once at the top of `ParseMonthReferences`
|
||||
|
||||
## Verification
|
||||
|
||||
End-to-end checks before marking M2.2 done:
|
||||
|
||||
1. `cd go && go build ./...` — clean compile.
|
||||
2. `cd go && go test ./internal/domain/czech/...` — all 35 table cases green.
|
||||
3. `cd go && go test -race ./...` — race-clean (regex compiles are global; verify no init races).
|
||||
4. `cd go && golangci-lint run` (or `make go-lint` from repo root) — clean, gofumpt-formatted.
|
||||
5. **Spot parity** (manual, will be automated in M3.x): each test input has its expected output captured from the live Python implementation on 2026-05-05; the test table itself is the parity record. If any case diverges during implementation, re-run Python with the exact input to confirm the truth and update either the Go code or the test entry.
|
||||
6. `make go-build && make go-test && make go-lint` from repo root — proves M1/M2.1 gate still passes.
|
||||
|
||||
## Branching & follow-up
|
||||
|
||||
Per [CLAUDE.md](../../CLAUDE.md), this is feature work → branch + Gitea MR via `tea`:
|
||||
|
||||
- Branch: `feat/m2-2-parse-month-references` off `main`.
|
||||
- Single focused commit, Co-Authored-By trailer.
|
||||
- Push with `-u`.
|
||||
- Open MR with `tea pr create --title "feat(go/M2.2): port czech.ParseMonthReferences" --description ... --base main --head feat/m2-2-parse-month-references`. Print the MR URL for the user.
|
||||
- User merges/deletes the branch in Gitea — never from the CLI.
|
||||
|
||||
After merge (small doc edits land straight on `main` per CLAUDE.md exception):
|
||||
|
||||
- Tick `M2.2` in the [progress tracker](2026-05-03-2349-go-backend-rewrite-progress.md) with the merge SHA.
|
||||
- Add a one-line `CHANGELOG.md` entry (timestamp via `date "+%Y-%m-%d %H:%M %Z"`).
|
||||
- Record any porting surprise (e.g. an unexpected diff between Go RE2 and Python `re`) in the tracker's "Notes & decisions" section.
|
||||
|
||||
Next task is **M2.3 `domain/fees.CalculateFee`** — straightforward constants table; no parser semantics to debate.
|
||||
199
docs/plans/2026-05-06-0928-go-m2-5-money-parse-czk.md
Normal file
199
docs/plans/2026-05-06-0928-go-m2-5-money-parse-czk.md
Normal file
@@ -0,0 +1,199 @@
|
||||
# M2.5 — Port `parse_czk_amount` to `domain/money.ParseCZK`
|
||||
|
||||
> On execution, this plan should be moved to
|
||||
> `docs/plans/2026-05-06-0928-go-m2-5-money-parse-czk.md` per project CLAUDE.md
|
||||
> (`docs/plans/YYYY-MM-DD-HHMM-<slug>.md`). Plan mode forces it to live under
|
||||
> `~/.claude/plans/` until then.
|
||||
|
||||
## Context
|
||||
|
||||
Continuing the Go backend rewrite tracked in
|
||||
[2026-05-03-2349-go-backend-rewrite-progress.md](../../srv/personal/fuj-management/docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md).
|
||||
M2.1–M2.4 are landed. Next leaf-level pure function is
|
||||
`parse_czk_amount` from [scripts/infer_payments.py:17-45](../../srv/personal/fuj-management/scripts/infer_payments.py#L17-L45),
|
||||
the Czech-locale amount parser used at [scripts/infer_payments.py:124](../../srv/personal/fuj-management/scripts/infer_payments.py#L124)
|
||||
when reading the `Inferred Amount` column out of the payments sheet.
|
||||
|
||||
It's a small, isolated string→float helper, but its heuristic for
|
||||
disambiguating `.` and `,` as decimal vs thousand separator is
|
||||
non-obvious and needs to behave identically in Go to keep parity once
|
||||
the Go infer pipeline lands in M4.8.
|
||||
|
||||
## Python behaviour (the spec)
|
||||
|
||||
```py
|
||||
def parse_czk_amount(val) -> float:
|
||||
if val is None or val == "":
|
||||
return 0.0
|
||||
if isinstance(val, (int, float)):
|
||||
return float(val)
|
||||
|
||||
val = str(val)
|
||||
val = val.replace("Kč", "").replace("CZK", "").strip()
|
||||
if "," in val:
|
||||
# 1.500,00 -> 1500.00 — comma is decimal sep
|
||||
val = val.replace(".", "").replace(" ", "").replace(",", ".")
|
||||
else:
|
||||
if val.count(".") > 1:
|
||||
# 1.500.000 -> 1500000 — multiple dots = thousand sep
|
||||
val = val.replace(".", "").replace(" ", "")
|
||||
else:
|
||||
# "1 500.00" -> "1500.00", "1.500" stays "1.500" (= 1.5)
|
||||
val = val.replace(" ", "")
|
||||
try:
|
||||
return float(val)
|
||||
except ValueError:
|
||||
return 0.0
|
||||
```
|
||||
|
||||
Key behavioural notes for the Go port:
|
||||
|
||||
1. Empty / None → 0, no error.
|
||||
2. `"1.500"` (single dot, no comma) is parsed as **1.5**, not 1500.
|
||||
The heuristic intentionally treats a lone dot as decimal.
|
||||
3. `"1.500,00"` → 1500.0 (comma wins, dots are thousand seps).
|
||||
4. `"1.500.000"` → 1500000.0 (multiple dots → all thousand seps).
|
||||
5. `"1 500"` / `"1 500.00"` / `"500 Kč"` → spaces stripped.
|
||||
6. Garbage → 0, no error in Python.
|
||||
7. Strips literal substrings `"Kč"` and `"CZK"` (case-sensitive in Python).
|
||||
|
||||
## Approach
|
||||
|
||||
Create new package `internal/domain/money` mirroring the layout of
|
||||
`internal/domain/fees` (single-file module + test file alongside).
|
||||
|
||||
### Signature
|
||||
|
||||
```go
|
||||
// Package money ports Czech-locale currency parsing from
|
||||
// scripts/infer_payments.py.
|
||||
package money
|
||||
|
||||
// ParseCZK parses a Czech-locale amount string and returns the value
|
||||
// in CZK as a float64.
|
||||
//
|
||||
// Mirrors scripts/infer_payments.py parse_czk_amount:
|
||||
// - empty input → (0, nil)
|
||||
// - "Kč"/"CZK" suffixes are stripped (case-sensitive, like Python)
|
||||
// - if input contains ",", comma is the decimal separator and
|
||||
// dots/spaces are thousand separators ("1.500,00" → 1500.0)
|
||||
// - else if input contains 2+ dots, all dots are thousand seps
|
||||
// ("1.500.000" → 1500000.0)
|
||||
// - else single dot stays as the decimal point ("1.500" → 1.5,
|
||||
// matching the Python heuristic)
|
||||
// - on parse failure, returns (0, ErrInvalidAmount). Callers wanting
|
||||
// Python-equivalent silent-zero behaviour can discard the error.
|
||||
func ParseCZK(s string) (float64, error)
|
||||
```
|
||||
|
||||
`ErrInvalidAmount` is a package-level sentinel:
|
||||
|
||||
```go
|
||||
var ErrInvalidAmount = errors.New("money: invalid CZK amount")
|
||||
```
|
||||
|
||||
Why `(float64, error)` instead of mirroring Python's silent zero:
|
||||
|
||||
- Go idiom prefers explicit errors.
|
||||
- The single Python call site doesn't distinguish parse-fail from
|
||||
empty-input (both → 0), so if we want byte-equal behaviour at the
|
||||
Go infer site (M4.8), the caller can `v, _ := money.ParseCZK(s)`
|
||||
and get exactly the Python result.
|
||||
- Future callers (e.g. user-facing import flows) may want to surface
|
||||
the error.
|
||||
|
||||
This matches the precedent set in M2.4 where we used
|
||||
`Expected{Unknown bool}` rather than copying the Python `"?"` sentinel
|
||||
verbatim — Go-idiomatic surface, parity-preserving semantics.
|
||||
|
||||
### Polymorphic input?
|
||||
|
||||
Python's `parse_czk_amount` also accepts raw int/float (passed through
|
||||
unchanged) because Google Sheets API can return numeric cells as
|
||||
`float64` rather than strings. **Skip this in Go.** The Sheets IO
|
||||
adapter is M4.2, and that's where the `[]any` → string normalisation
|
||||
will live. Keeping `ParseCZK` string-only keeps the leaf function tiny.
|
||||
|
||||
### Tests
|
||||
|
||||
`money_test.go` mirrors the existing `fees_test.go` table-driven style,
|
||||
including the verification comment showing the Python command used to
|
||||
confirm each expected value:
|
||||
|
||||
```sh
|
||||
PYTHONPATH=scripts:. python -c '
|
||||
from infer_payments import parse_czk_amount
|
||||
for v in [None, "", "0", "500", "500 Kč", "500 CZK",
|
||||
"1 500", "1500.00", "1 500.00",
|
||||
"1.500,00", "1500,5", "1.500.000",
|
||||
"1.500", "abc", " ", "100,5 Kč"]:
|
||||
print(repr(v), "->", parse_czk_amount(v))
|
||||
'
|
||||
```
|
||||
|
||||
Cases to cover (all numeric outputs verified against the Python output
|
||||
of the snippet above):
|
||||
|
||||
| input | expected |
|
||||
|---|---|
|
||||
| `""` | 0 |
|
||||
| `"0"` | 0 |
|
||||
| `"500"` | 500 |
|
||||
| `"500 Kč"` | 500 |
|
||||
| `"500 CZK"` | 500 |
|
||||
| `"1 500"` | 1500 |
|
||||
| `"1500.00"` | 1500 |
|
||||
| `"1 500.00"` | 1500 |
|
||||
| `"1.500,00"` | 1500 |
|
||||
| `"1500,5"` | 1500.5 |
|
||||
| `"1.500.000"` | 1500000 |
|
||||
| `"1.500"` | 1.5 *(heuristic — single dot = decimal)* |
|
||||
| `"100,5 Kč"` | 100.5 |
|
||||
| `"abc"` | 0, returns `ErrInvalidAmount` |
|
||||
| `" "` | 0, returns `ErrInvalidAmount` *(or 0 nil — confirm against Python; trim leaves `""`, then `float("")` raises → Python returns 0; Go test will assert whichever Python actually produces)* |
|
||||
|
||||
The `" "` row is the only one that needs the Python verification step
|
||||
to settle — once verified, lock the behaviour in.
|
||||
|
||||
Also add a "documentation example" assertion in the test that
|
||||
`v, _ := ParseCZK(s)` recovers the Python silent-zero contract for
|
||||
every garbage input, so we don't lose that property at the Go infer
|
||||
call site.
|
||||
|
||||
## Files to create
|
||||
|
||||
- `go/internal/domain/money/money.go` — package + `ParseCZK` + `ErrInvalidAmount`
|
||||
- `go/internal/domain/money/money_test.go` — table-driven tests
|
||||
|
||||
No existing Go files need editing.
|
||||
|
||||
## Verification
|
||||
|
||||
```sh
|
||||
cd go && go test ./internal/domain/money/...
|
||||
make go-lint
|
||||
make go-build # sanity: nothing else broke
|
||||
```
|
||||
|
||||
Also run the Python snippet from the Tests section above and diff its
|
||||
output against the test table to confirm parity.
|
||||
|
||||
## Out of scope (explicit non-goals)
|
||||
|
||||
- Polymorphic `any` input — leave for M4.2 IO adapter.
|
||||
- Hooking into the Tier-1 parity runner — that comes with M3.5
|
||||
(`-tags=parity` build constraint). M2.5 just needs unit tests.
|
||||
- Any callsite migration — `infer_payments.py` keeps using its own
|
||||
Python function until M4.8.
|
||||
|
||||
## Progress tracker + changelog
|
||||
|
||||
After the commit lands:
|
||||
|
||||
- Tick `M2.5` in [docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md](../../srv/personal/fuj-management/docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md)
|
||||
with the commit SHA, mirroring the M2.4 entry style.
|
||||
- Add a CHANGELOG.md entry at top:
|
||||
`## YYYY-MM-DD HH:MM TZ — feat(go/M2.5): port domain/money.ParseCZK`.
|
||||
|
||||
Branch: `feat/m2-5-money-parse-czk` (per CLAUDE.md branch-per-feature
|
||||
workflow). Push, open MR via `tea pr create`, leave merge to the user.
|
||||
265
docs/plans/2026-05-06-1236-go-m2-6-synch-generate-sync-id.md
Normal file
265
docs/plans/2026-05-06-1236-go-m2-6-synch-generate-sync-id.md
Normal file
@@ -0,0 +1,265 @@
|
||||
|
||||
## Context
|
||||
|
||||
Continuing the Go backend rewrite tracked in
|
||||
[2026-05-03-2349-go-backend-rewrite-progress.md](../../srv/personal/fuj-management/docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md).
|
||||
M2.1–M2.5 are landed. Next leaf-level pure function is `generate_sync_id`
|
||||
from [scripts/sync_fio_to_sheets.py:62-77](../../srv/personal/fuj-management/scripts/sync_fio_to_sheets.py#L62-L77).
|
||||
|
||||
It computes a SHA-256 hash over a fixed seven-field projection of a Fio
|
||||
transaction (`date|amount|currency|sender|vs|message|bank_id`) and is
|
||||
the deduplication key written into column K (`Sync ID`) of the payments
|
||||
sheet. The Go port must produce a **byte-identical** digest for the same
|
||||
transaction; otherwise the Go-side sync (M4.7) would re-append rows
|
||||
already written by the Python sync, double-counting payments.
|
||||
|
||||
The non-trivial part is the `amount` field's string serialisation:
|
||||
upstream `fio_utils.py` always supplies `amount` as a Python `float`
|
||||
(API path: `float(val(1) or 0)`; HTML path: `parse_czech_amount(...)`
|
||||
which returns `float`). Python's `str(float)` produces `"500.0"` for
|
||||
whole-valued floats; Go's `strconv.FormatFloat(f, 'g', -1, 64)` produces
|
||||
`"500"`. This is the gotcha called out in the M2.6 line of the progress
|
||||
tracker.
|
||||
|
||||
## Python behaviour (the spec)
|
||||
|
||||
```py
|
||||
def generate_sync_id(tx: dict) -> str:
|
||||
components = [
|
||||
str(tx.get("date", "")),
|
||||
str(tx.get("amount", "")),
|
||||
str(tx.get("currency", "CZK")),
|
||||
str(tx.get("sender", "")),
|
||||
str(tx.get("vs", "")),
|
||||
str(tx.get("message", "")),
|
||||
str(tx.get("bank_id", "")),
|
||||
]
|
||||
raw_str = "|".join(components).lower()
|
||||
return hashlib.sha256(raw_str.encode("utf-8")).hexdigest()
|
||||
```
|
||||
|
||||
Behavioural notes for the Go port:
|
||||
|
||||
1. **Field order is load-bearing.** `date|amount|currency|sender|vs|message|bank_id` exactly.
|
||||
2. **Separator is `"|"`.**
|
||||
3. **Whole string is `.lower()`-ed before hashing** (so e.g. "ABC" sender vs "abc" hash identically). Unicode lower; in practice Fio data is ASCII + Czech diacritics.
|
||||
4. **`currency` defaults to `"CZK"`** when missing from the dict (HTML scraper path never sets it). Other fields default to `""`.
|
||||
5. **`amount` is a `float`.** Always. Real Fio data is `500.0`, `1234.56`, etc. — no NaN/Inf, but parity test must pin the format.
|
||||
6. **Output is `hashlib.sha256(...).hexdigest()`** — 64-char lowercase hex.
|
||||
7. **Encoding is UTF-8.**
|
||||
|
||||
### `str(float)` cases observed in real Fio amounts
|
||||
|
||||
| float64 | Python `str(f)` | Go `strconv.FormatFloat(f,'g',-1,64)` | Need |
|
||||
|---|---|---|---|
|
||||
| `500.0` | `"500.0"` | `"500"` | append `.0` |
|
||||
| `1234.56` | `"1234.56"` | `"1234.56"` | matches |
|
||||
| `0.0` | `"0.0"` | `"0"` | append `.0` |
|
||||
| `-500.0` | `"-500.0"` | `"-500"` | append `.0` |
|
||||
| `0.1` | `"0.1"` | `"0.1"` | matches |
|
||||
| `99999.99` | `"99999.99"` | `"99999.99"` | matches |
|
||||
|
||||
For the Fio amount domain (signed CZK, ≤ ~7 digits, ≤2 decimal places),
|
||||
the rule "`'g'` with prec -1, then append `.0` if result has no `.` and
|
||||
no `e`/`E`" is exact. We do not need to handle Python's
|
||||
scientific-notation crossover (`>= 1e16`) for real data, but the
|
||||
implementation should still cope with it correctly via the same rule.
|
||||
|
||||
## Approach
|
||||
|
||||
Create new package `internal/domain/synch` mirroring the layout of
|
||||
`internal/domain/money` (single-file module + test file alongside).
|
||||
|
||||
### Package + signature
|
||||
|
||||
```go
|
||||
// Package synch ports the bank-sync deduplication helper from
|
||||
// scripts/sync_fio_to_sheets.py.
|
||||
package synch
|
||||
|
||||
// Transaction is the projection of a Fio transaction that participates
|
||||
// in the Sync ID hash. Other fields (ks, ss, sender_account, …) are
|
||||
// intentionally excluded — they are not part of the Python hash.
|
||||
//
|
||||
// Currency: leave "" to inherit the Python default of "CZK" (matches
|
||||
// the HTML scraper path which omits the key entirely).
|
||||
type Transaction struct {
|
||||
Date string
|
||||
Amount float64
|
||||
Currency string
|
||||
Sender string
|
||||
VS string
|
||||
Message string
|
||||
BankID string
|
||||
}
|
||||
|
||||
// GenerateSyncID returns the lowercase SHA-256 hex digest of
|
||||
// "date|amount|currency|sender|vs|message|bank_id" (lower-cased), used
|
||||
// as the dedup key in column K of the payments sheet.
|
||||
//
|
||||
// Byte-stable with scripts/sync_fio_to_sheets.py generate_sync_id.
|
||||
func GenerateSyncID(tx Transaction) string
|
||||
```
|
||||
|
||||
### `Currency` default
|
||||
|
||||
In Go every struct field is always present, so we lose Python's
|
||||
"missing key vs empty string" distinction. Real-world data either sets
|
||||
`currency = "CZK"` (API path) or omits the key (HTML path → `"CZK"`
|
||||
default). Empty string never occurs in practice. The Go port collapses
|
||||
the two by treating `Currency == ""` as "use `CZK`":
|
||||
|
||||
```go
|
||||
currency := tx.Currency
|
||||
if currency == "" {
|
||||
currency = "CZK"
|
||||
}
|
||||
```
|
||||
|
||||
This is byte-equal to Python for every input we will ever see in
|
||||
production, and avoids forcing callers to pass a `*string`.
|
||||
|
||||
### Float formatter
|
||||
|
||||
Internal helper, unexported:
|
||||
|
||||
```go
|
||||
// formatAmount mimics Python's str(float) for the float values that
|
||||
// appear in Fio transactions. For mundane decimal amounts the rule
|
||||
// is: format with 'g' precision -1, then append ".0" if the result
|
||||
// has no decimal point and no exponent.
|
||||
func formatAmount(f float64) string {
|
||||
s := strconv.FormatFloat(f, 'g', -1, 64)
|
||||
if !strings.ContainsAny(s, ".eE") {
|
||||
s += ".0"
|
||||
}
|
||||
return s
|
||||
}
|
||||
```
|
||||
|
||||
Tested explicitly (see Tests below) so the edge cases (`0`, whole
|
||||
numbers, negatives, large/small with exponent) stay locked.
|
||||
|
||||
### Hash composition
|
||||
|
||||
```go
|
||||
func GenerateSyncID(tx Transaction) string {
|
||||
currency := tx.Currency
|
||||
if currency == "" {
|
||||
currency = "CZK"
|
||||
}
|
||||
raw := strings.ToLower(strings.Join([]string{
|
||||
tx.Date,
|
||||
formatAmount(tx.Amount),
|
||||
currency,
|
||||
tx.Sender,
|
||||
tx.VS,
|
||||
tx.Message,
|
||||
tx.BankID,
|
||||
}, "|"))
|
||||
sum := sha256.Sum256([]byte(raw))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
```
|
||||
|
||||
(`crypto/sha256` + `encoding/hex` — both stdlib, no `go.mod` change.)
|
||||
|
||||
## Tests
|
||||
|
||||
`synch_test.go` mirrors `money_test.go`'s table-driven style with the
|
||||
verification snippet at the top of the function. Two test functions:
|
||||
|
||||
### 1. `TestGenerateSyncID`
|
||||
|
||||
Each row's expected digest is computed from the Python source:
|
||||
|
||||
```sh
|
||||
PYTHONPATH=scripts:. python -c '
|
||||
from sync_fio_to_sheets import generate_sync_id
|
||||
cases = [
|
||||
{"date":"2026-01-15","amount":500.0,"currency":"CZK","sender":"Jan Novak","vs":"123","message":"clenske 1/2026","bank_id":"abc123"},
|
||||
{"date":"2026-01-15","amount":500.0,"sender":"Jan Novak","vs":"123","message":"clenske 1/2026","bank_id":"abc123"}, # currency missing → CZK
|
||||
{"date":"2026-02-10","amount":1234.56,"currency":"CZK","sender":"ABC SRO","vs":"","message":"FAKTURA 42","bank_id":"xyz"}, # mixed case → lowercased
|
||||
{"date":"2026-03-01","amount":-500.0,"currency":"CZK","sender":"refund","vs":"","message":"","bank_id":""}, # negative
|
||||
{"date":"2026-04-01","amount":0.0,"currency":"CZK","sender":"","vs":"","message":"","bank_id":""}, # zero amount
|
||||
{}, # empty dict — every field falls back to default
|
||||
]
|
||||
for c in cases:
|
||||
print(repr(c), "->", generate_sync_id(c))
|
||||
'
|
||||
```
|
||||
|
||||
Cases (one row per dict above), each asserting the exact 64-char hex
|
||||
digest the snippet prints. Cover:
|
||||
|
||||
- Happy path with all fields set.
|
||||
- `Currency: ""` → `"CZK"` default (parity with missing key).
|
||||
- Mixed-case sender/message → lowercased before hashing.
|
||||
- Negative amount.
|
||||
- Zero amount.
|
||||
- Zero-value `Transaction{}` — every field at Go zero, currency defaults
|
||||
to `"CZK"`, hash matches Python `generate_sync_id({})`.
|
||||
|
||||
### 2. `TestFormatAmount`
|
||||
|
||||
Pin the float formatter against Python's `str(float)`:
|
||||
|
||||
```sh
|
||||
PYTHONPATH=scripts:. python -c '
|
||||
for v in [0.0, 500.0, -500.0, 0.1, 1234.56, 99999.99, 1500000.0, 1e16, 1e-5]:
|
||||
print(repr(v), "->", repr(str(v)))
|
||||
'
|
||||
```
|
||||
|
||||
Table of `(float64, expected string)` pairs. Whole numbers must end in
|
||||
`.0`; existing decimal representations pass through unchanged;
|
||||
exponent-form floats (`1e16`, `1e-5`) keep their format.
|
||||
|
||||
## Files to create
|
||||
|
||||
- `go/internal/domain/synch/synch.go` — package, `Transaction`,
|
||||
`GenerateSyncID`, internal `formatAmount`.
|
||||
- `go/internal/domain/synch/synch_test.go` — `TestGenerateSyncID` +
|
||||
`TestFormatAmount`.
|
||||
|
||||
No existing Go files need editing.
|
||||
|
||||
## Verification
|
||||
|
||||
```sh
|
||||
cd go && go test ./internal/domain/synch/...
|
||||
make go-lint
|
||||
make go-build # sanity: nothing else broke
|
||||
```
|
||||
|
||||
Plus run the two Python snippets in the Tests section and diff their
|
||||
output against the test tables to confirm parity.
|
||||
|
||||
## Out of scope (explicit non-goals)
|
||||
|
||||
- **Hooking into the Tier-1 parity runner.** That comes with M3.5
|
||||
(`-tags=parity` build constraint and `tests/fixtures/pure/`). M2.6
|
||||
ships with hand-written, Python-verified test tables — same approach
|
||||
used by M2.1–M2.5.
|
||||
- **A richer `Transaction` struct** covering ks/ss/note/sender_account.
|
||||
Those fields aren't part of the hash. M4.4 (Fio IO adapter) will
|
||||
decide whether to reuse `synch.Transaction` or define its own struct
|
||||
and convert at the boundary.
|
||||
- **Polymorphic input** (e.g. accepting a `map[string]any`). Python's
|
||||
duck-typing is a non-goal in Go.
|
||||
- **Any Python callsite migration.** `sync_fio_to_sheets.py` keeps using
|
||||
its own `generate_sync_id` until M4.7 ports the sync service.
|
||||
|
||||
## Progress tracker + changelog
|
||||
|
||||
After the commit lands:
|
||||
|
||||
- Tick `M2.6` in
|
||||
[docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md](../../srv/personal/fuj-management/docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md)
|
||||
with the commit SHA, mirroring the M2.5 entry style.
|
||||
- Add a `CHANGELOG.md` entry at top:
|
||||
`## YYYY-MM-DD HH:MM TZ — feat(go/M2.6): port domain/synch.GenerateSyncID`.
|
||||
|
||||
Branch: `feat/m2-6-synch-generate-sync-id` (per CLAUDE.md
|
||||
branch-per-feature workflow). Push, open MR via `tea pr create`, leave
|
||||
merge to the user.
|
||||
126
docs/plans/2026-05-06-1305-go-m2-7-2-9-matching.md
Normal file
126
docs/plans/2026-05-06-1305-go-m2-7-2-9-matching.md
Normal file
@@ -0,0 +1,126 @@
|
||||
# M2.7 + M2.8 + M2.9 — Port `matching` package to Go
|
||||
|
||||
> On approval: copy this plan to `docs/plans/2026-05-06-1305-go-m2-7-2-9-matching.md` per [CLAUDE.md](../../srv/personal/fuj-management/CLAUDE.md) plan-location convention.
|
||||
|
||||
## Context
|
||||
|
||||
The Go rewrite (tracked in [docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md](../../srv/personal/fuj-management/docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md)) is in milestone M2 — porting pure-domain helpers leaf-first from Python to Go. M2.1 through M2.6 are complete (`czech.Normalize`, `czech.ParseMonthReferences`, `fees.CalculateFee`, `fees.CalculateJuniorFee`, `money.ParseCZK`, `synch.GenerateSyncID`).
|
||||
|
||||
M2.7, M2.8, and M2.9 cover three helpers from [scripts/match_payments.py](../../srv/personal/fuj-management/scripts/match_payments.py) that form a tight chain: `InferTransactionDetails` calls `MatchMembers` which calls `BuildNameVariants` and the same Sheets-serial date logic that `FormatDate` uses. The user requested they be done together because the dependency graph makes per-milestone commits awkward — `MatchMembers` would either reference an unexported helper not yet committed or commit dead code.
|
||||
|
||||
This unblocks M2.10 (`reconcile`, the load-bearing function) and M5 parity tests, since reconciliation consumes `InferTransactionDetails` output.
|
||||
|
||||
## Approach
|
||||
|
||||
**One commit, one branch, one MR.** Branch: `feat/m2-7-2-9-matching-package`. The three milestone checkboxes get ticked together on merge.
|
||||
|
||||
### Package layout
|
||||
|
||||
New package `go/internal/domain/matching/` mirroring the existing `go/internal/domain/{czech,fees,money,synch}` convention (one file per public symbol, tests alongside as `*_test.go`):
|
||||
|
||||
| File | Contents |
|
||||
|---|---|
|
||||
| `doc.go` | `// Package matching ports name/member matching from scripts/match_payments.py.` |
|
||||
| `name_variants.go` | `BuildNameVariants` + unexported `wordIn` helper (mirrors Python's `_word_in` co-location at [match_payments.py:60-62](../../srv/personal/fuj-management/scripts/match_payments.py#L60)) |
|
||||
| `match_members.go` | `Confidence` typed string + constants, `Match` struct, `MatchMembers` |
|
||||
| `infer.go` | `Transaction`, `InferredDetails`, `InferTransactionDetails` |
|
||||
| `format_date.go` | `FormatDate` |
|
||||
| `name_variants_test.go`, `match_members_test.go`, `infer_test.go`, `format_date_test.go` | table-driven tests, each with a top-of-file comment quoting the live Python one-liner used to verify expected values (mirrors [synch_test.go:7-20](../../srv/personal/fuj-management/go/internal/domain/synch/synch_test.go#L7)) |
|
||||
|
||||
### Public API
|
||||
|
||||
```go
|
||||
type Confidence string
|
||||
const (
|
||||
ConfidenceAuto Confidence = "auto"
|
||||
ConfidenceReview Confidence = "review"
|
||||
)
|
||||
type Match struct {
|
||||
Name string
|
||||
Confidence Confidence
|
||||
}
|
||||
|
||||
func BuildNameVariants(name string) []string
|
||||
func MatchMembers(text string, memberNames []string) []Match
|
||||
|
||||
type Transaction struct {
|
||||
Sender string
|
||||
Message string
|
||||
UserID string
|
||||
Date any // string | int | float64 — see "Parity concerns"
|
||||
}
|
||||
type InferredDetails struct {
|
||||
Members []Match
|
||||
Months []string
|
||||
SearchText string // matches Python's "search_text" key, not the misleading "matched_text" docstring
|
||||
}
|
||||
func InferTransactionDetails(tx Transaction, memberNames []string, defaultYear int) InferredDetails
|
||||
|
||||
func FormatDate(val any) string
|
||||
```
|
||||
|
||||
### Algorithms (port verbatim — these are the load-bearing details)
|
||||
|
||||
**`BuildNameVariants`** ([match_payments.py:33-57](../../srv/personal/fuj-management/scripts/match_payments.py#L33)): extract `(nickname)` regex, strip parens for `base`, normalize via `czech.Normalize`, append last + first when ≥2 parts, **filter <3 chars**. `variants[0]` must always be the full normalized base — `MatchMembers` relies on this.
|
||||
|
||||
**`MatchMembers`** ([match_payments.py:65-137](../../srv/personal/fuj-management/scripts/match_payments.py#L65)):
|
||||
1. **Exact short-circuit** ([:77-84](../../srv/personal/fuj-management/scripts/match_payments.py#L77)): if any member's `variants[0]` whole-word matches in `Normalize(text)`, return ONLY those `(name, auto)`. Prevents nickname `tov` matching inside `ottova`.
|
||||
2. Otherwise per-member first-match-wins: full-name substring → `\b first \b` AND `\b last \b` (any order) → `\b nickname \b` — each yields `auto` and continues.
|
||||
3. **Review tier** ([:113-129](../../srv/personal/fuj-management/scripts/match_payments.py#L113)): ≥2-part names → last name `len ≥ 4` AND not in `{"novak","novakova","prach"}` → review; else first name `len ≥ 3` → review. 1-part names → `len ≥ 4` → review.
|
||||
4. **Final filter** ([:131-137](../../srv/personal/fuj-management/scripts/match_payments.py#L131)): if ANY auto exists, drop ALL review. Two-pass — don't try to fuse with the loop.
|
||||
|
||||
**`InferTransactionDetails`** ([match_payments.py:144-184](../../srv/personal/fuj-management/scripts/match_payments.py#L144)): `search_text = sender + " " + message + " " + user_id`; month parse uses `message + " " + user_id` (excludes sender); fallback 1 retries members on sender alone; fallback 2 derives months from `tx.Date` (Sheets serial or `YYYY-MM-DD`).
|
||||
|
||||
**`FormatDate`** ([match_payments.py:187-206](../../srv/personal/fuj-management/scripts/match_payments.py#L187)): nil/empty → `""`; int/float → Sheets serial since 1899-12-30 formatted `YYYY-MM-DD`; pre-formatted `YYYY-MM-DD` (length 10, dashes at idx 4/7) → as-is; else `strings.TrimSpace(fmt.Sprint(v))`. **No raise on bad input** — parity contract.
|
||||
|
||||
## Parity concerns
|
||||
|
||||
- **RE2 `\b`**: Equivalent to Python `\b` on ASCII-folded input (`Normalize` strips diacritics + lowercases). Use `regexp.QuoteMeta` for `re.escape`.
|
||||
- **Sheets epoch**: 1899-12-30 (NOT 1900-01-01). `time.Date(1899, 12, 30, 0, 0, 0, 0, time.UTC)`.
|
||||
- **Fractional serials**: Python `timedelta(days=44197.5)` adds 12 hours, then `.strftime("%Y-%m-%d")` discards time. To match exactly use `base.Add(time.Duration(val * 24 * float64(time.Hour)))` then `Format("2006-01-02")`. **Do NOT** use `base.AddDate(0, 0, int(val))` — that silently drops fractional days from real Sheets exports of timestamped cells.
|
||||
- **`Transaction.Date any`**: Python `tx["date"]` accepts int/float/string transparently. Sheets API returns serial dates as `float64` from JSON; FIO scraper returns `string`. `any` is the faithful port; type-switch inside `FormatDate` and the date fallback in `InferTransactionDetails`.
|
||||
- **`SearchText` vs `MatchedText`**: Python docstring says `matched_text`, code returns `"search_text"`. Port the code, not the docstring.
|
||||
- **Default year plumbing**: Go's `czech.ParseMonthReferences(text, defaultYear)` requires explicit year. Python defaults to 2026. Plumb `defaultYear` as the third arg to `InferTransactionDetails`.
|
||||
- **Empty slices not nil**: Python `match_members` returns `[]` when nothing matches; ensure Go returns `[]Match{}` not `nil` so consumers don't have to nil-check (matches `synch` package style).
|
||||
|
||||
## Tests
|
||||
|
||||
Port all 6 cases from [tests/test_match_members.py](../../srv/personal/fuj-management/tests/test_match_members.py) verbatim into `match_members_test.go` as one table-driven `TestMatchMembers`. Each row: `name`, `text`, `wantContains []string`, `wantExcludes []string`, `wantAllAuto bool`.
|
||||
|
||||
Add table cases for:
|
||||
- `BuildNameVariants` — docstring example `František Vrbík (Štrúdl)` → 4 variants; nickname filtered (len<3); single-part name; whitespace inside parens
|
||||
- `FormatDate` — `nil` → `""`, `""` → `""`, `int(44197)` → `"2020-12-31"`, `float64(44197.5)` → `"2020-12-31"`, `"2026-04-15"` → `"2026-04-15"`, `"garbage"` → `"garbage"`, `" 2026-04-15 "` → `"2026-04-15"`
|
||||
- `InferTransactionDetails` — members from search_text, members from sender fallback, months from date-string fallback, months from serial-date fallback, both-paths-fail returns empty slices
|
||||
|
||||
Verify expectations against live Python and quote the one-liner in a top-of-file comment, e.g.:
|
||||
|
||||
```
|
||||
PYTHONPATH=scripts:. python -c '
|
||||
from match_payments import format_date
|
||||
for v in [None, "", 44197, 44197.5, "2026-04-15", "garbage", " 2026-04-15 "]: print(repr(format_date(v)))
|
||||
'
|
||||
```
|
||||
|
||||
## Critical files
|
||||
|
||||
- **Read for parity** — [scripts/match_payments.py:33-206](../../srv/personal/fuj-management/scripts/match_payments.py#L33), [tests/test_match_members.py](../../srv/personal/fuj-management/tests/test_match_members.py)
|
||||
- **Reuse** — `czech.Normalize` ([go/internal/domain/czech/normalize.go](../../srv/personal/fuj-management/go/internal/domain/czech/normalize.go#L15)), `czech.ParseMonthReferences` ([parse_month_references.go:61](../../srv/personal/fuj-management/go/internal/domain/czech/parse_month_references.go#L61))
|
||||
- **Mirror conventions** — [go/internal/domain/synch/synch.go](../../srv/personal/fuj-management/go/internal/domain/synch/synch.go), [go/internal/domain/synch/synch_test.go](../../srv/personal/fuj-management/go/internal/domain/synch/synch_test.go)
|
||||
- **New** — `go/internal/domain/matching/{doc,name_variants,match_members,infer,format_date}.go` + `*_test.go`
|
||||
|
||||
## Out of scope (M2.10 / M4 territory — DO NOT touch)
|
||||
|
||||
- `canonical_member_key` ([match_payments.py:20](../../srv/personal/fuj-management/scripts/match_payments.py#L20))
|
||||
- `reconcile`, `fetch_sheet_data`, `fetch_exceptions` — M2.10 / M4
|
||||
- Sheets/Drive/FIO I/O glue
|
||||
- Fixture capture (`tests/fixtures/pure/`) — M3.3 separately
|
||||
|
||||
## Verification
|
||||
|
||||
1. `cd go && make go-build` — clean build.
|
||||
2. `cd go && make go-test ./internal/domain/matching/...` — all table tests green.
|
||||
3. `cd go && make go-lint` — clean (govet, staticcheck, errcheck, gofumpt, unused).
|
||||
4. Spot-check: pick 2–3 random non-trivial cases (e.g. `MatchMembers` with mixed auto/review, `FormatDate(44197.5)`) and run the live Python one-liner from each test's comment block to confirm bytes match.
|
||||
5. Append CHANGELOG entry per [CLAUDE.md](../../srv/personal/fuj-management/CLAUDE.md) (timestamp via `date "+%Y-%m-%d %H:%M %Z"`).
|
||||
6. Tick M2.7, M2.8, M2.9 in [docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md](../../srv/personal/fuj-management/docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md) with the merge SHA.
|
||||
7. Push branch, open MR via `tea pr create --title "feat(go): port matching helpers (M2.7-2.9)" --base main --head feat/m2-7-2-9-matching-package`, print URL, leave merge to user.
|
||||
129
docs/plans/2026-05-06-1626-infer-payments-junior-roster.md
Normal file
129
docs/plans/2026-05-06-1626-infer-payments-junior-roster.md
Normal file
@@ -0,0 +1,129 @@
|
||||
# Include junior members in payment inference roster
|
||||
|
||||
## Context
|
||||
|
||||
A bank payment from sender `JIŘÍ KUBÍK` with the message
|
||||
`Jáchym Kubík: 01/2026+03/2026+04/2026` is being inferred as
|
||||
`[?] Jáchym Hrušák (G)` instead of the obvious `Jáchym Kubík`, even though
|
||||
the message contains his exact full name.
|
||||
|
||||
**Root cause** (confirmed with the user): `Jáchym Kubík` is in the **junior**
|
||||
attendance sheet only — he does not appear on the main/adults sheet. But
|
||||
[scripts/infer_payments.py:101-102](scripts/infer_payments.py#L101-L102)
|
||||
builds `member_names` by calling `get_members_with_fees()`
|
||||
([scripts/attendance.py:170](scripts/attendance.py#L170)), which reads only
|
||||
`EXPORT_URL` (the adults sheet). Junior-only members are therefore invisible
|
||||
to the matcher.
|
||||
|
||||
With Kubík absent from `member_names`, the matcher in
|
||||
[scripts/match_payments.py:65](scripts/match_payments.py#L65) processes the
|
||||
combined text `jiri kubik jachym kubik: 01/2026+03/2026+04/2026` against an
|
||||
adults-only roster:
|
||||
|
||||
- The exact-full-name short-circuit (`match_payments.py:75-84`) finds nothing —
|
||||
no adult's full name is in the text.
|
||||
- Hrušák `(G)` is the only adult with first name `Jáchym`. He fails the
|
||||
auto-rules (his surname isn't in the text) but hits the partial-first-name
|
||||
review rule (`match_payments.py:123-125`) → returned as `("Jáchym Hrušák (G)",
|
||||
"review")`, rendered as `[?] Jáchym Hrušák (G)`.
|
||||
|
||||
The user's original framing — "exact match in message should win over
|
||||
everything" — is already implemented for any candidate that **is** in the
|
||||
roster (the May-04 short-circuit). The bug is upstream: the right candidate
|
||||
was never even considered.
|
||||
|
||||
**Goal:** make `infer_payments` consider junior members as candidates, so
|
||||
junior-only names like `Jáchym Kubík` get matched correctly.
|
||||
|
||||
## Approach
|
||||
|
||||
Single-file change in [scripts/infer_payments.py](scripts/infer_payments.py).
|
||||
|
||||
Replace the adults-only roster lookup with a union of the adult and junior
|
||||
rosters. `attendance.py` already exposes both:
|
||||
[`get_members_with_fees()`](scripts/attendance.py#L170) for adults (and tier-J
|
||||
juniors who train with adults) and
|
||||
[`get_junior_members_with_fees()`](scripts/attendance.py#L208) for everyone in
|
||||
the junior sheet.
|
||||
|
||||
### Edit at [scripts/infer_payments.py:15](scripts/infer_payments.py#L15)
|
||||
|
||||
```python
|
||||
from attendance import get_members_with_fees, get_junior_members_with_fees
|
||||
```
|
||||
|
||||
### Edit at [scripts/infer_payments.py:99-102](scripts/infer_payments.py#L99-L102)
|
||||
|
||||
```python
|
||||
print("Fetching member list for matching...")
|
||||
adult_members, _ = get_members_with_fees()
|
||||
junior_members, _ = get_junior_members_with_fees()
|
||||
|
||||
# Union rosters, preserving first-seen order, deduping by canonical key
|
||||
seen: set[str] = set()
|
||||
member_names: list[str] = []
|
||||
for m in adult_members + junior_members:
|
||||
name = m[0]
|
||||
key = canonical_member_key(name)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
member_names.append(name)
|
||||
```
|
||||
|
||||
`canonical_member_key` already lives in
|
||||
[scripts/match_payments.py:20](scripts/match_payments.py#L20) — import it
|
||||
alongside `infer_transaction_details`. It normalizes diacritics/case/whitespace,
|
||||
so `"Maria Maco"` and `"Mária Maco"` collapse to the same key.
|
||||
|
||||
### Why downstream reconciliation still works
|
||||
|
||||
`reconcile()` is invoked twice per page — once with the adults roster
|
||||
([app.py:200](app.py#L200)) and once with the juniors roster
|
||||
([app.py:384](app.py#L384)). Each call resolves the `Person` cell against its
|
||||
own roster; a junior name resolves cleanly in the juniors call and lands in
|
||||
"unmatched" in the adults call. That's already the existing behavior for any
|
||||
junior payment manually entered into the `Person` column, so no further
|
||||
changes are needed.
|
||||
|
||||
### Files to modify
|
||||
|
||||
- [scripts/infer_payments.py](scripts/infer_payments.py) — only the
|
||||
import + roster construction. ~10-line change.
|
||||
|
||||
### Files to read for confidence (no edits)
|
||||
|
||||
- [scripts/attendance.py:208-289](scripts/attendance.py#L208-L289) —
|
||||
`get_junior_members_with_fees` returns `(name, tier, …)` tuples just like
|
||||
the adults version, so `m[0]` works for both.
|
||||
- [scripts/match_payments.py:65-137](scripts/match_payments.py#L65-L137) —
|
||||
`match_members` already handles the precedence the user wants (exact full-name
|
||||
short-circuit), so once Kubík is in `member_names`, the case will be auto-matched
|
||||
with no `[?]`.
|
||||
|
||||
## Verification
|
||||
|
||||
1. **Manual sanity** — re-run inference on the offending row:
|
||||
- Clear `Person`/`Purpose` for the Kubík row in the payments sheet.
|
||||
- `make infer`.
|
||||
- Expect `Person = Jáchym Kubík`, `Purpose = 2026-01, 2026-03, 2026-04`,
|
||||
no `[?]`.
|
||||
|
||||
2. **Unit test** — extend
|
||||
[tests/test_match_members.py](tests/test_match_members.py) (or add a small
|
||||
`tests/test_infer_payments.py`) to assert that, given a roster that
|
||||
includes `Jáchym Hrušák (G)` and `Jáchym Kubík`, the message
|
||||
`Jáchym Kubík: 01/2026+03/2026+04/2026` resolves to
|
||||
`[("Jáchym Kubík", "auto")]` only. This is really a regression test for
|
||||
the May-04 short-circuit — the new behavior under test is just that
|
||||
`infer_payments` now feeds in juniors.
|
||||
|
||||
3. **Run the suite**: `make test`.
|
||||
|
||||
4. **Dashboard smoke** — `make web`, open `/payments`, confirm the row now
|
||||
shows the correct member; open `/juniors`, confirm the payment is
|
||||
credited to Kubík for the three months listed.
|
||||
|
||||
5. **Changelog** — once the user confirms the fix, append an entry to
|
||||
[CHANGELOG.md](CHANGELOG.md) per [CLAUDE.md](CLAUDE.md):
|
||||
`## YYYY-MM-DD HH:MM TZ — fix: include juniors in payment-inference roster`.
|
||||
215
docs/plans/2026-05-06-1738-go-m2-11-12-fees-reconcile-cli.md
Normal file
215
docs/plans/2026-05-06-1738-go-m2-11-12-fees-reconcile-cli.md
Normal file
@@ -0,0 +1,215 @@
|
||||
# M2.11 + M2.12 — `fuj fees` and `fuj reconcile` subcommands (stubbed IO)
|
||||
|
||||
> On approval: copy this plan to `docs/plans/2026-05-06-1738-go-m2-11-12-fees-reconcile-cli.md` per [CLAUDE.md](../../CLAUDE.md) plan-location convention.
|
||||
|
||||
## Context
|
||||
|
||||
The Go rewrite (tracked in [docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md](2026-05-03-2349-go-backend-rewrite-progress.md)) finished M2.1–M2.10 — every pure-domain helper (`czech`, `fees`, `money`, `synch`, `matching`, `reconcile`) is ported. M2.11 and M2.12 close out the M2 milestone by wiring two CLI subcommands to those helpers.
|
||||
|
||||
Both subcommands today are reported as "not implemented" by the dispatcher in [go/cmd/fuj/main.go:32-34](../../go/cmd/fuj/main.go#L32). After this change:
|
||||
|
||||
- `fuj fees` will compose `domain/fees` with a (stubbed) attendance loader and a fees-table formatter.
|
||||
- `fuj reconcile` will compose `domain/reconcile` with stubbed transaction + exception + attendance loaders and a balance-report formatter.
|
||||
- Both will exit with a clean, actionable error message until M4 wires real Google Sheets IO behind the loader interfaces.
|
||||
|
||||
The user asked to do M2.11 and M2.12 together because they share the same loader scaffolding and formatter package — splitting them would either commit half the package or duplicate work.
|
||||
|
||||
## Approach
|
||||
|
||||
**One commit, one branch, one MR.** Branch: `feat/m2-11-12-fees-reconcile-cli`. Both M2.11 and M2.12 checkboxes get ticked on merge.
|
||||
|
||||
The CLI subcommand is the user-facing layer. It owns nothing. All work lives in a new `services/membership` package: a) loader interfaces, b) the stub implementations that fail with a clear error, c) the orchestration functions, and d) the text formatters. M4 will later add real loader implementations behind the same interfaces — no other code needs to change.
|
||||
|
||||
### Package layout
|
||||
|
||||
| Path | Contents |
|
||||
|---|---|
|
||||
| `go/internal/services/membership/doc.go` | Package doc: orchestrates `domain/fees` + `domain/reconcile` against pluggable IO loaders. |
|
||||
| `go/internal/services/membership/loader.go` | `AttendanceLoader`, `TransactionLoader`, `ExceptionLoader` interfaces. Aggregate `Sources` interface. `ErrIOPending` sentinel. `NewStubSources()` factory returning a struct that satisfies all three with `ErrIOPending`. |
|
||||
| `go/internal/services/membership/fees.go` | `FeesReport(ctx, AttendanceLoader, io.Writer) error` — loads adults, formats, writes. |
|
||||
| `go/internal/services/membership/reconcile.go` | `ReconcileReport(ctx, Sources, defaultYear int, io.Writer) error` — loads adults + txns + exceptions, calls `reconcile.Reconcile`, formats, writes. |
|
||||
| `go/internal/services/membership/format_fees.go` | `printFeesTable(w, members, sortedMonths)` — fixed-width table mirroring [calculate_fees.py:9-49](../../scripts/calculate_fees.py#L9). |
|
||||
| `go/internal/services/membership/format_reconcile.go` | `printReconcileReport(w, result, sortedMonths)` — header / summary table / credits / debts / unmatched / matched-tx detail mirroring [match_payments.py:521-640](../../scripts/match_payments.py#L521). |
|
||||
| `go/internal/services/membership/*_test.go` | Table tests for both formatters using fixture data (no IO); separate test confirms `NewStubSources()` returns `ErrIOPending` from each method. |
|
||||
| `go/cmd/fuj/main.go` | Add `feesCmd(args)` and `reconcileCmd(args)`. Drop `"fees"` and `"reconcile"` from the not-implemented case (leave `"sync"`, `"infer"`). |
|
||||
|
||||
### Public API
|
||||
|
||||
```go
|
||||
// loader.go
|
||||
type AttendanceLoader interface {
|
||||
LoadAdults(ctx context.Context) (members []reconcile.Member, sortedMonths []string, err error)
|
||||
}
|
||||
type TransactionLoader interface {
|
||||
LoadTransactions(ctx context.Context) ([]reconcile.Transaction, error)
|
||||
}
|
||||
type ExceptionLoader interface {
|
||||
LoadExceptions(ctx context.Context) (map[reconcile.ExceptionKey]reconcile.Exception, error)
|
||||
}
|
||||
|
||||
// Sources is the aggregate that fuj reconcile needs.
|
||||
type Sources interface {
|
||||
AttendanceLoader
|
||||
TransactionLoader
|
||||
ExceptionLoader
|
||||
}
|
||||
|
||||
// ErrIOPending is returned by every stub loader method.
|
||||
var ErrIOPending = errors.New(
|
||||
"io layer not yet wired up; lands in milestone M4 (sheets/drive/fio)")
|
||||
|
||||
func NewStubSources() Sources
|
||||
|
||||
// fees.go
|
||||
func FeesReport(ctx context.Context, l AttendanceLoader, out io.Writer) error
|
||||
|
||||
// reconcile.go
|
||||
func ReconcileReport(ctx context.Context, s Sources, defaultYear int, out io.Writer) error
|
||||
```
|
||||
|
||||
### Loader return types
|
||||
|
||||
The interfaces return `domain/reconcile` types directly (`reconcile.Member`, `reconcile.Transaction`, `reconcile.ExceptionKey`, `reconcile.Exception`). These are already shaped for the reconcile algorithm and cover the fees report's needs (`Member.Fees[month].Expected` is precomputed by whatever loader populates it — the Python `get_members_with_fees()` does the same). No translation layer needed; no parallel struct hierarchy.
|
||||
|
||||
### Stub implementation pattern
|
||||
|
||||
```go
|
||||
type stubSources struct{}
|
||||
|
||||
func (stubSources) LoadAdults(context.Context) ([]reconcile.Member, []string, error) {
|
||||
return nil, nil, ErrIOPending
|
||||
}
|
||||
func (stubSources) LoadTransactions(context.Context) ([]reconcile.Transaction, error) {
|
||||
return nil, ErrIOPending
|
||||
}
|
||||
func (stubSources) LoadExceptions(context.Context) (
|
||||
map[reconcile.ExceptionKey]reconcile.Exception, error) {
|
||||
return nil, ErrIOPending
|
||||
}
|
||||
|
||||
func NewStubSources() Sources { return stubSources{} }
|
||||
```
|
||||
|
||||
### Subcommand wiring
|
||||
|
||||
```go
|
||||
// cmd/fuj/main.go
|
||||
case "fees":
|
||||
feesCmd(args)
|
||||
case "reconcile":
|
||||
reconcileCmd(args)
|
||||
case "sync", "infer": // keep these in the M4 placeholder
|
||||
fmt.Fprintf(os.Stderr, "fuj %s: not implemented yet (lands in M4)\n", cmd)
|
||||
os.Exit(2)
|
||||
|
||||
func feesCmd(args []string) {
|
||||
fs := flag.NewFlagSet("fees", flag.ExitOnError)
|
||||
fs.Usage = func() { fmt.Fprintln(os.Stderr, "usage: fuj fees") }
|
||||
if err := fs.Parse(args); err != nil { ... }
|
||||
|
||||
sources := membership.NewStubSources()
|
||||
ctx := context.Background()
|
||||
if err := membership.FeesReport(ctx, sources, os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "fuj fees: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func reconcileCmd(args []string) {
|
||||
fs := flag.NewFlagSet("reconcile", flag.ExitOnError)
|
||||
fs.Usage = func() { fmt.Fprintln(os.Stderr, "usage: fuj reconcile") }
|
||||
if err := fs.Parse(args); err != nil { ... }
|
||||
|
||||
sources := membership.NewStubSources()
|
||||
ctx := context.Background()
|
||||
if err := membership.ReconcileReport(ctx, sources, time.Now().Year(), os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "fuj reconcile: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Both subcommands accept zero positional args today (matching Python `calculate_fees.py` which has none, and skipping the Python `match_payments.py` flags `--sheet-id` / `--credentials` / `--bank` until M4 needs them — no point pre-empting flag design while there's nothing to plumb them into). Update `usage()` in `main.go` to remove the `[M2]` annotations from `fees` / `reconcile`.
|
||||
|
||||
### Formatter ports — what to mirror byte-for-byte
|
||||
|
||||
The formatters are pure post-processing. Ship them now so M4 only adds real loader plumbing.
|
||||
|
||||
**`printFeesTable`** ports [calculate_fees.py:9-49](../../scripts/calculate_fees.py#L9):
|
||||
- Filter to `tier == "A"` only.
|
||||
- Month label format: `"Jan 2026"` (`time.Parse("2006-01", m).Format("Jan 2006")`).
|
||||
- Column widths: `name_width = max(len(name))`, `col_width = 15`.
|
||||
- Cell format: `"%d CZK (%d)"` when `count > 0`, else `"-"`. Right-aligned in col_width.
|
||||
- Totals row: monthly sums, label `"TOTAL"`, cells `"%d CZK"`.
|
||||
- Print "No data." when members is empty.
|
||||
|
||||
**`printReconcileReport`** ports [match_payments.py:521-640](../../scripts/match_payments.py#L521):
|
||||
- Header banner (`"=" * 80`, "PAYMENT RECONCILIATION REPORT", banner).
|
||||
- Per-adult summary table — cell logic: `expected==0 && paid==0 → "-"`, `paid>=expected && expected>0 → "OK"`, `paid>0 → "{paid}/{expected}"`, else `"UNPAID {expected}"`. Balance column: `"+N"` / `"-N"` / `"0"`.
|
||||
- TOTAL footer line carrying the `Expected/Paid/Balance` summary.
|
||||
- Optional sections: `TOTAL CREDITS` (positive total balances), `TOTAL DEBTS` (negative — print `abs`), `UNMATCHED TRANSACTIONS`, `MATCHED TRANSACTION DETAILS`.
|
||||
- Use sorted member-name iteration (`sort.Strings`) — Python uses `sorted(adults.keys())`.
|
||||
- Float printing: amounts use `%.0f` (Python `:.0f`), `paid` is cast via `int(...)` before formatting in some places — preserve these (cast to `int` then `%d`).
|
||||
|
||||
Tests use a small handcrafted `reconcile.Result` with one paid member, one debtor, one credit, one unmatched tx and assert exact byte equality of the formatted output (golden string in the test source — not file-based).
|
||||
|
||||
### Tests
|
||||
|
||||
**`format_fees_test.go`**: handcrafted `[]reconcile.Member` covering: tier-filter (J/X excluded), zero-attendance cell, single-attendance cell, multi-attendance cell, empty result → `"No data."`. Golden output strings inline.
|
||||
|
||||
**`format_reconcile_test.go`**: handcrafted `reconcile.Result` exercising every branch in the cell logic + each optional section. Golden strings inline. (Don't blindly copy the Python `print(...)` string-formatting bugs — the live Python f-string `f"\n{'TOTAL CREDITS (advance payments or surplus):'}"` is intentional whitespace; reproduce as plain `"\nTOTAL CREDITS (advance payments or surplus):"` and verify identical bytes by running the live Python on the same fixture.)
|
||||
|
||||
**`stub_test.go`**: assert each `NewStubSources()` method returns `ErrIOPending` (use `errors.Is`).
|
||||
|
||||
**`fees_test.go` / `reconcile_test.go`**: pass a fake loader that returns canned `[]reconcile.Member` / `[]reconcile.Transaction` / exceptions; assert `FeesReport` / `ReconcileReport` write the expected formatter output. This proves the orchestration glues correctly without involving stubs.
|
||||
|
||||
Verify formatter golden strings against live Python with one-liner comments at top of each test file, e.g.:
|
||||
|
||||
```
|
||||
PYTHONPATH=scripts:. python -c '
|
||||
from match_payments import print_report
|
||||
result = {"members": {...}, "unmatched": [...]}
|
||||
print_report(result, ["2026-04"])
|
||||
'
|
||||
```
|
||||
|
||||
## Parity concerns
|
||||
|
||||
- **`czech.Normalize` keeps "%" semantics** — exception keys in `reconcile.Reconcile` use `czech.Normalize(name)` and `czech.Normalize(period)`. The `ExceptionLoader` stub doesn't return any, so this isn't exercised in M2.11/M2.12 — but real loaders in M4 must also normalize at load time (matching Python `fetch_exceptions`).
|
||||
- **Sorted month iteration** — formatter must respect the `sortedMonths` argument order, not iterate maps directly (Go map iteration is randomized).
|
||||
- **Sorted member iteration** — adults sorted by name (`sort.Strings`); Python uses `sorted(adults.keys())` which is byte-order. Czech-diacritic names sort by codepoint either way.
|
||||
- **Empty unmatched list** — Python prints nothing; Go must skip the section header when `len(result.Unmatched) == 0`.
|
||||
- **`int(paid)` truncation** — Python `int(mdata["paid"])` truncates toward zero. Go `int(float64)` matches. Use `int(paid)` not `math.Round`.
|
||||
- **Stub error stable string** — `ErrIOPending.Error()` text is part of the user-facing CLI contract for the duration of M2.11..M3; tests assert `errors.Is`, not the string. Don't change the wording without bumping the changelog.
|
||||
- **Default year = `time.Now().Year()`** — `reconcile.Reconcile` needs `defaultYear` for the inference fallback. `time.Now().Year()` matches the Python implicit default for current-year operation. Tests use a fixed year (2026).
|
||||
|
||||
## Critical files
|
||||
|
||||
- **Read for parity** — [scripts/calculate_fees.py](../../scripts/calculate_fees.py) (full), [scripts/match_payments.py:521-640](../../scripts/match_payments.py#L521), [scripts/match_payments.py:647-684](../../scripts/match_payments.py#L647) (CLI entry shape).
|
||||
- **Reuse** — `domain/reconcile.{Member, Transaction, ExceptionKey, Exception, Result, Reconcile}` ([reconcile.go](../../go/internal/domain/reconcile/reconcile.go)), `domain/fees.{CalculateFee, AdultFeeMonthlyRate}` ([fees.go](../../go/internal/domain/fees/fees.go)).
|
||||
- **Mirror conventions** — package layout from [go/internal/domain/matching/](../../go/internal/domain/matching/) (one symbol per file, `*_test.go` siblings, top-of-test live-Python verification comments).
|
||||
- **New** — `go/internal/services/membership/{doc,loader,fees,reconcile,format_fees,format_reconcile}.go` + `*_test.go`.
|
||||
- **Modify** — [go/cmd/fuj/main.go](../../go/cmd/fuj/main.go) (add `feesCmd`/`reconcileCmd`, drop `"fees"`/`"reconcile"` from the M2 not-implemented case, drop `[M2]` annotations from `usage()`).
|
||||
|
||||
## Out of scope (explicitly DO NOT touch)
|
||||
|
||||
- Real Google Sheets / Drive / Fio loader implementations — M4.1–M4.6.
|
||||
- Web routes / handlers — M5.
|
||||
- `fuj sync` and `fuj infer` subcommands — M4.7/M4.8.
|
||||
- Junior fees report — current Python `make fees` only prints adults; preserve. (`get_junior_members_with_fees` is consumed by the web frontend, not the CLI.)
|
||||
- Bank-direct mode (`--bank` flag in [match_payments.py:659](../../scripts/match_payments.py#L659)) — M4 territory.
|
||||
- Fixture capture (`tests/fixtures/`) — M3 milestone.
|
||||
|
||||
## Verification
|
||||
|
||||
1. `cd go && go build ./...` — clean build.
|
||||
2. `cd go && go test -race ./internal/services/membership/...` — formatter golden strings match, stub returns `ErrIOPending`, orchestration glues fake loader → formatter correctly.
|
||||
3. `cd go && make go-lint` — clean (govet, staticcheck, errcheck, gofumpt, unused).
|
||||
4. **End-to-end CLI smoke**:
|
||||
- `make go-build && ./bin/fuj fees` → exits non-zero, stderr contains `"io layer not yet wired up; lands in milestone M4"`.
|
||||
- `./bin/fuj reconcile` → same shape.
|
||||
- `./bin/fuj help` → no longer says `[M2]` next to `fees`/`reconcile`; still says `[M4]` next to `sync`/`infer`.
|
||||
5. **Formatter parity spot-check** — pick one fixture (1 adult with 2 months of fees, 1 unmatched tx); run the Python equivalent with the same input; confirm Go output is byte-identical (modulo trailing-whitespace lines — `diff -w` if needed, but aim for clean `diff` first).
|
||||
6. Append CHANGELOG entry per [CLAUDE.md](../../CLAUDE.md) (timestamp via `date "+%Y-%m-%d %H:%M %Z"`).
|
||||
7. Tick M2.11 and M2.12 in [docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md](2026-05-03-2349-go-backend-rewrite-progress.md) with the merge SHA. Update the M2 milestone summary line if M2 is now fully closed.
|
||||
8. Push branch, open MR via `tea pr create --title "feat(go): wire fuj fees + fuj reconcile (M2.11-12)" --base main --head feat/m2-11-12-fees-reconcile-cli`, print URL, leave merge to user.
|
||||
261
docs/plans/2026-05-06-2111-go-m3-fixture-capture.md
Normal file
261
docs/plans/2026-05-06-2111-go-m3-fixture-capture.md
Normal file
@@ -0,0 +1,261 @@
|
||||
# M3 — Fixture capture + characterization framework
|
||||
|
||||
> On approval: copy this plan to `docs/plans/2026-05-06-2111-go-m3-fixture-capture.md` per [CLAUDE.md](../../srv/personal/fuj-management/CLAUDE.md) plan-location convention.
|
||||
|
||||
## Context
|
||||
|
||||
The Go rewrite (tracked in [docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md](../../srv/personal/fuj-management/docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md)) finished M2.1–M2.12 — every pure-domain helper is ported and the `fuj fees` / `fuj reconcile` CLIs are wired. M3 closes the loop: it builds the **parity safety net** that proves Go output matches Python output for every ported function. Without it, M2 is "trust me", and the rewrite has no defensible cutover criterion.
|
||||
|
||||
M3 has three deliverables:
|
||||
|
||||
1. **A capture pipeline** (`scripts/capture_fixtures.py` + `scripts/scrub_fixtures.py`) that produces deterministic, PII-free JSON fixtures from the live Python implementations.
|
||||
2. **A fixture corpus** at [go/tests/fixtures/](../../srv/personal/fuj-management/go/tests/fixtures/) covering the 10 pure functions of M2 (M2.1–M2.9) plus 10 reconcile cases spanning every code path of `reconcile()` (M2.10).
|
||||
3. **A parity test runner** in [go/tests/parity/](../../srv/personal/fuj-management/go/tests/parity/) under `//go:build parity` that replays each fixture and asserts byte/value equality against the Go port.
|
||||
|
||||
User-confirmed scope decisions:
|
||||
- **Single MR** for all six sub-tasks (M3.1–M3.6) — they're tightly coupled; no half-state is committable.
|
||||
- **Type envelope only where it matters** — four fields (`generate_sync_id.tx.amount`, `parse_czk_amount.val`, `format_date.val`, `infer_transaction_details.tx.date`) use `{"type":..., "value":...}` to disambiguate int/float/none. Everything else uses raw JSON.
|
||||
- **Real seeds for `parse_month_references` and `match_members` only** — read curated message strings from `tmp/payments_transactions_cache.json`, scrub, ship. Other functions stay on handcrafted seeds.
|
||||
- **Plan committed at `docs/plans/2026-05-06-2111-go-m3-fixture-capture.md`** — same convention as every M-series predecessor.
|
||||
|
||||
## Branch + landing
|
||||
|
||||
- Branch: `feat/m3-fixture-capture`. Single MR via `tea pr create`. Tick M3.1–M3.6 on merge with the SHA.
|
||||
- No edits to existing Python or Go production code. M3 is purely additive: new scripts, new fixtures, new test files, new Makefile targets, README, CHANGELOG entry, plan archive, progress tracker tick.
|
||||
|
||||
## File layout
|
||||
|
||||
**Python (capture pipeline):**
|
||||
- [scripts/capture_fixtures.py](../../srv/personal/fuj-management/scripts/capture_fixtures.py) — dispatcher CLI; one entry per function via `--func`.
|
||||
- [scripts/scrub_fixtures.py](../../srv/personal/fuj-management/scripts/scrub_fixtures.py) — stdin→stdout deterministic bijection scrubber.
|
||||
- [scripts/_fixture_seeds.py](../../srv/personal/fuj-management/scripts/_fixture_seeds.py) — internal: handcrafted seeds keyed by `(func, case_id)`, plus the curated real-message extractor.
|
||||
|
||||
**Fixture corpus** (committed, PII-free):
|
||||
- [go/tests/fixtures/README.md](../../srv/personal/fuj-management/go/tests/fixtures/README.md) — refresh workflow + scrubbing audit guide.
|
||||
- `go/tests/fixtures/pure/<func>/<case>.json` — one directory per function (10 functions: `normalize`, `parse_month_references`, `calculate_fee`, `calculate_junior_fee`, `parse_czk_amount`, `generate_sync_id`, `build_name_variants`, `match_members`, `infer_transaction_details`, `format_date`).
|
||||
- `go/tests/fixtures/reconcile/<NN>_<case>.json` — 10 numbered reconcile cases.
|
||||
|
||||
**Go parity tests** (all under `//go:build parity`):
|
||||
- [go/tests/parity/parityio.go](../../srv/personal/fuj-management/go/tests/parity/parityio.go) — shared loader with generic `Case[I,O]` walker, type envelopes mirrored from §3, float tolerance helper.
|
||||
- [go/tests/parity/pure/<func>/<func>_parity_test.go](../../srv/personal/fuj-management/go/tests/parity/pure/) — one file per function, ~30 lines each.
|
||||
- [go/tests/parity/reconcile/reconcile_parity_test.go](../../srv/personal/fuj-management/go/tests/parity/reconcile/) — bespoke comparator using `math.Abs(got-want) <= 0.01` for `paid` floats, exact equality on int balances.
|
||||
|
||||
**Modified:**
|
||||
- [Makefile](../../srv/personal/fuj-management/Makefile) — append `go-parity`, `go-test-all`, `capture-fixtures` targets.
|
||||
- [CHANGELOG.md](../../srv/personal/fuj-management/CHANGELOG.md) — single entry at top.
|
||||
- [docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md](../../srv/personal/fuj-management/docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md) — tick M3.1–M3.6 with SHA.
|
||||
|
||||
## Capture invocation interface
|
||||
|
||||
Two-stage pipeline (capture | scrub) so each stage is independently debuggable:
|
||||
|
||||
```bash
|
||||
python scripts/capture_fixtures.py --func <name> --case <id> --input-seed <id> \
|
||||
| python scripts/scrub_fixtures.py \
|
||||
> go/tests/fixtures/pure/<func>/<id>.json
|
||||
```
|
||||
|
||||
Capture flags:
|
||||
- `--func` — target function (`normalize`, `reconcile`, etc.).
|
||||
- `--case` — human-authored case ID, becomes the file stem. Never auto-generated (auto-IDs cause git churn).
|
||||
- `--input-seed <id>` — pull from `_fixture_seeds.py` registry (the default mode for handcrafted cases).
|
||||
- `--input-stdin` — read a single JSON `{"args":[...], "kwargs":{...}}` doc from stdin (used by the real-message extractor for `parse_month_references` / `match_members`).
|
||||
- `--all` — iterate every seed for one function, emit newline-delimited JSON to stdout. Used by the `make capture-fixtures` recipe.
|
||||
|
||||
Capture **never writes files**. Output goes to stdout; the caller redirects. The scrubber is always stdin→stdout. Both are pure transforms.
|
||||
|
||||
The `make capture-fixtures` target codifies the full refresh workflow. Humans read the target before they read the README.
|
||||
|
||||
## Fixture JSON shape (normative)
|
||||
|
||||
One JSON object per case:
|
||||
|
||||
```json
|
||||
{
|
||||
"case": "range_wrap_nov_to_jan",
|
||||
"func": "scripts.czech_utils.parse_month_references",
|
||||
"captured_at": "2026-05-06",
|
||||
"input": { ... },
|
||||
"output": { ... }
|
||||
}
|
||||
```
|
||||
|
||||
`captured_at` is date-only — same-day re-runs produce byte-identical files. No git SHA, no hostname, no time component.
|
||||
|
||||
### Per-function input/output schemas
|
||||
|
||||
The schema is the **stable contract** between Python capture and Go consumption. Where Python returns heterogeneous types, the capture step pre-translates to the typed shape Go expects.
|
||||
|
||||
| Function | Input | Output |
|
||||
|---|---|---|
|
||||
| `normalize` | `{"text":"…"}` | `{"text":"…"}` |
|
||||
| `parse_month_references` | `{"text":"…","default_year":2026}` | `{"months":["2026-01",…]}` |
|
||||
| `calculate_fee` | `{"attendance_count":3,"month_key":"2026-02"}` | `{"fee":750}` |
|
||||
| `calculate_junior_fee` | `{"attendance_count":1,"month_key":"2026-02"}` | `{"value":0,"unknown":true}` (mirrors `fees.Expected{Value, Unknown}`) |
|
||||
| `parse_czk_amount` | `{"val":<envelope>}` | `{"amount":1500.0}` |
|
||||
| `generate_sync_id` | `{"tx":{"date":"…","amount":<envelope>,"currency":"CZK","sender":"…","vs":"…","message":"…","bank_id":"…"}}` | `{"sync_id":"<sha256-hex>"}` |
|
||||
| `_build_name_variants` | `{"name":"…"}` | `{"variants":["…"]}` |
|
||||
| `match_members` | `{"text":"…","member_names":["…"]}` | `{"matches":[{"name":"…","confidence":"auto"}]}` |
|
||||
| `infer_transaction_details` | `{"tx":{"sender":"…","message":"…","user_id":"…","date":<envelope>},"member_names":[…],"default_year":2026}` | `{"members":[…],"months":[…],"search_text":"…"}` |
|
||||
| `format_date` | `{"val":<envelope>}` | `{"date":"…"}` |
|
||||
|
||||
**Type envelope** (used in 4 fields above):
|
||||
|
||||
```json
|
||||
{"type":"int","value":750} // distinguishes 750 from 750.0
|
||||
{"type":"float","value":750.0}
|
||||
{"type":"string","value":"…"}
|
||||
{"type":"none"}
|
||||
```
|
||||
|
||||
The envelope is the answer to the `generate_sync_id` parity risk: Python's `str(750.0) == "750.0"` vs `str(750) == "750"` produces different SHA-256 inputs. JSON natively conflates these; the envelope round-trips them. Go's loader switches on `type` and constructs the matching native value before calling the port.
|
||||
|
||||
**`reconcile`** uses raw JSON for everything (its inputs are typed maps/slices already), with one nuance: the `Member.fees[month]` value can be an `int` or a `(fee, count)` tuple per [match_payments.py:339-340](../../srv/personal/fuj-management/scripts/match_payments.py#L339). Capture normalises both to `{"fee":int,"count":int}` so Go side has one shape.
|
||||
|
||||
## Scrubber strategy
|
||||
|
||||
`scrub_fixtures.py`: stdin → stdout, no state, no salt, no random. Deterministic plain SHA-256. Re-runs are idempotent. Trade-off acknowledged: an attacker with the script can mathematically reverse the mapping. That's fine — the scrubber's job is to keep PII out of git diffs and Claude transcripts, not to defend against an adversary with the source tree.
|
||||
|
||||
### Scramble whitelist (only these field keys are scrambled)
|
||||
|
||||
`name`, `member_names[]`, `person`, `sender`, `sender_account`, `account`, `vs`, `bank_id`, `user_id`, `note`. Plus a per-document name-substring sweep over `message` strings — applied **before** the field-key walk, because real names show up embedded in message text.
|
||||
|
||||
Everything else (dates, amounts, currency, `month_key`, `attendance_count`, `purpose`, `confidence`, `expected`, `paid`, `total_balance`, `fee`, all `YYYY-MM` keys, `match`/`matches` structure) is preserved verbatim. **Whitelist-of-scramble** (not blacklist-of-preserve): when a new field appears, it stays raw until someone explicitly adds it to the list. Fails safe.
|
||||
|
||||
### Scrambling functions
|
||||
|
||||
- **Names**: `Member_<8hex>` where `<8hex> = sha256(name).hexdigest()[:8]`. Same name → same pseudonym across the whole document and across all fixtures. Stable diffs.
|
||||
- **Account numbers** (`[0-9]+/[0-9]{4}`): scramble prefix and bank-suffix separately, preserving length and format.
|
||||
- **VS / bank_id / user_id**: digit-string-preserving hash to a same-length numeric token. Non-numeric input → `id_<8hex>`.
|
||||
- **Note**: replaced verbatim with `"<scrubbed>"`. Notes are never load-bearing for any test.
|
||||
- **Message** (free text): name-sweep applied; rest preserved. Corpus author spot-checks before commit. README §5 documents the audit grep.
|
||||
|
||||
## Reconcile fixtures (10 handcrafted cases)
|
||||
|
||||
All seeds live in `_fixture_seeds.py` as triples `(members, sorted_months, transactions, exceptions, default_year)`. Capture runs the live Python `reconcile()` and emits canonical JSON; scrubber is a no-op for handcrafted synthetic names but runs anyway for uniformity.
|
||||
|
||||
| File | Branch exercised |
|
||||
|---|---|
|
||||
| `01_greedy_exact.json` | Greedy: amount == sum(expected); zero credit. |
|
||||
| `02_greedy_overpayment_credit.json` | Greedy with overflow → credit. |
|
||||
| `03_proportional_remainder.json` | Underpayment across 3 months with non-integer split (last month absorbs float remainder per [match_payments.py:421+](../../srv/personal/fuj-management/scripts/match_payments.py#L421)). |
|
||||
| `04_even_split_prepayment.json` | All `expected == 0` → even-split fallback. |
|
||||
| `05_out_of_window_credit.json` | Month outside `sorted_months` → that share goes to credits, in-window proportional for the rest. |
|
||||
| `06_exception_override.json` | Exception entry overrides expected. |
|
||||
| `07_other_purpose_split.json` | `purpose="other:tournament"` with two members. |
|
||||
| `08_junior_question_mark.json` | Junior with attendance count 1 → `Expected{Unknown:true}`; reconcile reads it as 0 expected. |
|
||||
| `09_multiperson_multimonth.json` | `person="Alice, Bob", purpose="2026-01, 2026-02"` → 2x2 fan-out: even-split-by-people then proportional-by-month. |
|
||||
| `10_unmatched.json` | Empty `person`, garbage message → goes to `unmatched`. |
|
||||
|
||||
The seed registry is the **single source of truth** for these inputs. If Python behaviour drifts intentionally, fixtures regenerate cleanly via `make capture-fixtures`.
|
||||
|
||||
## Real-data seeds (for `parse_month_references` and `match_members` only)
|
||||
|
||||
`_fixture_seeds.py` reads `tmp/payments_transactions_cache.json` (already gitignored) and selects:
|
||||
|
||||
- **`parse_month_references`**: ~15 distinct messages exercising the 45 Czech month declensions, range wraps (`"prosinec-leden"`), year inference, and the `m >= 10 → previous year` heuristic. Selection done once interactively, the chosen indices hardcoded into `_fixture_seeds.py` so re-runs are deterministic. Messages flow through capture (which calls `parse_month_references(msg, default_year=2026)`) then scrubber (name-sweep against the live member roster).
|
||||
- **`match_members`**: ~10 distinct `(message, member_names)` pairs exercising auto vs review confidence, common-surname filter, exact-short-circuit. Same pipeline.
|
||||
|
||||
**Out of scope for real seeds**: `normalize`, `_build_name_variants`, `reconcile`. These either don't benefit from real data (synthetic exhaustively covers `normalize`, `_build_name_variants`) or have surgical-input requirements that real data can't reliably hit (`reconcile`'s 10 branches).
|
||||
|
||||
## Go parity-test layout
|
||||
|
||||
One file per function, one Go package per function, mirroring the fixture tree. Each file is short (~30 lines):
|
||||
|
||||
```go
|
||||
//go:build parity
|
||||
|
||||
package normalize_parity_test
|
||||
|
||||
import (
|
||||
"fuj-management/go/internal/domain/czech"
|
||||
"fuj-management/go/tests/parity"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNormalizeParity(t *testing.T) {
|
||||
t.Parallel()
|
||||
parity.RunAll(t, "../../../fixtures/pure/normalize",
|
||||
func(in parity.NormalizeIn) parity.NormalizeOut {
|
||||
return parity.NormalizeOut{Text: czech.Normalize(in.Text)}
|
||||
})
|
||||
}
|
||||
```
|
||||
|
||||
The shared [go/tests/parity/parityio.go](../../srv/personal/fuj-management/go/tests/parity/parityio.go) (also `//go:build parity`) provides:
|
||||
|
||||
- `Case[I, O any]` generic loader: walks a fixture directory, decodes each `.json`, returns `(name, input, want)` triples.
|
||||
- `RunAll[I, O any](t, dir, fn func(I) O)`: invokes `fn`, compares against `want` with `reflect.DeepEqual` (sorted-slice normalisation for the few sets-cast-to-lists Python returns); for floats uses `math.Abs(got-want) <= 0.01`.
|
||||
- One typed `<Func>In` / `<Func>Out` struct pair per function (10 pairs), mirroring §3's JSON shape exactly. Envelope decoder helpers (`AmountEnvelope`, `ValueEnvelope`) live here.
|
||||
|
||||
**Reconcile is bespoke** — `reconcile/reconcile_parity_test.go` doesn't use `RunAll` because it needs cell-by-cell tolerant float compare across nested maps. It walks the fixture dir directly.
|
||||
|
||||
**Why one-file-per-function** (instead of an umbrella runner): each function lives in a different domain package, so tests must `import` a different package; an umbrella would obscure which package is being checked. Split also enables `go test -tags=parity ./tests/parity/pure/normalize/` to iterate on a single port.
|
||||
|
||||
**Why a separate test tree** (instead of co-located parity tests): the M2 unit tests are co-located by convention (e.g. [go/internal/domain/czech/normalize_test.go](../../srv/personal/fuj-management/go/internal/domain/czech/normalize_test.go)). The progress tracker explicitly says fixtures live at `go/tests/fixtures/` and the gate is `go test -tags=parity ./tests/parity/pure/...`. Co-location would scatter fixtures across packages — messy. Separate tree wins.
|
||||
|
||||
## Build tag + Makefile
|
||||
|
||||
Every parity test file starts with `//go:build parity`. Default `make go-test` excludes them; `make go-parity` runs them:
|
||||
|
||||
```makefile
|
||||
go-parity:
|
||||
cd $(GO_SRC) && go test -tags=parity ./tests/parity/...
|
||||
|
||||
go-test-all: go-test go-parity
|
||||
|
||||
capture-fixtures:
|
||||
@bash scripts/capture_all_fixtures.sh # invokes capture | scrub for every seed
|
||||
```
|
||||
|
||||
Parity is **not** folded into default `go-test`: keeps the M2 unit-test loop fast, and a missing-fixture failure shouldn't block routine work. CI runs both targets independently so a parity break is a distinct red signal from a unit-test break.
|
||||
|
||||
## README content (`go/tests/fixtures/README.md`)
|
||||
|
||||
Six sections, ~120 lines:
|
||||
|
||||
1. **What's in this tree** — directory map; one line per fixture function explaining what it validates.
|
||||
2. **Fixture format** — link to schemas in §3; worked example for `parse_month_references` and one for `reconcile`.
|
||||
3. **Refresh workflow** — `make capture-fixtures` regenerates everything; single-file recipe for incremental updates. Always diff before committing.
|
||||
4. **When to refresh** — bullet list (schema change, new Czech declension, new fee tier, new reconcile branch). **Do not refresh to "fix" a parity failure** without first proving the Python behaviour is the intended one.
|
||||
5. **Verifying scrubbing** — `git diff` should show only `Member_<hex>`-shaped names, `<scrubbed>` notes, structurally-preserved account/VS digits. Audit grep: `git ls-files go/tests/fixtures | xargs grep -l '<your real name>'` should return zero before commit.
|
||||
6. **Adding a new fixture** — three steps (add to `_fixture_seeds.py`, run capture, add `In/Out` Go struct fields if needed).
|
||||
|
||||
## Parity concerns
|
||||
|
||||
- **Float arithmetic in reconcile proportional phase**: ordering-sensitive, may diverge between Python and Go due to FMA. Tolerance `0.01` already in [go/internal/domain/reconcile/reconcile_test.go](../../srv/personal/fuj-management/go/internal/domain/reconcile/reconcile_test.go); parity uses the same tolerance.
|
||||
- **Sync-ID float-vs-int stringification**: handled by the envelope (§3). Capture two paired cases per amount value (`amount_750_int.json`, `amount_750_float.json`) so any Go-side conflation surfaces immediately.
|
||||
- **NFKD edge cases**: capture set must include rare characters from real names. The handcrafted `normalize` seeds enumerate every distinct character observed in the live member roster (extracted once from `tmp/attendance_regular_cache.json`, hardcoded into `_fixture_seeds.py` as a single-character-per-case sweep).
|
||||
- **Czech month declensions**: the real-message seeds for `parse_month_references` cover the wild; handcrafted seeds cover the corner cases (`prosinec-leden` wrap, `m >= 10` heuristic).
|
||||
- **Insertion-order determinism in `reconcile`**: Python 3.7+ dict iteration is insertion-ordered; the seed registry preserves order. Go side iterates `sortedMonths` slice explicitly; the parity test verifies this.
|
||||
- **`infer_transaction_details` default_year**: Python signature defaults to 2026; capture passes `default_year` as an explicit input. Go side reads it from the fixture.
|
||||
|
||||
## Out of scope (explicitly DO NOT touch)
|
||||
|
||||
- Real Google Sheets / Drive / Fio loader implementations — M4.1–M4.6.
|
||||
- Web routes / handlers — M5.
|
||||
- `fuj sync` and `fuj infer` subcommands — M4.7/M4.8.
|
||||
- Tier-2 JSON-API parity (`cmd/parity/main.go`) — M5.4.
|
||||
- Any change to existing Python code (capture is read-only against the production scripts).
|
||||
- Any change to existing Go production code under `go/internal/`.
|
||||
|
||||
## Verification
|
||||
|
||||
1. `make go-build` — clean build (parity tests excluded by default tag).
|
||||
2. `make go-test` — all M2 unit tests still green; no parity test runs.
|
||||
3. `make go-parity` — every fixture in `go/tests/fixtures/pure/` and `go/tests/fixtures/reconcile/` deserialises and passes its parity assertion.
|
||||
4. `make go-lint` — clean (parity test files lint-clean under `-tags=parity` since `golangci-lint` honours build tags via `.golangci.yml`).
|
||||
5. **Capture round-trip**: pick one fixture (e.g. `parse_month_references/range_wrap_nov_to_jan.json`), regenerate via `python scripts/capture_fixtures.py --func parse_month_references --case range_wrap_nov_to_jan --input-seed range_wrap_nov_to_jan | python scripts/scrub_fixtures.py`, confirm byte-identical to the committed file.
|
||||
6. **Scrubbing audit**: run the README §5 grep against any name from the live roster — zero hits.
|
||||
7. **Reconcile branch coverage**: read each of the 10 reconcile fixture files, confirm the `output` field shows the expected branch (e.g. `02_greedy_overpayment_credit.json` has a non-zero `credits` entry; `04_even_split_prepayment.json` has equal `paid` across all months).
|
||||
8. Append CHANGELOG entry per [CLAUDE.md](../../srv/personal/fuj-management/CLAUDE.md) (timestamp via `date "+%Y-%m-%d %H:%M %Z"`).
|
||||
9. Tick M3.1–M3.6 in [docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md](../../srv/personal/fuj-management/docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md) with the merge SHA. Update the M3 milestone summary line if M3 is now fully closed.
|
||||
10. Push branch, open MR via `tea pr create --title "feat(go): fixture capture + characterization framework (M3)" --base main --head feat/m3-fixture-capture`, print URL, leave merge to user.
|
||||
|
||||
## Critical files
|
||||
|
||||
- **Read for parity** — [scripts/czech_utils.py:22](../../srv/personal/fuj-management/scripts/czech_utils.py#L22), [scripts/czech_utils.py:28](../../srv/personal/fuj-management/scripts/czech_utils.py#L28), [scripts/attendance.py:91](../../srv/personal/fuj-management/scripts/attendance.py#L91), [scripts/attendance.py:100](../../srv/personal/fuj-management/scripts/attendance.py#L100), [scripts/infer_payments.py:17](../../srv/personal/fuj-management/scripts/infer_payments.py#L17), [scripts/sync_fio_to_sheets.py:62](../../srv/personal/fuj-management/scripts/sync_fio_to_sheets.py#L62), [scripts/match_payments.py:33](../../srv/personal/fuj-management/scripts/match_payments.py#L33), [scripts/match_payments.py:65](../../srv/personal/fuj-management/scripts/match_payments.py#L65), [scripts/match_payments.py:144](../../srv/personal/fuj-management/scripts/match_payments.py#L144), [scripts/match_payments.py:187](../../srv/personal/fuj-management/scripts/match_payments.py#L187), [scripts/match_payments.py:304](../../srv/personal/fuj-management/scripts/match_payments.py#L304).
|
||||
- **Reuse** — `domain/czech.{Normalize, ParseMonthReferences}`, `domain/fees.{CalculateFee, CalculateJuniorFee, Expected}`, `domain/money.ParseCZK`, `domain/synch.GenerateSyncID`, `domain/matching.{BuildNameVariants, MatchMembers, InferTransactionDetails, FormatDate}`, `domain/reconcile.{Member, Transaction, ExceptionKey, Exception, Result, Reconcile}`.
|
||||
- **Mirror conventions** — package layout from [go/internal/domain/matching/](../../srv/personal/fuj-management/go/internal/domain/matching/) (one symbol per file, top-of-test provenance comments, `t.Parallel()`, `// [Go]` markers for Go-only cases).
|
||||
- **New** — `scripts/{capture_fixtures,scrub_fixtures,_fixture_seeds}.py`; `go/tests/fixtures/README.md` + the corpus; `go/tests/parity/parityio.go` + 10 parity test files + 1 reconcile parity test file.
|
||||
- **Modify** — `Makefile` (3 new targets), `CHANGELOG.md` (1 entry), `docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md` (tick M3.1–M3.6).
|
||||
313
docs/plans/2026-05-06-2341-go-m4-io-layer.md
Normal file
313
docs/plans/2026-05-06-2341-go-m4-io-layer.md
Normal file
@@ -0,0 +1,313 @@
|
||||
# Plan: Go rewrite — M4 IO layer behind interfaces
|
||||
|
||||
Companion to [2026-05-03-2349-go-backend-rewrite.md](2026-05-03-2349-go-backend-rewrite.md)
|
||||
and [2026-05-03-2349-go-backend-rewrite-progress.md](2026-05-03-2349-go-backend-rewrite-progress.md).
|
||||
|
||||
## Context
|
||||
|
||||
M1–M3 are merged: skeleton + tooling, every pure-domain function ported and
|
||||
parity-tested against PII-scrubbed fixtures, and the `fuj fees` / `fuj
|
||||
reconcile` subcommands wired but stubbed (`membership.NewStubSources()`
|
||||
returns `ErrIOPending` for every loader). M4's job is to replace that stub
|
||||
with real IO: read attendance CSVs, read the payments sheet + exceptions
|
||||
tab, fetch Drive `modifiedTime` for cache gating, fetch Fio bank
|
||||
transactions, and append/update rows on the payments sheet — all behind
|
||||
narrow Go interfaces that have in-memory fakes for tests.
|
||||
|
||||
Once M4 lands, `fuj fees`, `fuj reconcile`, `fuj sync`, and `fuj infer` all
|
||||
work end-to-end against the real Google Sheets and the real Fio account, and
|
||||
M5 can start porting the JSON API on top of that IO.
|
||||
|
||||
User-confirmed scope choices for this milestone:
|
||||
- **No live integration tests.** Fakes-only at unit level; live
|
||||
verification deferred to manual smoke during M7.
|
||||
- **Three PRs** (sheets/drive/cache → fio/sync → infer), one per major
|
||||
area, each independently reviewable.
|
||||
- **Attendance stays on CSV-via-public-URL** — matches Python, no extra
|
||||
service-account grant needed.
|
||||
|
||||
## Approach
|
||||
|
||||
### Layering
|
||||
|
||||
```
|
||||
internal/io/ ← raw, narrow clients (one per external system)
|
||||
sheets/ ← typed wrapper around google.golang.org/api/sheets/v4
|
||||
drive/ ← Drive v3, only ModifiedTime
|
||||
attendance/ ← CSV-via-public-URL fetcher (no auth, no Sheets API)
|
||||
fio/ ← FioClient interface + apiClient + transparentClient
|
||||
cache/ ← FileCache: modifiedTime gate + two-TTL fallback + atomic write
|
||||
|
||||
internal/services/membership/ ← already exists; M4 adds adapters that satisfy
|
||||
AttendanceLoader / TransactionLoader / ExceptionLoader
|
||||
by composing io/sheets + io/drive + io/cache + io/attendance.
|
||||
|
||||
internal/services/banksync/ ← new: SyncToSheets (M4.7) + InferPayments (M4.8)
|
||||
composing fio + sheets + attendance loaders.
|
||||
```
|
||||
|
||||
The existing interfaces in [go/internal/services/membership/loader.go](../../go/internal/services/membership/loader.go)
|
||||
(`AttendanceLoader`, `TransactionLoader`, `ExceptionLoader`, `Sources`) are
|
||||
the seam — M4 adds a `NewSources(cfg config.Config) (Sources, error)`
|
||||
constructor next to `NewStubSources()`, and `cmd/fuj/main.go` swaps the
|
||||
stub for it.
|
||||
|
||||
### Auth — service-account only
|
||||
|
||||
Drop the OAuth+`token.pickle` path entirely (the production already uses a
|
||||
service account; the fallback only existed because the original Python
|
||||
script ran from a developer laptop). Sheets and Drive both authenticate via
|
||||
`option.WithCredentialsFile(cfg.CredentialsPath)` plus
|
||||
`option.WithScopes(...)`. Single shared `*http.Client` per backend with a
|
||||
10s timeout (matches `DRIVE_TIMEOUT`).
|
||||
|
||||
### Cache shape
|
||||
|
||||
Match Python's wire format so the `tmp/*_cache.json` directory is shared
|
||||
safely while both backends run side-by-side:
|
||||
|
||||
```json
|
||||
{ "modifiedTime": "<RFC3339>", "data": <list|object>, "cachedAt": "<RFC3339>" }
|
||||
```
|
||||
|
||||
Improvements over Python:
|
||||
- Atomic write: marshal → `os.WriteFile(path+".tmp", ..., 0o600)` →
|
||||
`os.Rename`. Python's plain truncate-write stays as-is until M8.
|
||||
- The two TTLs (`CacheTTL` and `CacheAPICheckTTL`) live in `config.Config`
|
||||
already; only the `CacheDir` field is new.
|
||||
|
||||
The four cache keys mirror Python's `CACHE_SHEET_MAP`:
|
||||
`attendance_regular`, `attendance_juniors`, `exceptions_dict`,
|
||||
`payments_transactions` → maps to either `AttendanceSheetID` or
|
||||
`PaymentsSheetID`.
|
||||
|
||||
When Drive fails, fall back to a synthetic key
|
||||
`fmt.Sprintf("ttl-5m-%d", time.Now().Unix()/300)` so cache still keys
|
||||
deterministically per 5-min bucket (same as Python).
|
||||
|
||||
### Fio: two impls behind one interface
|
||||
|
||||
```go
|
||||
type Client interface {
|
||||
FetchTransactions(ctx context.Context, from, to time.Time) ([]Transaction, error)
|
||||
}
|
||||
```
|
||||
|
||||
`apiClient` (when `cfg.FioAPIToken != ""`) hits
|
||||
`https://fioapi.fio.cz/v1/rest/periods/{token}/{from}/{to}/transactions.json`,
|
||||
unmarshals via a typed struct, and maps `column0..column22` to fields per
|
||||
[scripts/fio_utils.py](../../scripts/fio_utils.py:90). Negative-amount rows
|
||||
dropped (matches Python).
|
||||
|
||||
`transparentClient` (fallback) GETs
|
||||
`https://ib.fio.cz/ib/transparent?a={accountNum}&f={DD.MM.YYYY}&t={DD.MM.YYYY}`
|
||||
and walks the response with `golang.org/x/net/html` token visitor, counting
|
||||
`<table class="table">` tags and grabbing rows from the **second** one
|
||||
(skipping `<thead>`). `bank_id`, `currency`, `user_id`, `sender_account`
|
||||
are empty (matches Python — known limitation).
|
||||
|
||||
`accountNum` is derived from `cfg.BankAccount` by stripping the IBAN prefix
|
||||
(`CZ85 2010 0000 0028 0035 9168` → `2800359168`); add a small helper in
|
||||
`config` for this since both the API URL and the transparent URL need it.
|
||||
|
||||
### Fakes
|
||||
|
||||
In-memory fakes live next to each real impl: `sheets/fake.go`,
|
||||
`drive/fake.go`, `fio/fake.go`, `attendance/fake.go`,
|
||||
`cache/fake.go` (a passthrough). All exported as `Fake` so tests do
|
||||
`sheets.NewFake(rows)` and inject. The membership-adapter tests use these
|
||||
fakes plus a couple of new raw-bytes fixtures under
|
||||
`go/internal/io/<pkg>/testdata/`:
|
||||
|
||||
- `sheets/testdata/payments_minimal.json` — 2D-string array shaped like
|
||||
`values.get` would return.
|
||||
- `sheets/testdata/exceptions_minimal.json` — same, for the exceptions tab.
|
||||
- `attendance/testdata/adults_minimal.csv` — small adult attendance CSV.
|
||||
- `attendance/testdata/juniors_minimal.csv` — small junior CSV.
|
||||
- `fio/testdata/api_response.json` — captured Fio API JSON shape.
|
||||
- `fio/testdata/transparent.html` — captured transparent-page HTML.
|
||||
|
||||
Existing M3 domain fixtures under `go/tests/fixtures/` stay where they are
|
||||
and continue to drive parity tests; they aren't reused for IO-layer tests
|
||||
because they're at the wrong layer (post-parse domain types).
|
||||
|
||||
## Tasks (mapped to tracker)
|
||||
|
||||
Same 8 sub-milestones as the tracker, grouped into 3 PRs.
|
||||
|
||||
### PR 1 — sheets / drive / cache + membership wiring (M4.1, M4.2, M4.3, M4.6)
|
||||
|
||||
1. **Add deps** in [go/go.mod](../../go/go.mod):
|
||||
`google.golang.org/api/{sheets/v4,drive/v3,option}`,
|
||||
`golang.org/x/oauth2/google` (transitively pulled), `golang.org/x/net/html`.
|
||||
2. **`internal/io/sheets/`**:
|
||||
- `client.go` — `Client` struct holding `*sheets.Service`; methods
|
||||
`GetValues(ctx, spreadsheetID, a1Range string) ([][]any, error)`,
|
||||
`AppendValues(ctx, spreadsheetID, a1Range string, rows [][]any) error`,
|
||||
`BatchUpdateValues(ctx, spreadsheetID, updates []ValueRange) error`,
|
||||
`SortByColumn(ctx, spreadsheetID, sheetGID int64, columnIndex int) error`.
|
||||
- `fake.go` — exported `Fake` with seedable `Values map[string][][]any`.
|
||||
3. **`internal/io/drive/`**:
|
||||
- `client.go` — `Client.ModifiedTime(ctx, fileID string) (string, error)`
|
||||
using `drive.New(...).Files.Get(fileID).Fields("modifiedTime").SupportsAllDrives(true)`.
|
||||
- `fake.go` with seedable `Times map[string]string`.
|
||||
4. **`internal/io/attendance/`** (new — public-URL CSV):
|
||||
- `client.go` — `Client.FetchAdults(ctx) ([][]string, error)` and
|
||||
`FetchJuniors(ctx) ([][]string, error)` using `http.Get` on
|
||||
`https://docs.google.com/spreadsheets/d/{ID}/export?format=csv&gid={GID}`,
|
||||
decoded via `encoding/csv`.
|
||||
- Add `AttendanceAdultSheetGID = "0"` constant in `internal/config`.
|
||||
5. **`internal/io/cache/`**:
|
||||
- `filecache.go` — `FileCache` with `Get(ctx, key string, fetch func(ctx) (any, error)) (any, error)`
|
||||
wired through `Drive.ModifiedTime` and the two TTL knobs. Atomic write
|
||||
via tmp-file + rename.
|
||||
- Cache key → sheet ID map mirrors Python's `CACHE_SHEET_MAP`.
|
||||
6. **`internal/services/membership/sources.go`** (new file in existing
|
||||
package):
|
||||
- `realSources struct { sheets *sheets.Client; drive *drive.Client; attendance *attendance.Client; cache *cache.FileCache }`.
|
||||
- Constructor `NewSources(ctx, cfg) (Sources, error)` builds all clients.
|
||||
- `LoadAdults` reads cached attendance CSV, runs through
|
||||
`domain/fees.CalculateFee` + merged-month logic (port of
|
||||
[scripts/attendance.py](../../scripts/attendance.py:170)
|
||||
`get_members_with_fees`), returns `[]reconcile.Member`.
|
||||
- `LoadTransactions` reads payments sheet rows via cache, parses to
|
||||
`[]reconcile.Transaction` (port of
|
||||
[match_payments.py:208](../../scripts/match_payments.py:208)
|
||||
`fetch_sheet_data`).
|
||||
- `LoadExceptions` reads `'exceptions'!A2:D` via cache, builds
|
||||
`map[ExceptionKey]Exception` (port of `match_payments.py:266`).
|
||||
7. **Add `LoadJuniors`** to the `AttendanceLoader` interface (Python infer
|
||||
pulls both adult + junior member lists; needed for M4.8).
|
||||
8. **Wire into [cmd/fuj/main.go](../../go/cmd/fuj/main.go)**: replace
|
||||
`membership.NewStubSources()` in `feesCmd` and `reconcileCmd` with
|
||||
`membership.NewSources(ctx, cfg)`.
|
||||
9. **Tests** (default tag, no live IO):
|
||||
- `sheets/client_test.go`, `drive/client_test.go`,
|
||||
`cache/filecache_test.go` — exercise fakes + parsing logic with
|
||||
testdata fixtures.
|
||||
- `membership/sources_test.go` — adapter tests with sheets/drive/cache
|
||||
fakes verify CSV→Member, rows→Transaction, exceptions tab → map.
|
||||
10. **Config additions**: `CacheDir` (default `tmp` relative to `$PWD`,
|
||||
overridable via `CACHE_DIR` env), `DriveTimeout` (default 10s).
|
||||
11. **Manual verification**: `make go-build && go run ./cmd/fuj fees` and
|
||||
`... reconcile` print real reports against the live sheet (with valid
|
||||
`.secret/...credentials.json`).
|
||||
12. CHANGELOG entry; tick M4.1, M4.2, M4.3, M4.6 in the progress tracker.
|
||||
|
||||
### PR 2 — fio + bank sync (M4.4, M4.5, M4.7)
|
||||
|
||||
1. **`internal/io/fio/`**:
|
||||
- `client.go` — `Client` interface, `Transaction` struct.
|
||||
- `api.go` — `apiClient` impl + URL builder + JSON struct definitions
|
||||
for `accountStatement.transactionList.transaction[].column{N}.value`.
|
||||
- `transparent.go` — `transparentClient` impl using
|
||||
`golang.org/x/net/html` token visitor; helper functions
|
||||
`parseCzechAmount` (NBSP/space strip + comma→dot) and
|
||||
`parseCzechDate` (DD.MM.YYYY / DD/MM/YYYY).
|
||||
- `fake.go`.
|
||||
- `New(cfg) Client` chooses impl based on `cfg.FioAPIToken`.
|
||||
- `accountNum(iban)` helper in `internal/config` strips IBAN prefix.
|
||||
2. **`internal/services/banksync/sync.go`** (new package):
|
||||
- `SyncToSheets(ctx, cfg, fio Client, sheets *sheets.Client, opts SyncOpts) (added int, err error)`.
|
||||
- Reads existing rows via `sheets.GetValues(... "A1:K")`, validates
|
||||
header against `COLUMN_LABELS`, writes header if missing, builds
|
||||
`existingIDs` from column K (`Sync ID`).
|
||||
- Computes date window: explicit `from`/`to` or `now - days*24h` (default 30d).
|
||||
- For each fetched tx, computes `domain/synch.GenerateSyncID`, skips if
|
||||
present, otherwise builds row in COLUMN_LABELS order with empty
|
||||
manual/person/purpose/inferred slots.
|
||||
- `sheets.AppendValues(... "A2", rows)`.
|
||||
- Optional sort: `sheets.SortByColumn(... gid, 0)` — sheet GID resolved
|
||||
once via `spreadsheets.Get`.
|
||||
3. **Wire `fuj sync` subcommand** in `cmd/fuj/main.go`:
|
||||
- Flags: `--days N` (default 30), `--from YYYY-MM-DD`, `--to YYYY-MM-DD`,
|
||||
`--sort` (default true matching `make sync-2026`).
|
||||
- Replace the M4-stub error path.
|
||||
4. **Tests** (default tag): `banksync/sync_test.go` with fakes — verify
|
||||
header insertion, dedup against existing sync IDs, multi-row append,
|
||||
sort call.
|
||||
5. **Manual verification**: dry-run sync against the real Fio account in a
|
||||
throwaway test sheet; or visually verify `--from --to` window in stdout
|
||||
with a no-write flag (only if cheap to add — otherwise skip per the
|
||||
"no live integration tests" decision).
|
||||
6. CHANGELOG entry; tick M4.4, M4.5, M4.7.
|
||||
|
||||
### PR 3 — infer (M4.8)
|
||||
|
||||
1. **`internal/services/banksync/infer.go`**:
|
||||
- `InferPayments(ctx, cfg, sheets *sheets.Client, attendanceLoader, juniorLoader, opts InferOpts) (updated int, err error)`.
|
||||
- Reads payments sheet `A1:Z` with case-insensitive header lookup.
|
||||
- Required columns: `Person, Purpose, Inferred Amount`. Optional input:
|
||||
`Date, Amount, Sender, Message, VS, manual fix`.
|
||||
- Skip rule (matches [scripts/infer_payments.py:127](../../scripts/infer_payments.py:127)):
|
||||
non-empty `manual fix` OR `Person` OR `Purpose` → leave row alone.
|
||||
- Member list = union of `LoadAdults` + `LoadJuniors` deduped via
|
||||
`domain/matching.CanonicalKey` (already exists from M2).
|
||||
- For each empty row: build tx dict, call
|
||||
`domain/matching.InferTransactionDetails`, prefix `[?] ` if
|
||||
confidence == "review", emit a `ValueRange` update with R1C1 range
|
||||
`R{i}C{personCol+1}:R{i}C{amountCol+1}`.
|
||||
- Single `sheets.BatchUpdateValues` call for all updates.
|
||||
2. **Wire `fuj infer` subcommand**: flags `--dry-run` (prints planned
|
||||
updates, no API write).
|
||||
3. **Tests** (default tag): `banksync/infer_test.go` — fixture rows,
|
||||
verify skip rule, verify `[?]` prefix on review matches, verify
|
||||
batchUpdate payload shape, verify `--dry-run` is no-op.
|
||||
4. CHANGELOG entry; tick M4.8 → milestone gate ✅.
|
||||
|
||||
## Critical files
|
||||
|
||||
To modify:
|
||||
- [go/internal/services/membership/loader.go](../../go/internal/services/membership/loader.go) — add `LoadJuniors` to `AttendanceLoader`, add `NewSources`.
|
||||
- [go/cmd/fuj/main.go](../../go/cmd/fuj/main.go) — swap stub for real sources, add `sync`/`infer` subcommands.
|
||||
- [go/internal/config/config.go](../../go/internal/config/config.go) — add `CacheDir`, `DriveTimeout`, `AttendanceAdultSheetGID` constant, IBAN→account-num helper.
|
||||
- [go/go.mod](../../go/go.mod) / `go.sum` — google APIs + `x/net/html`.
|
||||
- [docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md](2026-05-03-2349-go-backend-rewrite-progress.md) — tick M4.x boxes after each PR.
|
||||
- [CHANGELOG.md](../../CHANGELOG.md) — entry per PR.
|
||||
|
||||
To create:
|
||||
- `go/internal/io/{sheets,drive,attendance,fio,cache}/{client,fake,*_test}.go`
|
||||
- `go/internal/io/{sheets,attendance,fio}/testdata/*`
|
||||
- `go/internal/services/membership/sources.go` (+ `sources_test.go`)
|
||||
- `go/internal/services/banksync/{sync,infer}.go` (+ tests)
|
||||
|
||||
## Reused existing helpers
|
||||
|
||||
- `domain/fees.CalculateFee` / `CalculateJuniorFee` — fee math (M2.3, M2.4).
|
||||
- `domain/matching.{BuildNameVariants,MatchMembers,InferTransactionDetails,FormatDate,CanonicalKey}` — match logic (M2.7–M2.9).
|
||||
- `domain/synch.GenerateSyncID` — dedup hash (M2.6).
|
||||
- `domain/reconcile.{Member,Transaction,Exception,ExceptionKey}` — domain types.
|
||||
- `domain/czech.{Normalize,ParseMonthReferences}` — used inside the
|
||||
attendance/exceptions parsers.
|
||||
- `domain/money.ParseCZK` — for parsing transparent-scrape amounts.
|
||||
|
||||
## Verification
|
||||
|
||||
End-to-end checks once all three PRs land:
|
||||
|
||||
1. `make go-build && make go-lint && make go-test` — clean.
|
||||
2. `make go-parity` — M3 fixtures still pass (no domain regressions).
|
||||
3. `./bin/fuj fees` — prints adult fee report matching Python `make fees`
|
||||
(visual diff acceptable for now; byte-equality enforced in M5).
|
||||
4. `./bin/fuj reconcile` — prints balance report comparable to
|
||||
[scripts/match_payments.py](../../scripts/match_payments.py) `print_balance_report`.
|
||||
5. `./bin/fuj sync --days 7` — appends new Fio rows to the payments sheet
|
||||
(run with a real but recent date window; verify by counting added rows
|
||||
and confirming no duplicates on a second run).
|
||||
6. `./bin/fuj infer --dry-run` — prints planned Person/Purpose/Inferred
|
||||
Amount updates without modifying the sheet. Then `./bin/fuj infer`
|
||||
applies them; second run is a no-op (skip rule).
|
||||
7. **Cache check**: delete `tmp/*_cache.json`, run `fuj fees`, verify file
|
||||
appears with `modifiedTime` matching Drive. Re-run within 5 min;
|
||||
verify no Drive call (debug log).
|
||||
8. **Cross-process cache safety**: while `make web-py` is running, run
|
||||
`fuj reconcile`; verify Python's cache file isn't corrupted and Go
|
||||
reads the same data.
|
||||
|
||||
Gate (per tracker):
|
||||
> `go test -tags=integration ./internal/io/...` round-trips against test sheet; default-tag tests run on fakes.
|
||||
|
||||
Per the user's scope decision, **the integration-test gate is downgraded
|
||||
to "default-tag tests on fakes" only**. Live verification is deferred to
|
||||
manual smoke during M7's parallel-run watch period. The progress tracker's
|
||||
M4 gate line will be amended in PR 1.
|
||||
36
docs/plans/2026-05-07-1033-fuj-sync-dry-run.md
Normal file
36
docs/plans/2026-05-07-1033-fuj-sync-dry-run.md
Normal file
@@ -0,0 +1,36 @@
|
||||
# Plan: add `--dry-run` to `fuj sync`
|
||||
|
||||
## Context
|
||||
|
||||
`fuj infer` already supports `--dry-run` (it builds the planned `BatchUpdateValues`
|
||||
operations, prints them, and skips the actual write — see
|
||||
`go/internal/services/banksync/infer.go:136-156` and the
|
||||
`Dry run: would update N row(s).` line in `go/cmd/fuj/main.go:209-213`).
|
||||
|
||||
`fuj sync` had no equivalent. It always committed three potential writes to the
|
||||
payments sheet: `WriteHeader` (if the header row is missing/wrong), `AppendValues`
|
||||
(for each new Fio transaction), and `SortByDateColumn` (if `--sort`, default true).
|
||||
For inspecting what a sync *would* do — useful when debugging dedupe, sanity-checking
|
||||
a date window, or wiring up the command for the first time on a new account — the
|
||||
only options were pointing at a throwaway spreadsheet or reading the diff after the fact.
|
||||
|
||||
This change mirrors `infer`'s read-only mode for `sync`: same flag name, same output
|
||||
style, same "build the data structures, print instead of writing" shape.
|
||||
|
||||
## Files modified
|
||||
|
||||
1. `go/internal/services/banksync/sync.go` — `DryRun bool` field added to `SyncOpts`; three write points gated on `opts.DryRun`
|
||||
2. `go/cmd/fuj/main.go` — `--dry-run` flag added to `syncCmd`; final println split on `*dryRun`
|
||||
3. `go/internal/services/banksync/sync_test.go` — `TestSyncToSheets_DryRun` added
|
||||
4. `CHANGELOG.md` — entry added
|
||||
|
||||
## Behaviour
|
||||
|
||||
When `--dry-run` is set:
|
||||
|
||||
- If the sheet header is missing/wrong → prints `Dry run: would write header row`; skips `WriteHeader`
|
||||
- For each non-deduped Fio transaction → prints `Dry run: would append date=… amount=… sender=… vs=… message=…`; skips `AppendValues`
|
||||
- If `--sort` is true → prints `Dry run: would sort by date`; skips `SortByDateColumn`
|
||||
- Returns `len(newRows)` so the caller can print `Dry run: would sync N new transaction(s).`
|
||||
|
||||
The existing ID-dedup logic runs in full even during dry-run (reads the sheet, builds `existingIDs`), so the output reflects exactly what the next real sync would do.
|
||||
29
docs/plans/2026-05-07-1321-fuj-sync-print-fio-table.md
Normal file
29
docs/plans/2026-05-07-1321-fuj-sync-print-fio-table.md
Normal file
@@ -0,0 +1,29 @@
|
||||
# Add `--print-fio-table` debug flag to `fuj sync`
|
||||
|
||||
## Context
|
||||
|
||||
The Go port of `fuj sync --dry-run` currently prints only the **new**
|
||||
transactions — i.e. rows that will be appended to the payments sheet after
|
||||
deduping against existing Sync IDs (see [sync.go:125-129](../../go/internal/services/banksync/sync.go#L125-L129)).
|
||||
When debugging Fio sync issues ("why isn't transaction X showing up?",
|
||||
"is the dedup working?"), there's no way to see what Fio actually
|
||||
returned versus what got filtered as a duplicate.
|
||||
|
||||
This change adds a `--print-fio-table` flag that, **only when combined
|
||||
with `--dry-run`**, prints an aligned table of every Fio transaction in
|
||||
the window with each row marked `NEW` (would be appended) or `DUP`
|
||||
(already in sheet, skipped). The flag is silently ignored without
|
||||
`--dry-run`, so it can't accidentally fire during a real sync.
|
||||
|
||||
## Decisions
|
||||
|
||||
- Flag name: `--print-fio-table` (specific, not generic `--verbose`).
|
||||
- Columns: `DATE | AMOUNT | SENDER | VS | MESSAGE | BANKID | STATUS`,
|
||||
with MESSAGE truncated and STATUS = `NEW` / `DUP`.
|
||||
- Scope: only effective when `--dry-run` is also set.
|
||||
|
||||
## Files modified
|
||||
|
||||
- [go/cmd/fuj/main.go](../../go/cmd/fuj/main.go) — new flag + SyncOpts field
|
||||
- [go/internal/services/banksync/sync.go](../../go/internal/services/banksync/sync.go) — SyncOpts struct + refactored step 4
|
||||
- [go/internal/services/banksync/debug.go](../../go/internal/services/banksync/debug.go) — printFioTable helper (new)
|
||||
185
docs/plans/2026-05-07-1431-m5-json-api-parity.md
Normal file
185
docs/plans/2026-05-07-1431-m5-json-api-parity.md
Normal file
@@ -0,0 +1,185 @@
|
||||
# Python view-model cleanup (M5 prep — Python side only)
|
||||
|
||||
Scoped-down precursor to M5 of the Go rewrite. See:
|
||||
- [2026-05-03-2349-go-backend-rewrite.md](2026-05-03-2349-go-backend-rewrite.md)
|
||||
- [2026-05-03-2349-go-backend-rewrite-progress.md](2026-05-03-2349-go-backend-rewrite-progress.md)
|
||||
|
||||
## Context
|
||||
|
||||
The Python app is still production. Before adding `/api/X` shadow
|
||||
routes, JSON DTOs, parity tooling, and Go handlers, get the Python
|
||||
side into a clean shape. Today the `/adults` and `/juniors` routes
|
||||
each carry 150–200 lines of inline view-model construction inside
|
||||
the Flask handler:
|
||||
|
||||
- [adults_view](app.py#L179) — 167 LOC, computes per-row
|
||||
status/cell_text/balance/credits/debts inline.
|
||||
- [juniors_view](app.py#L347) — 205 LOC, same plus `"?"` sentinel
|
||||
branching and `:NJ,MA` breakdown.
|
||||
- [payments](app.py#L553) — 35 LOC, lighter but still mixes IO and
|
||||
grouping.
|
||||
|
||||
Pulling that into pure builder functions:
|
||||
- shrinks `app.py` and makes each route handler do only what a route
|
||||
handler should (IO + cache + step timing + render);
|
||||
- gives us **already-tested** pure functions that future M5 work can
|
||||
call from a `/api/X` shadow endpoint with one line of `jsonify(...)`;
|
||||
- has zero behavioural change (existing tests in
|
||||
[test_app.py](tests/test_app.py) act as the regression guard).
|
||||
|
||||
This is a Python-only change. No Go work, no JSON contract, no shadow
|
||||
routes — those come after.
|
||||
|
||||
## Approach
|
||||
|
||||
Three pure builder functions in a new `scripts/views.py` module. Each
|
||||
takes already-loaded, deserialized inputs (no Flask globals, no IO,
|
||||
no cache calls) and returns the exact dict that today's route passes
|
||||
to `render_template`.
|
||||
|
||||
```python
|
||||
def build_adults_view_model(
|
||||
members, sorted_months, transactions, exceptions, current_month,
|
||||
*, attendance_url, payments_url, bank_account,
|
||||
) -> dict: ...
|
||||
|
||||
def build_juniors_view_model(
|
||||
junior_members, sorted_months, transactions, exceptions, current_month,
|
||||
*, attendance_url, payments_url, bank_account,
|
||||
) -> dict: ...
|
||||
|
||||
def build_payments_view_model(
|
||||
transactions, member_names,
|
||||
*, attendance_url, payments_url,
|
||||
) -> dict: ...
|
||||
```
|
||||
|
||||
The route handlers shrink to: load data (with `get_cached_data` and
|
||||
`record_step` calls staying in `app.py`), call the builder, render.
|
||||
|
||||
```python
|
||||
@app.route("/adults")
|
||||
def adults_view():
|
||||
members, sorted_months = ... # cached loads + record_step calls
|
||||
transactions = ...
|
||||
exceptions = ...
|
||||
result_meta = ...
|
||||
record_step("process_data")
|
||||
vm = build_adults_view_model(
|
||||
members, sorted_months, transactions, exceptions,
|
||||
datetime.now().strftime("%Y-%m"),
|
||||
attendance_url=..., payments_url=..., bank_account=BANK_ACCOUNT,
|
||||
)
|
||||
return render_template("adults.html", **vm)
|
||||
```
|
||||
|
||||
## Decisions
|
||||
|
||||
1. **Pure builders, no Flask state.** No `record_step`, no `g.*`, no
|
||||
`get_cached_data` inside builders. They take plain args, return
|
||||
plain dicts. This is what makes them trivially unit-testable and,
|
||||
later, trivially reusable from `/api/X`.
|
||||
2. **Preserve byte-equal behaviour.** The dicts returned must match
|
||||
today's `render_template(...)` kwargs key-for-key, value-for-value
|
||||
— including the existing `json.dumps(member_data)` /
|
||||
`json.dumps(month_labels_json)` / `json.dumps(raw_payments_json)`
|
||||
wrappers. Those wrappers exist for inline JS in templates; the
|
||||
refactor doesn't touch them. (When `/api/X` lands later, that route
|
||||
will produce a sibling dict with the wrappers stripped, but that's
|
||||
future work.)
|
||||
3. **`reconcile()` stays inside the route, not the builder.** It
|
||||
crosses the IO/cache boundary in spirit (its inputs come from
|
||||
cache); but it's also pure-domain and called by both adults and
|
||||
juniors. Keep the call site in the route so `record_step("reconcile")`
|
||||
timing isn't lost. Builder takes `result` as an argument.
|
||||
4. **Shared helpers stay in `app.py`** for now —
|
||||
[get_month_labels](app.py#L41) and
|
||||
[group_payments_by_person](app.py#L60) are already module-level
|
||||
pure functions, used by routes and now by builders. Either leave
|
||||
them in `app.py` and import into `scripts/views.py`, or move them
|
||||
into `scripts/views.py`. **Choose: move into `scripts/views.py`**
|
||||
— they're view-model concerns, not Flask concerns, and `app.py`
|
||||
should keep shrinking.
|
||||
5. **No new test file needed.** Existing
|
||||
[tests/test_app.py](tests/test_app.py) tests
|
||||
`test_adults_route`, `test_juniors_route`, `test_payments_route`
|
||||
exercise the rendered HTML end-to-end. If they pass after the
|
||||
refactor, behaviour is preserved. Adding builder-level unit tests
|
||||
is a *nice-to-have* but not required for this iteration.
|
||||
|
||||
## Tasks
|
||||
|
||||
### 1. Create `scripts/views.py` with the three builders + shared helpers
|
||||
|
||||
- Move `get_month_labels` and `group_payments_by_person` from
|
||||
[app.py:41–77](app.py#L41-L77) into `scripts/views.py`. Update the
|
||||
import in `app.py`.
|
||||
- Implement `build_adults_view_model` by extracting
|
||||
[app.py:200–344](app.py#L200-L344) (everything between `result =
|
||||
reconcile(...)` and `return render_template(...)`). Take `result`
|
||||
as a parameter; emit the same dict that's currently passed as
|
||||
`**kwargs` to `render_template`.
|
||||
- Implement `build_juniors_view_model` by extracting
|
||||
[app.py:370–550](app.py#L370-L550). Same shape — including the
|
||||
`adapted_members` adapter loop, the junior `?`-sentinel branches,
|
||||
and the `:NJ,MA` breakdown.
|
||||
- Implement `build_payments_view_model` by extracting
|
||||
[app.py:570–586](app.py#L570-L586) (the `group_payments_by_person`
|
||||
call + `Unmatched / Unknown` bucket + sort).
|
||||
|
||||
### 2. Slim down the route handlers in `app.py`
|
||||
|
||||
Each handler keeps:
|
||||
- `attendance_url`/`payments_url` URL building
|
||||
- `get_cached_data` calls (with `record_step` between)
|
||||
- `reconcile(...)` call (adults/juniors) with `record_step("reconcile")`
|
||||
- `record_step("process_data")` after the builder call
|
||||
- `return render_template("X.html", **view_model)`
|
||||
|
||||
Each handler **drops** all the per-row computation, totals
|
||||
formatting, credits/debts sorting, `raw_payments_by_person` building,
|
||||
and `import json` lines.
|
||||
|
||||
Target post-refactor LOC for each route handler: ~25–30 lines
|
||||
(currently 167 / 205 / 35).
|
||||
|
||||
### 3. Run the existing test suite + manual smoke
|
||||
|
||||
```
|
||||
make test # all tests green
|
||||
make web-py # browse /adults /juniors /payments visually
|
||||
```
|
||||
|
||||
## Critical files
|
||||
|
||||
- [app.py](app.py) — routes shrink dramatically; `get_month_labels`
|
||||
and `group_payments_by_person` get moved out.
|
||||
- New: `scripts/views.py` — three builders + the two helpers.
|
||||
- [tests/test_app.py](tests/test_app.py) — unchanged; serves as the
|
||||
regression guard.
|
||||
|
||||
## Verification
|
||||
|
||||
1. `make test` — all four `TestWebApp` tests pass unchanged.
|
||||
2. `make web-py` and visit `/adults`, `/juniors`, `/payments` — each
|
||||
renders identically to before (same table contents, same totals,
|
||||
same credits/debts, same `?` rendering on juniors).
|
||||
3. `git diff app.py` shows substantial deletions (route bodies
|
||||
shrink) and only thin glue calling the new builders.
|
||||
4. Optional sanity check: temporarily add `print(repr(view_model))`
|
||||
in the route before `render_template` on `main` and on the branch,
|
||||
diff for one fixture run — should be byte-identical dicts.
|
||||
|
||||
## Out of scope (future iterations, not this plan)
|
||||
|
||||
- `/api/<route>` shadow endpoints in Flask
|
||||
- Go `internal/web/api/` DTOs and JSON Schemas
|
||||
- Go `internal/services/membership/views.go` aggregators
|
||||
- Go HTTP handlers for `/api/X`
|
||||
- `cmd/parity/main.go` and `make parity` target
|
||||
- Junior breakdown sidecar work in
|
||||
`go/internal/services/membership/sources.go`
|
||||
|
||||
These are the original M5 tasks (M5.1–M5.4 in the progress tracker).
|
||||
They become much easier once today's refactor lands, because the
|
||||
shadow `/api/X` routes will be one-liners over the new builders.
|
||||
@@ -0,0 +1,240 @@
|
||||
# M5.1 — Hand-author Go API structs + emit JSON Schemas
|
||||
|
||||
Companion to:
|
||||
- [2026-05-03-2349-go-backend-rewrite.md](2026-05-03-2349-go-backend-rewrite.md) (master design)
|
||||
- [2026-05-03-2349-go-backend-rewrite-progress.md](2026-05-03-2349-go-backend-rewrite-progress.md) (progress tracker — M5.1 row)
|
||||
- [2026-05-07-1431-m5-json-api-parity.md](2026-05-07-1431-m5-json-api-parity.md) (Python view-model extraction prep — already merged as `b562ce3` / `32a16ff` / `59223c0`)
|
||||
|
||||
## Context
|
||||
|
||||
M4 (IO layer behind interfaces) just landed. M5 is the JSON-parity contract phase — byte-equal JSON between Python and Go for `/api/adults`, `/api/juniors`, `/api/payments`, `/api/version`. Within M5, the work splits four ways:
|
||||
|
||||
- **M5.1 — this plan.** Define the wire contract: hand-authored Go structs with explicit `json:` tags matching Python keys, plus committed JSON Schemas generated by `github.com/invopop/jsonschema`. **Schemas only — no handlers, no Python `/api/X` routes, no parity tool.**
|
||||
- M5.2 — Implement Go handlers that compose `services/*` results into these structs.
|
||||
- M5.3 — Add Python `/api/X` shadow endpoints in [app.py](app.py).
|
||||
- M5.4 — `cmd/parity/main.go` + `make parity` target.
|
||||
|
||||
The recent Python view-model extraction (`scripts/views.py`) lays the groundwork: every Python builder now returns a plain dict that an `/api/X` shadow can `jsonify` (with one minor unwrap step — see decision #1 below). M5.1 is the matching Go side: types and schemas that pin down the contract before any code writes JSON to a wire.
|
||||
|
||||
## Key design decisions
|
||||
|
||||
1. **Wire format is nested objects, not strings-of-JSON.** The Python view-model dicts contain three template-only fields that are pre-serialized JSON strings: `member_data`, `month_labels_json`, `raw_payments_json`. Those exist purely to feed inline `<script>` blocks in Jinja templates ([scripts/views.py:227-237](scripts/views.py#L227-L237)). The `/api/X` JSON contract uses the **un-stringified** nested form. Rationale:
|
||||
- The master design doc references `total_balance`, `original_expected`, `attendance_count` as struct fields ([2026-05-03-2349-go-backend-rewrite.md:223-225](docs/plans/2026-05-03-2349-go-backend-rewrite.md#L223-L225)), which are *inside* `member_data` — only meaningful if it's a nested object.
|
||||
- The `Expected` `MarshalJSON` design (emit `42` or `"?"`) ([same doc:233-238](docs/plans/2026-05-03-2349-go-backend-rewrite.md#L233-L238)) only fires inside `member_data`; pointless if that field is a string.
|
||||
- Strings-of-JSON make `invopop/jsonschema` schemas useless for those fields (`{"type": "string"}` describes nothing).
|
||||
- The Python prep plan ([2026-05-07-1431-m5-json-api-parity.md:87-89](docs/plans/2026-05-07-1431-m5-json-api-parity.md#L87-L89)) already anticipates this: "that route will produce a sibling dict with the wrappers stripped".
|
||||
|
||||
This refines (does not contradict) M5.3's "no transformation" wording. M5.3's `/api/X` will be `jsonify(unwrap_json_strings(view_model_dict))` — a 4-line shim, not real transformation logic.
|
||||
|
||||
2. **New package: `internal/web/api/`.** The Go side has no `api/` package today — only [internal/web/server.go](go/internal/web/server.go) with one hello route. The api package owns wire types and (in M5.2) handlers. Sub-files per route keep diffs small.
|
||||
|
||||
3. **Wire types are separate from `domain/reconcile.Result`.** [domain/reconcile/reconcile.go:88-92](go/internal/domain/reconcile/reconcile.go#L88-L92) defines `Result`, `MemberResult`, `MonthData`, `TxEntry`, etc. — none have `json:` tags. **Don't tag the domain types**: that bleeds wire concerns into pure logic and locks the JSON contract to internal field names. Define wire types in `internal/web/api/` and convert in M5.2's handlers.
|
||||
|
||||
4. **`Expected{Value int; Unknown bool}` with custom `MarshalJSON`/`UnmarshalJSON`** for junior `expected` and `original_expected`. Already prescribed by the master design.
|
||||
|
||||
5. **Transaction `amount` / `inferred_amount` may be `int` or `""`** (Sheets `UNFORMATTED_VALUE` returns empty string for blank cells per `scripts/match_payments.py` rows 250-260 in the views.py exploration). Use a custom `SheetsNumber` type with `MarshalJSON`/`UnmarshalJSON` that emits `0`/`null` for empty and the number otherwise. Document in code comment with a one-liner. Verify exact behavior by inspecting one or two existing scrubbed reconcile fixtures before coding.
|
||||
|
||||
6. **Schema generation lives in `internal/web/api/schemagen_test.go`** with `-update` flag, à la golden tests. Default test run (`go test ./internal/web/api/...`) re-generates schemas in memory and asserts byte-equality vs the committed files in `tests/fixtures/api-schema/`. `go test -update ./internal/web/api/...` rewrites the committed files. Avoids a separate `cmd/gen-api-schema` binary; CI catches drift automatically.
|
||||
|
||||
7. **One schema per route**, named to match the Go type:
|
||||
```
|
||||
go/tests/fixtures/api-schema/adults.schema.json
|
||||
go/tests/fixtures/api-schema/juniors.schema.json
|
||||
go/tests/fixtures/api-schema/payments.schema.json
|
||||
go/tests/fixtures/api-schema/version.schema.json
|
||||
```
|
||||
|
||||
8. **Adults and Juniors are *not* the same type.** Their outer shapes match (same keys), but `MonthCell.Text` semantics differ (juniors render `"?"`, `"?(3)"`, `:NJ,MA` breakdowns) and `member_data` semantics differ (juniors carry `Expected` sentinel). Define `AdultsResponse` and `JuniorsResponse` separately even if they share most field types — clearer schemas and easier to evolve independently. Share scalar types (`Transaction`, `Exception`, `MonthCell`, `TotalCell`) in `types.go`.
|
||||
|
||||
9. **Money is integer CZK** ([master design:55-56](docs/plans/2026-05-03-2349-go-backend-rewrite.md#L55-L56)). All amount fields use `int`. The one exception: `member_data[name].months[YYYY-MM].paid` is a `float64` in the Python output (proportional allocation produces fractional CZK like `33.333333`). Use `float64` only there; document the why in a one-line comment.
|
||||
|
||||
## Files to create
|
||||
|
||||
```
|
||||
go/internal/web/api/
|
||||
├── types.go # SheetsNumber, Expected, Exception, Transaction, MonthCell, TotalCell, MemberRow
|
||||
├── adults.go # AdultsResponse + adults-specific MemberData
|
||||
├── juniors.go # JuniorsResponse + juniors-specific MemberData (with Expected fields)
|
||||
├── payments.go # PaymentsResponse
|
||||
├── version.go # VersionResponse
|
||||
└── schemagen_test.go # generates + golden-asserts schemas
|
||||
|
||||
go/tests/fixtures/api-schema/
|
||||
├── adults.schema.json
|
||||
├── juniors.schema.json
|
||||
├── payments.schema.json
|
||||
└── version.schema.json
|
||||
```
|
||||
|
||||
## Struct skeleton (illustrative, not final wording)
|
||||
|
||||
```go
|
||||
// internal/web/api/types.go
|
||||
|
||||
// SheetsNumber wraps a value that may arrive from Google Sheets as either
|
||||
// a JSON number or "" (empty string for blank cells). Marshals as 0 when
|
||||
// missing; the consumer treats Missing+Value=0 as "no data".
|
||||
type SheetsNumber struct {
|
||||
Value float64
|
||||
Missing bool
|
||||
}
|
||||
|
||||
// Expected carries a junior's expected fee or the "?" sentinel
|
||||
// (single-attendance month requires manual review).
|
||||
type Expected struct {
|
||||
Value int
|
||||
Unknown bool
|
||||
}
|
||||
|
||||
type Exception struct {
|
||||
Amount int `json:"amount"`
|
||||
Note string `json:"note"`
|
||||
}
|
||||
|
||||
type Transaction struct {
|
||||
Date string `json:"date"`
|
||||
Amount SheetsNumber `json:"amount"`
|
||||
ManualFix string `json:"manual_fix"`
|
||||
Person string `json:"person"`
|
||||
Purpose string `json:"purpose"`
|
||||
InferredAmount SheetsNumber `json:"inferred_amount"`
|
||||
Sender string `json:"sender"`
|
||||
Message string `json:"message"`
|
||||
BankID string `json:"bank_id"`
|
||||
}
|
||||
|
||||
type MonthCell struct {
|
||||
Text string `json:"text"`
|
||||
Overridden bool `json:"overridden"`
|
||||
Status string `json:"status"` // "empty"|"ok"|"partial"|"unpaid"|"surplus"
|
||||
Amount int `json:"amount"`
|
||||
Month string `json:"month"` // display label, e.g. "Apr+May 2025"
|
||||
RawMonth string `json:"raw_month"` // YYYY-MM
|
||||
Tooltip string `json:"tooltip"`
|
||||
}
|
||||
|
||||
type TotalCell struct {
|
||||
Text string `json:"text"`
|
||||
Status string `json:"status"`
|
||||
}
|
||||
|
||||
type MemberRow struct {
|
||||
Name string `json:"name"`
|
||||
Months []MonthCell `json:"months"`
|
||||
Balance int `json:"balance"`
|
||||
UnpaidPeriods string `json:"unpaid_periods"`
|
||||
RawUnpaidPeriods string `json:"raw_unpaid_periods"`
|
||||
PayableAmount int `json:"payable_amount"`
|
||||
}
|
||||
|
||||
type Credit struct {
|
||||
Name string `json:"name"`
|
||||
Amount int `json:"amount"`
|
||||
}
|
||||
```
|
||||
|
||||
```go
|
||||
// internal/web/api/adults.go
|
||||
type AdultsMonthData struct {
|
||||
Status string `json:"status"`
|
||||
Expected int `json:"expected"`
|
||||
Paid float64 `json:"paid"` // float — proportional allocator can produce 33.333…
|
||||
Exception *Exception `json:"exception"`
|
||||
AmountToPay int `json:"amount_to_pay"`
|
||||
// ... transactions, others, etc — match exact keys from result["members"][name]["months"][YYYY-MM]
|
||||
}
|
||||
|
||||
type AdultsMemberData struct {
|
||||
TotalBalance int `json:"total_balance"`
|
||||
Months map[string]AdultsMonthData `json:"months"` // YYYY-MM key
|
||||
Transactions []Transaction `json:"transactions"`
|
||||
}
|
||||
|
||||
type AdultsResponse struct {
|
||||
Months []string `json:"months"`
|
||||
RawMonths []string `json:"raw_months"`
|
||||
Results []MemberRow `json:"results"`
|
||||
Totals []TotalCell `json:"totals"`
|
||||
MemberData map[string]AdultsMemberData `json:"member_data"`
|
||||
MonthLabels map[string]string `json:"month_labels"` // was month_labels_json (string)
|
||||
RawPayments map[string][]Transaction `json:"raw_payments"` // was raw_payments_json (string)
|
||||
Credits []Credit `json:"credits"`
|
||||
Debts []Credit `json:"debts"`
|
||||
Unmatched []Transaction `json:"unmatched"`
|
||||
AttendanceURL string `json:"attendance_url"`
|
||||
PaymentsURL string `json:"payments_url"`
|
||||
BankAccount string `json:"bank_account"`
|
||||
CurrentMonth string `json:"current_month"`
|
||||
}
|
||||
```
|
||||
|
||||
`JuniorsResponse` mirrors `AdultsResponse` but the inner `MonthData` carries `Expected` and `OriginalExpected` (both `Expected` type), and adds the `:NJ,MA` breakdown fields produced by [scripts/views.py:290-298](scripts/views.py#L290-L298).
|
||||
|
||||
`PaymentsResponse`:
|
||||
|
||||
```go
|
||||
type PaymentsResponse struct {
|
||||
GroupedPayments map[string][]Transaction `json:"grouped_payments"`
|
||||
SortedPeople []string `json:"sorted_people"`
|
||||
AttendanceURL string `json:"attendance_url"`
|
||||
PaymentsURL string `json:"payments_url"`
|
||||
}
|
||||
```
|
||||
|
||||
`VersionResponse` mirrors Python's `BUILD_META` ([app.py:67](app.py#L67)):
|
||||
|
||||
```go
|
||||
type VersionResponse struct {
|
||||
Tag string `json:"tag"`
|
||||
Commit string `json:"commit"`
|
||||
BuildDate string `json:"build_date"`
|
||||
}
|
||||
```
|
||||
|
||||
Exact fields inside the per-month `MonthData`/`MemberData` will be finalized by inspecting **one** scrubbed `member_data` JSON dump from a current `/adults` and `/juniors` call (see Verification step 1) — names and types have to be identical.
|
||||
|
||||
## Reusable existing code
|
||||
|
||||
- `domain/reconcile.Result` ([go/internal/domain/reconcile/reconcile.go:88](go/internal/domain/reconcile/reconcile.go#L88)) — source data for adults/juniors. M5.2 maps it to wire types; M5.1 only needs to know the field set.
|
||||
- `web.BuildInfo` ([go/internal/web/server.go:11-15](go/internal/web/server.go#L11-L15)) — already wired through `cmd/fuj/main.go:79`. The `VersionResponse` type is a thin renaming of this with json tags. Decide in M5.2 whether to *also* tag `BuildInfo` directly (it's not really domain) or keep `VersionResponse` separate. M5.1 just defines the wire struct.
|
||||
- `scripts/views.py` builders — the spec for every key/type. Treat as authoritative.
|
||||
|
||||
## Tasks
|
||||
|
||||
1. **Add `github.com/invopop/jsonschema` dependency.** `cd go && go get github.com/invopop/jsonschema && go mod tidy`. Single commit.
|
||||
2. **Capture one fresh `member_data` dump for adults and juniors** to pin down inner-dict field names and types precisely (especially `paid` precision, `transactions[]` shape, `others[]` if present). Run `make web-py` on a known-good cache, hit `/adults` and `/juniors`, dump `view_model["member_data"]` to a scratch file (gitignored), and inspect. **Do not commit raw dumps** — PII rule. This is for shape inspection only.
|
||||
3. **Author `internal/web/api/types.go`** with `SheetsNumber`, `Expected`, `Exception`, `Transaction`, `MonthCell`, `TotalCell`, `MemberRow`, `Credit`. Implement `MarshalJSON`/`UnmarshalJSON` for `SheetsNumber` and `Expected`. Use struct tags `json:"snake_case_key"` matching Python exactly.
|
||||
4. **Author `internal/web/api/adults.go`** with `AdultsMonthData`, `AdultsMemberData`, `AdultsResponse`. Cross-check every key against the dump from step 2.
|
||||
5. **Author `internal/web/api/juniors.go`** similarly, with `Expected` fields and the J/A breakdown.
|
||||
6. **Author `internal/web/api/payments.go` and `version.go`** (small).
|
||||
7. **Author `internal/web/api/schemagen_test.go`** that:
|
||||
- Imports `github.com/invopop/jsonschema`.
|
||||
- Defines a `schemaCases` slice: `{name: "adults", typ: AdultsResponse{}, ...}` etc.
|
||||
- For each case, generates a schema, marshals indented JSON, compares against committed file at `../../tests/fixtures/api-schema/<name>.schema.json`.
|
||||
- Honors a `-update` flag (`flag.Bool`) that rewrites the committed file instead of asserting.
|
||||
- One `t.Run(case.name, ...)` per route for clear failure output.
|
||||
8. **Run with `-update` once to populate** `tests/fixtures/api-schema/*.schema.json`. Eyeball each schema: required fields list, oneOf for `Expected`, additionalProperties on map types. Commit.
|
||||
9. **Lint + test:** `cd go && go vet ./... && make go-test && make go-lint`. Fix any issues. (Expect zero — this is read-only data structures.)
|
||||
10. **CHANGELOG entry** + tick `M5.1` in [docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md:100](docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md#L100) with the merge SHA.
|
||||
11. **Branch + MR per CLAUDE.md workflow:** branch `feat/go-m5-1-api-structs`, push with `-u`, open MR via `tea pr create`. Do not merge from CLI.
|
||||
|
||||
## Verification
|
||||
|
||||
1. `cd go && go test ./internal/web/api/...` — **schemas regenerate identically** to committed files.
|
||||
2. `cd go && go test ./internal/web/api/... -update` then `git diff go/tests/fixtures/api-schema/` — diff is empty (idempotent generation).
|
||||
3. `cd go && make go-lint` — clean.
|
||||
4. `cd go && go vet ./...` — clean.
|
||||
5. Manual schema inspection: open `adults.schema.json`, confirm:
|
||||
- Top-level `required` list contains every Python key.
|
||||
- `member_data` is `additionalProperties: { ... AdultsMemberData ... }` (a map keyed by name).
|
||||
- `expected` and `original_expected` (juniors only) are `oneOf: [{type: integer}, {const: "?"}]`.
|
||||
- `amount` and `inferred_amount` on `Transaction` accept number or empty/null.
|
||||
6. **No production code paths exercised yet** — handlers come in M5.2. Compile-time success + schema golden-test = M5.1 done.
|
||||
|
||||
## Out of scope (later M5 tasks)
|
||||
|
||||
- Wiring Go HTTP handlers for `/api/X` (M5.2).
|
||||
- Adding Python `/api/X` shadow endpoints (M5.3) — including the `unwrap_json_strings(view_model)` shim noted in design decision #1.
|
||||
- `cmd/parity/main.go` and `make parity` target (M5.4).
|
||||
- Tagging `domain/reconcile.Result` with `json:` tags — explicitly avoided.
|
||||
- Refactoring the strings-of-JSON fields out of the Python view-model — they stay in `views.py` for the template path, the `/api/X` shadow unwraps them.
|
||||
113
docs/plans/2026-05-07-2114-go-rewrite-m5-3-python-api-shadow.md
Normal file
113
docs/plans/2026-05-07-2114-go-rewrite-m5-3-python-api-shadow.md
Normal file
@@ -0,0 +1,113 @@
|
||||
# M5.3 — Python `/api/X` shadow endpoints
|
||||
|
||||
Companion to:
|
||||
- [2026-05-03-2349-go-backend-rewrite.md](2026-05-03-2349-go-backend-rewrite.md) (master design)
|
||||
- [2026-05-03-2349-go-backend-rewrite-progress.md](2026-05-03-2349-go-backend-rewrite-progress.md) (M5.3 row)
|
||||
- [2026-05-07-1431-m5-json-api-parity.md](2026-05-07-1431-m5-json-api-parity.md) (Python view-model extraction prep)
|
||||
- [2026-05-07-1650-go-rewrite-m5-1-api-structs-schemas.md](2026-05-07-1650-go-rewrite-m5-1-api-structs-schemas.md) (Go wire types)
|
||||
|
||||
## Context
|
||||
|
||||
M5.1 (Go wire types + JSON Schemas) and M5.2 (Go HTTP handlers for `/api/adults` `/api/juniors` `/api/payments` `/api/version`) have merged. M5.3 mirrors the same four endpoints on the Python Flask side so M5.4's `cmd/parity` tool can hit both backends and diff the JSON. After M5.3, every byte the Go side emits has a Python counterpart to compare against.
|
||||
|
||||
The Python view-model builders ([scripts/views.py](scripts/views.py)) already produce dicts very close to the wire shape — except three template-only fields (`member_data`, `month_labels_json`, `raw_payments_json`) are pre-`json.dumps`'d for inline `<script>` blocks. M5.1's plan called this out explicitly: M5.3's `/api/X` is `jsonify(unwrap_json_strings(view_model_dict))` — a 4-line shim, not real transformation logic.
|
||||
|
||||
## Approach
|
||||
|
||||
Add four shadow routes to [app.py](app.py) and one private unwrap helper. Builders and templates are untouched.
|
||||
|
||||
### Decisions
|
||||
|
||||
1. **Unwrap shim lives in `app.py`**, not `scripts/views.py`. It's 4 lines, only the API routes use it, and it's parity-only scaffolding that M8 will delete. Keeping `views.py` free of HTTP-layer concerns means cleaner deletion later.
|
||||
2. **No data-loading helper extraction.** Each shadow route duplicates ~8 lines of cache loads from its sibling HTML route. A helper would have to thread `attendance_url` / `payments_url` / `bank_account` and the adults-vs-juniors-vs-payments branching back out — net negative for code that M8 will erase wholesale.
|
||||
3. **Drop `record_step` calls in API routes.** `record_step` only feeds `inject_render_time` (a Jinja `context_processor`); JSON responses don't go through templates, so timing breakdown has no consumer.
|
||||
4. **`/api/version` is a one-liner.** `BUILD_META` already has the keys (`tag`, `commit`, `build_date`) Go emits. Just `jsonify(BUILD_META)`. Existing `/version` route stays as-is — the new endpoint sits alongside.
|
||||
5. **Tests assert key sets and unwrap, not values.** Hard-code `EXPECTED_ADULTS_KEYS` etc. as module constants in [tests/test_app.py](tests/test_app.py). Catches drift in unit tests rather than waiting for M5.4 parity diffs. Type-check the unwrapped fields (`isinstance(member_data, dict)` etc.) to prove the shim ran.
|
||||
|
||||
### The shim
|
||||
|
||||
```python
|
||||
def _unwrap_view_model_for_api(vm: dict) -> dict:
|
||||
out = dict(vm)
|
||||
out["member_data"] = _json.loads(out.pop("member_data"))
|
||||
out["month_labels"] = _json.loads(out.pop("month_labels_json"))
|
||||
out["raw_payments"] = _json.loads(out.pop("raw_payments_json"))
|
||||
return out
|
||||
```
|
||||
|
||||
Note the **rename**: `month_labels_json` → `month_labels`, `raw_payments_json` → `raw_payments` (matches Go contract per [adults.go](go/internal/web/api/adults.go) JSON tags).
|
||||
|
||||
### Route skeleton (`/api/adults`)
|
||||
|
||||
```python
|
||||
@app.route("/api/adults")
|
||||
def api_adults():
|
||||
attendance_url = f"https://docs.google.com/spreadsheets/d/{ATTENDANCE_SHEET_ID}/edit"
|
||||
payments_url = f"https://docs.google.com/spreadsheets/d/{PAYMENTS_SHEET_ID}/edit"
|
||||
members_data = get_cached_data("attendance_regular", ATTENDANCE_SHEET_ID, get_members_with_fees)
|
||||
if not members_data:
|
||||
return jsonify({"error": "no data"}), 503
|
||||
members, sorted_months = members_data
|
||||
transactions = get_cached_data("payments_transactions", PAYMENTS_SHEET_ID, fetch_sheet_data, PAYMENTS_SHEET_ID, CREDENTIALS_PATH)
|
||||
exceptions = get_cached_data(
|
||||
"exceptions_dict", PAYMENTS_SHEET_ID, fetch_exceptions,
|
||||
PAYMENTS_SHEET_ID, CREDENTIALS_PATH,
|
||||
serialize=lambda d: [[list(k), v] for k, v in d.items()],
|
||||
deserialize=lambda c: {tuple(k): v for k, v in c},
|
||||
)
|
||||
result = reconcile(members, sorted_months, transactions, exceptions)
|
||||
vm = build_adults_view_model(
|
||||
members, sorted_months, result, transactions,
|
||||
datetime.now().strftime("%Y-%m"),
|
||||
attendance_url=attendance_url, payments_url=payments_url, bank_account=BANK_ACCOUNT,
|
||||
)
|
||||
return jsonify(_unwrap_view_model_for_api(vm))
|
||||
```
|
||||
|
||||
`/api/juniors` mirrors with `adapt_junior_members` + `JUNIOR_SHEET_GID` + `build_juniors_view_model`. `/api/payments` skips the unwrap (its builder has no JSON-string fields): `return jsonify(vm)`. `/api/version`: `return jsonify(BUILD_META)`.
|
||||
|
||||
## Files to modify
|
||||
|
||||
- [app.py](app.py)
|
||||
- L10: add `jsonify` to `from flask import ...`.
|
||||
- Add `_unwrap_view_model_for_api` near [BUILD_META](app.py#L67) (it already imports `json as _json`).
|
||||
- Add four routes: `/api/version`, `/api/adults`, `/api/juniors`, `/api/payments`. Place them after [`/version`](app.py#L143) for grouping.
|
||||
- [tests/test_app.py](tests/test_app.py)
|
||||
- Add `EXPECTED_ADULTS_KEYS`, `EXPECTED_JUNIORS_KEYS`, `EXPECTED_PAYMENTS_KEYS`, `EXPECTED_VERSION_KEYS` module constants (sourced from [adults.go](go/internal/web/api/adults.go) / [juniors.go](go/internal/web/api/juniors.go) / [payments.go](go/internal/web/api/payments.go) / [version.go](go/internal/web/api/version.go) JSON tags).
|
||||
- Four new test functions: `test_api_adults`, `test_api_juniors`, `test_api_payments`, `test_api_version`. Reuse the existing `_bypass_cache` patcher and the `fetch_sheet_data` / `fetch_exceptions` / `get_members_with_fees` / `get_junior_members_with_fees` mocks already in the file.
|
||||
- Each adults/juniors test asserts: `200`, `response.is_json`, `set(json.keys()) == EXPECTED_*_KEYS`, `isinstance(json["member_data"], dict)`, `isinstance(json["month_labels"], dict)`, `isinstance(json["raw_payments"], dict)`.
|
||||
- [CHANGELOG.md](CHANGELOG.md): post-merge entry per CLAUDE.md format.
|
||||
- [docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md:102](docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md#L102): tick M5.3 with merge SHA.
|
||||
|
||||
## Reusable existing code
|
||||
|
||||
- [`build_adults_view_model`](scripts/views.py#L64), [`build_juniors_view_model`](scripts/views.py#L240), [`build_payments_view_model`](scripts/views.py#L432) — call as-is, no changes.
|
||||
- [`reconcile`](scripts/match_payments.py) and [`adapt_junior_members`](scripts/views.py#L48) — same.
|
||||
- [`get_cached_data`](app.py#L36), [`fetch_sheet_data`](scripts/match_payments.py), [`fetch_exceptions`](scripts/match_payments.py), [`get_members_with_fees`](scripts/attendance.py), [`get_junior_members_with_fees`](scripts/attendance.py) — same call sites as the HTML routes.
|
||||
- [`BUILD_META`](app.py#L67) — already shaped to match Go's `VersionResponse`.
|
||||
|
||||
## Verification
|
||||
|
||||
1. `make test` — all existing tests still pass; four new `test_api_*` tests pass.
|
||||
2. `python -c "import app"` — catches the missing `jsonify` import.
|
||||
3. `make web-py`, then:
|
||||
- `curl -s localhost:5001/api/version | jq .` → `{tag, commit, build_date}`.
|
||||
- `curl -s localhost:5001/api/adults | jq 'keys'` → 14 keys, no `_json` suffix anywhere.
|
||||
- `curl -s localhost:5001/api/adults | jq '.member_data | type, .month_labels | type, .raw_payments | type'` → all `"object"` (proves unwrap).
|
||||
- Same checks on `/api/juniors` and `/api/payments`.
|
||||
4. Visit `/adults`, `/juniors`, `/payments` in browser — HTML still renders identically (regression check on builders).
|
||||
5. **Optional pre-M5.4 peek:** `make web-go` on :8080 + `make web-py` on :5001, then `diff <(curl -s :5001/api/adults | jq -S .) <(curl -s :8080/api/adults | jq -S .)`. Expect non-zero diff (raw transaction key shape, see below) — that is fine; M5.4 surfaces and resolves these.
|
||||
|
||||
## Out of scope (M5.4 will surface and resolve)
|
||||
|
||||
These are known parity friction points; **don't fix in M5.3** — the whole point of M5.4 is to enumerate them:
|
||||
|
||||
- **`raw_payments[name][i]` row shape**: Python emits raw Google Sheets row dicts with column-header keys (e.g. `"VS"`, `"Sync ID"`); Go's `RawTransaction` uses snake_case (`vs`, `sync_id`). Keys and types will diverge.
|
||||
- **`unmatched[]`**: same divergence (same raw row dicts).
|
||||
- **`null` vs missing keys**: Python omits keys never set; Go zero-value structs may emit `null`/`""` depending on `omitempty`.
|
||||
- **`Decimal` / float precision** in `grouped_payments` amounts.
|
||||
- **Field insertion order**: `jsonify` preserves insertion order (Python 3.7+); Go marshals struct fields in declaration order. Likely fine, parity tool will tell.
|
||||
|
||||
## Branch + MR
|
||||
|
||||
Per CLAUDE.md: branch `feat/go-m5-3-python-api-shadow`, push with `-u`, open MR via `tea pr create --title ... --description ... --base main --head feat/go-m5-3-python-api-shadow`. Do not merge from CLI.
|
||||
152
docs/plans/2026-05-07-2254-m5-4-parity-binary.md
Normal file
152
docs/plans/2026-05-07-2254-m5-4-parity-binary.md
Normal file
@@ -0,0 +1,152 @@
|
||||
# M5.4 — Parity diff binary (`cmd/parity`) + `make parity`
|
||||
|
||||
## Context
|
||||
|
||||
Per [docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md:103](docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md#L103), M5.4 is the next milestone in the Go rewrite. M5.1–M5.3 already landed:
|
||||
|
||||
- M5.1: hand-authored Go response structs at [go/internal/web/api/](go/internal/web/api/) + JSON Schemas in [go/tests/fixtures/api-schema/](go/tests/fixtures/api-schema/).
|
||||
- M5.2: Go `/api/version|adults|juniors|payments` handlers in [go/internal/web/api/handler.go](go/internal/web/api/handler.go).
|
||||
- M5.3: Python shadow endpoints in [app.py:157-224](app.py#L157-L224) using `_unwrap_view_model_for_api` for the same JSON shape.
|
||||
|
||||
What's missing: a tool that proves the two backends actually agree on the wire. The M5 gate says "byte-equal JSON between Python and Go for every route." Without a diffing tool, drift between the two implementations slips in silently and we can't gate further milestones (M6 frontend, M7 watch period) on parity.
|
||||
|
||||
This task delivers the **parity contract enforcer**: a Go binary that fetches `/api/*` from both backends, scrubs an allowlist of expected diffs, and prints `cmp.Diff` for everything else. Both backends read the same live Google Sheets — that shared state is what makes parity meaningful, no scenario fixtures needed.
|
||||
|
||||
## Scope
|
||||
|
||||
In scope:
|
||||
- New binary `go/cmd/parity/main.go` plus a small support package for the allowlist scrubber.
|
||||
- A unit test for the scrubber.
|
||||
- New `make parity` target.
|
||||
- `go-cmp` promoted to a direct dependency.
|
||||
|
||||
Out of scope (explicitly):
|
||||
- CI integration / nightly job — that's M7.2.
|
||||
- Fixture-driven offline parity — pure-fn parity already runs via `make go-parity` (M3).
|
||||
- Hooking `make parity` into `make go-test-all` — leave it manual since it requires two live servers.
|
||||
|
||||
## Approach
|
||||
|
||||
### 1. New binary: `go/cmd/parity/main.go`
|
||||
|
||||
Standalone binary (mirrors task spec — not a subcommand of `fuj`). Stdlib `flag`, no third-party CLI lib.
|
||||
|
||||
**Flags:**
|
||||
- `-py` — Python base URL, default `http://localhost:5001`
|
||||
- `-go` — Go base URL, default `http://localhost:8080`
|
||||
- `-route` — optional single route to diff (e.g. `/api/adults`); empty means iterate all four
|
||||
- `-timeout` — per-request timeout, default `30s` (sheet fetches are slow on cold cache)
|
||||
|
||||
**Routes**, hard-coded:
|
||||
```go
|
||||
var routes = []string{"/api/version", "/api/adults", "/api/juniors", "/api/payments"}
|
||||
```
|
||||
All `GET`, no query params (verified in [app.py:157-224](app.py#L157-L224) and [go/internal/web/api/handler.go:27-68](go/internal/web/api/handler.go#L27-L68)).
|
||||
|
||||
**Per route, the binary:**
|
||||
1. `GET py+route` and `GET go+route` (sequential — keep it simple; total wall time ~1–2s on warm cache).
|
||||
2. Verify both return HTTP 200; on non-200, print body and mark route as ERROR.
|
||||
3. Decode each body into `map[string]any` (NOT into `api.AdultsResponse` — using the typed struct would silently drop unknown Python-side keys, defeating the diff. Map gives `cmp.Diff` clean dotted-path field names for free.).
|
||||
4. Run `scrub(m, allowlist)` on both decoded maps.
|
||||
5. `diff := cmp.Diff(pyMap, goMap, cmpopts.EquateEmpty())`.
|
||||
6. If `diff != ""`, print `=== /api/X ===` header followed by the diff; track for exit code.
|
||||
|
||||
**Exit codes:**
|
||||
- `0` — all routes match (or the only diffs were under the allowlist).
|
||||
- `1` — at least one route had a non-allowlisted diff.
|
||||
- `2` — at least one route failed to fetch / parse (HTTP error, timeout, non-JSON body).
|
||||
|
||||
This makes the binary CI-friendly when M7.2 lands.
|
||||
|
||||
**Output**: human-readable to stdout — header per route, then "OK" or the diff. Final summary line: `parity: 4/4 routes match` or `parity: 2/4 match, 1 diff, 1 error`.
|
||||
|
||||
### 2. Allowlist scrubber
|
||||
|
||||
Live in same package (`main`). Keep it tiny — no need for a separate sub-package.
|
||||
|
||||
```go
|
||||
var defaultAllowlist = []string{"render_time.total", "build_meta"}
|
||||
|
||||
// scrub walks m and deletes any key whose dotted path matches an allowlist entry.
|
||||
// "render_time.total" deletes m["render_time"]["total"] only (preserves render_time.breakdown).
|
||||
// "build_meta" (no dot) deletes m["build_meta"] entirely.
|
||||
func scrub(m map[string]any, paths []string) { ... }
|
||||
```
|
||||
|
||||
Today these fields **don't appear in the JSON** — they're Jinja template-only ([app.py:91-110](app.py#L91-L110)) and the Go side only logs render time via [go/internal/web/middleware/timer.go](go/internal/web/middleware/timer.go). So today the scrub is a no-op. The implementation is forward-compatible insurance: if someone later adds either field to a JSON response on one side only, the parity binary already tolerates it.
|
||||
|
||||
The implementation note in [go/internal/web/middleware/timer.go](go/internal/web/middleware/timer.go) ("the elapsed value maps to render_time.total in the M5 JSON allowlist") confirms this is the intended design.
|
||||
|
||||
### 3. Unit test
|
||||
|
||||
Add `go/cmd/parity/scrub_test.go` with a couple of cases:
|
||||
- Scrubbing top-level key (`build_meta`) — key removed; siblings untouched.
|
||||
- Scrubbing nested key (`render_time.total`) — only `total` removed; `breakdown` preserved.
|
||||
- Path that doesn't exist — no-op, no error.
|
||||
- Map without the parent key — no-op (don't panic).
|
||||
|
||||
Runs as part of normal `make go-test` (no `-tags` needed).
|
||||
|
||||
### 4. Makefile target
|
||||
|
||||
Add to [Makefile](Makefile):
|
||||
|
||||
```make
|
||||
parity:
|
||||
cd $(GO_SRC) && go run ./cmd/parity $(ARGS)
|
||||
```
|
||||
|
||||
Add `parity` to the `.PHONY` line at [Makefile:1](Makefile#L1) and a `help` entry like:
|
||||
|
||||
```
|
||||
@echo " make parity - Diff /api/* between web-py (:5001) and web-go (:8080); both must be running"
|
||||
```
|
||||
|
||||
Don't depend on `go-build` — `go run` compiles ad-hoc, and parity is interactive enough that the slight rebuild cost doesn't matter.
|
||||
|
||||
### 5. Dependency
|
||||
|
||||
`github.com/google/go-cmp v0.7.0` is already a transitive dep ([go/go.sum:24-25](go/go.sum#L24-L25)) but not in `go.mod`'s `require` block. After adding the import, run `go mod tidy` inside `go/` to promote it to direct. No new external deps.
|
||||
|
||||
## Files to add / modify
|
||||
|
||||
**Add:**
|
||||
- [go/cmd/parity/main.go](go/cmd/parity/main.go) — flags, route loop, fetch+scrub+diff, exit codes
|
||||
- [go/cmd/parity/scrub_test.go](go/cmd/parity/scrub_test.go) — unit tests for the scrubber
|
||||
|
||||
**Modify:**
|
||||
- [Makefile](Makefile) — `.PHONY`, help block, `parity` target
|
||||
- [go/go.mod](go/go.mod) — promote `go-cmp` to direct (via `go mod tidy`)
|
||||
- [go/go.sum](go/go.sum) — likely unchanged (already pinned)
|
||||
- [docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md](docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md) — tick M5.4, add commit SHA
|
||||
- [CHANGELOG.md](CHANGELOG.md) — entry per project convention
|
||||
|
||||
**Per [CLAUDE.md](CLAUDE.md) plans convention:** copy this plan to `docs/plans/2026-05-07-HHMM-m5-4-parity-binary.md` (timestamp via `date "+%Y-%m-%d-%H%M"`) before opening the MR, since plan files are committed for posterity.
|
||||
|
||||
## Existing utilities to reuse
|
||||
|
||||
- `cmp.Diff` + `cmpopts.EquateEmpty()` from `github.com/google/go-cmp/cmp` (already in go.sum).
|
||||
- Stdlib `net/http`, `encoding/json`, `flag` — sufficient; no need to pull a CLI framework.
|
||||
- No existing scrubber to reuse — the one in [scripts/scrub_fixtures.py](scripts/scrub_fixtures.py) operates on the *capture* path and renames PII; the parity scrubber is a different concern (path-based deletion of expected diffs).
|
||||
|
||||
## Verification
|
||||
|
||||
Manual smoke test (the binary is meant to be run against live servers):
|
||||
|
||||
1. **Sanity / build:** `cd go && go build ./cmd/parity && go test ./cmd/parity/...` — compiles + scrubber unit test passes.
|
||||
2. **Both backends up:**
|
||||
- Terminal A: `make web-py`
|
||||
- Terminal B: `make web-go`
|
||||
- Terminal C: `make parity`
|
||||
Expected: `parity: 4/4 routes match`, exit 0.
|
||||
3. **Negative test:** add a literal extra field to one Go handler temporarily (e.g. `"diagnostic": "test"` in `VersionResponse`), rebuild, re-run `make parity`. Expected: non-zero exit, diff shown for `/api/version`. Revert.
|
||||
4. **Allowlist test:** in a unit test (or by manually constructing a payload with `render_time.total` injected), confirm the scrubber removes it before the diff stage.
|
||||
5. **CHANGELOG** entry added; progress tracker ticked with the merge commit SHA.
|
||||
|
||||
Per [CLAUDE.md](CLAUDE.md) branching policy: this is a feature, so work happens on `feat/go-m5-4-parity-binary`, push with `-u`, open MR with `tea pr create --base main --head feat/go-m5-4-parity-binary`. User merges in Gitea.
|
||||
|
||||
## Notes & risks
|
||||
|
||||
- **Cache-warmth dependency:** Both backends cache aggressively. A cold-cache fetch on Python can take 30s+ — hence the configurable timeout. If parity is run immediately after `flush-cache`, expect the first run to be slow.
|
||||
- **Order-sensitive lists:** `cmp.Diff` on `map[string]any` treats slices as ordered. If either backend ever returns members/transactions in a different order, the diff will flag it — that's a real parity bug, not a false positive, so this is correct behavior. If we hit ordering instability later, fix the source, don't add `cmpopts.SortSlices`.
|
||||
- **Float formatting:** Both sides go through `encoding/json` (Go) and `jsonify` (Python `json.dumps`). Floats in totals/balances may format differently (`123` vs `123.0`). If this surfaces, the Go side already uses typed structs with `int` where appropriate — investigate at the source rather than allowlisting.
|
||||
11
go/.golangci.yml
Normal file
11
go/.golangci.yml
Normal file
@@ -0,0 +1,11 @@
|
||||
linters:
|
||||
enable:
|
||||
- govet
|
||||
- staticcheck
|
||||
- errcheck
|
||||
- gofumpt
|
||||
- unused
|
||||
|
||||
linters-settings:
|
||||
gofumpt:
|
||||
extra-rules: true
|
||||
30
go/build/Dockerfile
Normal file
30
go/build/Dockerfile
Normal file
@@ -0,0 +1,30 @@
|
||||
FROM golang:1.26 AS build
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
COPY . .
|
||||
|
||||
ARG GIT_TAG=unknown
|
||||
ARG GIT_COMMIT=unknown
|
||||
ARG BUILD_DATE=unknown
|
||||
|
||||
RUN CGO_ENABLED=0 go build -trimpath \
|
||||
-ldflags "-s -w \
|
||||
-X main.version=${GIT_TAG} \
|
||||
-X main.commit=${GIT_COMMIT} \
|
||||
-X main.buildDate=${BUILD_DATE}" \
|
||||
-o /out/fuj ./cmd/fuj
|
||||
|
||||
FROM alpine:3
|
||||
|
||||
RUN addgroup -S fuj && adduser -S fuj -G fuj
|
||||
|
||||
COPY --from=build /out/fuj /usr/local/bin/fuj
|
||||
|
||||
EXPOSE 8080
|
||||
USER fuj
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/fuj", "server"]
|
||||
244
go/cmd/fuj/main.go
Normal file
244
go/cmd/fuj/main.go
Normal file
@@ -0,0 +1,244 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"fuj-management/go/internal/config"
|
||||
"fuj-management/go/internal/io/fio"
|
||||
"fuj-management/go/internal/io/sheets"
|
||||
"fuj-management/go/internal/logging"
|
||||
"fuj-management/go/internal/services/banksync"
|
||||
"fuj-management/go/internal/services/membership"
|
||||
"fuj-management/go/internal/web"
|
||||
"log/slog"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Injected at build time via -ldflags "-X main.version=... -X main.commit=... -X main.buildDate=..."
|
||||
var (
|
||||
version = "dev"
|
||||
commit = "unknown"
|
||||
buildDate = "unknown"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 2 {
|
||||
usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
// Honour LOG_LEVEL for slog calls in any package (e.g. internal/io/fio debug logs).
|
||||
slog.SetDefault(logging.New(os.Getenv("LOG_LEVEL")))
|
||||
|
||||
cmd, args := os.Args[1], os.Args[2:]
|
||||
|
||||
switch cmd {
|
||||
case "server":
|
||||
serverCmd(args)
|
||||
case "version":
|
||||
versionCmd()
|
||||
case "fees":
|
||||
feesCmd(args)
|
||||
case "reconcile":
|
||||
reconcileCmd(args)
|
||||
case "sync":
|
||||
syncCmd(args)
|
||||
case "infer":
|
||||
inferCmd(args)
|
||||
case "-h", "--help", "help":
|
||||
usage()
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "fuj: unknown command %q\n\n", cmd)
|
||||
usage()
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
|
||||
func serverCmd(args []string) {
|
||||
fs := flag.NewFlagSet("server", flag.ExitOnError)
|
||||
addr := fs.String("addr", "", "listen address (default from SERVER_ADDR env or :8080)")
|
||||
fs.Usage = func() {
|
||||
fmt.Fprintln(os.Stderr, "usage: fuj server [--addr :8080]")
|
||||
fs.PrintDefaults()
|
||||
}
|
||||
if err := fs.Parse(args); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
cfg := config.Load()
|
||||
if *addr != "" {
|
||||
cfg.ServerAddr = *addr
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
logger := logging.New(cfg.LogLevel)
|
||||
|
||||
sources, err := membership.NewSources(ctx, cfg)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "fuj server: init sources: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
build := web.BuildInfo{Version: version, Commit: commit, BuildDate: buildDate}
|
||||
|
||||
if err := web.Run(logger, cfg.ServerAddr, build, sources, cfg); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func feesCmd(args []string) {
|
||||
fs := flag.NewFlagSet("fees", flag.ExitOnError)
|
||||
fs.Usage = func() {
|
||||
fmt.Fprintln(os.Stderr, "usage: fuj fees")
|
||||
}
|
||||
if err := fs.Parse(args); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
cfg := config.Load()
|
||||
sources, err := membership.NewSources(ctx, cfg)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "fuj fees: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := membership.FeesReport(ctx, sources, os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "fuj fees: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func reconcileCmd(args []string) {
|
||||
fs := flag.NewFlagSet("reconcile", flag.ExitOnError)
|
||||
fs.Usage = func() {
|
||||
fmt.Fprintln(os.Stderr, "usage: fuj reconcile")
|
||||
}
|
||||
if err := fs.Parse(args); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
cfg := config.Load()
|
||||
sources, err := membership.NewSources(ctx, cfg)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "fuj reconcile: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := membership.ReconcileReport(ctx, sources, time.Now().Year(), os.Stdout); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "fuj reconcile: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func versionCmd() {
|
||||
fmt.Printf("fuj %s (%s) built %s\n", version, commit, buildDate)
|
||||
}
|
||||
|
||||
func syncCmd(args []string) {
|
||||
fs := flag.NewFlagSet("sync", flag.ExitOnError)
|
||||
days := fs.Int("days", 30, "look-back window in days (ignored when --from/--to are set)")
|
||||
fromStr := fs.String("from", "", "start date YYYY-MM-DD")
|
||||
toStr := fs.String("to", "", "end date YYYY-MM-DD")
|
||||
sort := fs.Bool("sort", true, "sort sheet by date after appending")
|
||||
dryRun := fs.Bool("dry-run", false, "print planned writes without modifying the sheet")
|
||||
printFioTable := fs.Bool("print-fio-table", false, "with --dry-run: print aligned table of every Fio transaction with NEW/DUP status")
|
||||
fs.Usage = func() {
|
||||
fmt.Fprintln(os.Stderr, "usage: fuj sync [--days N] [--from YYYY-MM-DD --to YYYY-MM-DD] [--sort] [--dry-run] [--print-fio-table]")
|
||||
fs.PrintDefaults()
|
||||
}
|
||||
if err := fs.Parse(args); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
cfg := config.Load()
|
||||
|
||||
sheetsCli, err := sheets.New(ctx, cfg.CredentialsPath, cfg.DriveTimeout)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "fuj sync: sheets client: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
fioCli := fio.New(cfg.FioAPIToken, config.IBANAccountNum(cfg.BankAccount), nil)
|
||||
|
||||
opts := banksync.SyncOpts{Days: *days, Sort: *sort, DryRun: *dryRun, PrintFioTable: *printFioTable}
|
||||
if *fromStr != "" && *toStr != "" {
|
||||
opts.From, err = time.Parse("2006-01-02", *fromStr)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "fuj sync: invalid --from: %v\n", err)
|
||||
os.Exit(2)
|
||||
}
|
||||
opts.To, err = time.Parse("2006-01-02", *toStr)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "fuj sync: invalid --to: %v\n", err)
|
||||
os.Exit(2)
|
||||
}
|
||||
}
|
||||
|
||||
n, err := banksync.SyncToSheets(ctx, config.PaymentsSheetID, fioCli, sheetsCli, opts)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "fuj sync: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if *dryRun {
|
||||
fmt.Printf("Dry run: would sync %d new transaction(s).\n", n)
|
||||
} else {
|
||||
fmt.Printf("Synced %d new transaction(s).\n", n)
|
||||
}
|
||||
}
|
||||
|
||||
func inferCmd(args []string) {
|
||||
fs := flag.NewFlagSet("infer", flag.ExitOnError)
|
||||
dryRun := fs.Bool("dry-run", false, "print planned updates without writing to the sheet")
|
||||
fs.Usage = func() {
|
||||
fmt.Fprintln(os.Stderr, "usage: fuj infer [--dry-run]")
|
||||
fs.PrintDefaults()
|
||||
}
|
||||
if err := fs.Parse(args); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
cfg := config.Load()
|
||||
|
||||
sheetsCli, err := sheets.New(ctx, cfg.CredentialsPath, cfg.DriveTimeout)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "fuj infer: sheets client: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
sources, err := membership.NewSources(ctx, cfg)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "fuj infer: sources: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
n, err := banksync.InferPayments(ctx, config.PaymentsSheetID, sheetsCli, sources, banksync.InferOpts{DryRun: *dryRun})
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "fuj infer: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if *dryRun {
|
||||
fmt.Printf("Dry run: would update %d row(s).\n", n)
|
||||
} else {
|
||||
fmt.Printf("Updated %d row(s).\n", n)
|
||||
}
|
||||
}
|
||||
|
||||
func usage() {
|
||||
fmt.Fprintln(os.Stderr, `usage: fuj <command> [flags]
|
||||
|
||||
Commands:
|
||||
server Start HTTP server (default :8080)
|
||||
version Print version information
|
||||
fees Calculate monthly fees
|
||||
reconcile Show balance report
|
||||
sync Sync Fio transactions to payments sheet
|
||||
infer Infer payment details in payments sheet`)
|
||||
}
|
||||
133
go/cmd/parity/main.go
Normal file
133
go/cmd/parity/main.go
Normal file
@@ -0,0 +1,133 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
)
|
||||
|
||||
// /api/version is intentionally excluded — it returns each binary's own build
|
||||
// identity (tag/commit/build_date), which differs by design between independently
|
||||
// built backends. Pass -route /api/version to inspect it manually.
|
||||
var allRoutes = []string{"/api/adults", "/api/juniors", "/api/payments"}
|
||||
|
||||
// defaultAllowlist holds dotted key paths to strip before diffing.
|
||||
// render_time.total is forward-compatible insurance: today it lives in the Jinja
|
||||
// template context only (app.py inject_render_time) and is logged via
|
||||
// middleware/timer.go on the Go side, so it isn't in any JSON response. If either
|
||||
// side ever surfaces it under a render_time envelope, the scrubber handles it.
|
||||
var defaultAllowlist = []string{"render_time.total"}
|
||||
|
||||
func main() {
|
||||
pyURL := flag.String("py", "http://localhost:5001", "Python backend base URL")
|
||||
goURL := flag.String("go", "http://localhost:8080", "Go backend base URL")
|
||||
route := flag.String("route", "", "single route to diff, e.g. /api/adults (default: all)")
|
||||
timeout := flag.Duration("timeout", 30*time.Second, "per-request HTTP timeout")
|
||||
flag.Parse()
|
||||
|
||||
client := &http.Client{Timeout: *timeout}
|
||||
|
||||
targets := allRoutes
|
||||
if *route != "" {
|
||||
targets = []string{*route}
|
||||
}
|
||||
|
||||
matched, diffs, errs := 0, 0, 0
|
||||
|
||||
for _, r := range targets {
|
||||
pyMap, err1 := fetch(client, *pyURL+r)
|
||||
goMap, err2 := fetch(client, *goURL+r)
|
||||
|
||||
if err1 != nil || err2 != nil {
|
||||
fmt.Printf("=== %s ===\n", r)
|
||||
if err1 != nil {
|
||||
fmt.Printf("ERROR (py): %v\n", err1)
|
||||
}
|
||||
if err2 != nil {
|
||||
fmt.Printf("ERROR (go): %v\n", err2)
|
||||
}
|
||||
fmt.Println()
|
||||
errs++
|
||||
continue
|
||||
}
|
||||
|
||||
scrub(pyMap, defaultAllowlist)
|
||||
scrub(goMap, defaultAllowlist)
|
||||
|
||||
diff := cmp.Diff(pyMap, goMap, cmpopts.EquateEmpty())
|
||||
if diff == "" {
|
||||
fmt.Printf("=== %s ===\nOK\n\n", r)
|
||||
matched++
|
||||
} else {
|
||||
fmt.Printf("=== %s ===\n%s\n", r, diff)
|
||||
diffs++
|
||||
}
|
||||
}
|
||||
|
||||
total := len(targets)
|
||||
fmt.Printf("parity: %d/%d routes match", matched, total)
|
||||
if diffs > 0 {
|
||||
fmt.Printf(", %d diff", diffs)
|
||||
if diffs > 1 {
|
||||
fmt.Print("s")
|
||||
}
|
||||
}
|
||||
if errs > 0 {
|
||||
fmt.Printf(", %d error", errs)
|
||||
if errs > 1 {
|
||||
fmt.Print("s")
|
||||
}
|
||||
}
|
||||
fmt.Println()
|
||||
|
||||
if errs > 0 {
|
||||
os.Exit(2)
|
||||
}
|
||||
if diffs > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func fetch(client *http.Client, url string) (map[string]any, error) {
|
||||
resp, err := client.Get(url) //nolint:noctx
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read body: %w", err)
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
|
||||
}
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(body, &m); err != nil {
|
||||
return nil, fmt.Errorf("decode JSON: %w", err)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// scrub removes keys from m whose dotted paths appear in paths.
|
||||
// A bare segment (no dot) deletes a top-level key.
|
||||
// A two-segment path "parent.child" deletes child from m["parent"] if it is a map.
|
||||
func scrub(m map[string]any, paths []string) {
|
||||
for _, path := range paths {
|
||||
parts := strings.SplitN(path, ".", 2)
|
||||
if len(parts) == 1 {
|
||||
delete(m, parts[0])
|
||||
} else {
|
||||
if child, ok := m[parts[0]].(map[string]any); ok {
|
||||
delete(child, parts[1])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
57
go/cmd/parity/scrub_test.go
Normal file
57
go/cmd/parity/scrub_test.go
Normal file
@@ -0,0 +1,57 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestScrubTopLevel(t *testing.T) {
|
||||
m := map[string]any{
|
||||
"build_meta": map[string]any{"tag": "v1"},
|
||||
"other": "keep",
|
||||
}
|
||||
scrub(m, []string{"build_meta"})
|
||||
if _, ok := m["build_meta"]; ok {
|
||||
t.Error("expected build_meta to be removed")
|
||||
}
|
||||
if m["other"] != "keep" {
|
||||
t.Error("expected other to be preserved")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScrubNested(t *testing.T) {
|
||||
m := map[string]any{
|
||||
"render_time": map[string]any{
|
||||
"total": "0.123",
|
||||
"breakdown": "fetch:0.1s",
|
||||
},
|
||||
"other": "keep",
|
||||
}
|
||||
scrub(m, []string{"render_time.total"})
|
||||
rt, ok := m["render_time"].(map[string]any)
|
||||
if !ok {
|
||||
t.Fatal("render_time should still be present")
|
||||
}
|
||||
if _, ok := rt["total"]; ok {
|
||||
t.Error("expected render_time.total to be removed")
|
||||
}
|
||||
if rt["breakdown"] != "fetch:0.1s" {
|
||||
t.Error("expected render_time.breakdown to be preserved")
|
||||
}
|
||||
if m["other"] != "keep" {
|
||||
t.Error("expected other to be preserved")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScrubMissingPath(t *testing.T) {
|
||||
m := map[string]any{"foo": "bar"}
|
||||
scrub(m, []string{"nonexistent", "render_time.total"})
|
||||
if m["foo"] != "bar" {
|
||||
t.Error("expected foo to be preserved")
|
||||
}
|
||||
}
|
||||
|
||||
func TestScrubNestedParentNotMap(t *testing.T) {
|
||||
m := map[string]any{"render_time": "not-a-map"}
|
||||
scrub(m, []string{"render_time.total"})
|
||||
if m["render_time"] != "not-a-map" {
|
||||
t.Error("expected render_time to be unchanged when it is not a map")
|
||||
}
|
||||
}
|
||||
40
go/go.mod
Normal file
40
go/go.mod
Normal file
@@ -0,0 +1,40 @@
|
||||
module fuj-management/go
|
||||
|
||||
go 1.26.1
|
||||
|
||||
require (
|
||||
github.com/google/go-cmp v0.7.0
|
||||
github.com/invopop/jsonschema v0.14.0
|
||||
golang.org/x/net v0.53.0
|
||||
golang.org/x/text v0.36.0
|
||||
google.golang.org/api v0.278.0
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/auth v0.20.0 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/buger/jsonparser v1.1.2 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.15 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.22.0 // indirect
|
||||
github.com/pb33f/ordered-map/v2 v2.3.1 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect
|
||||
go.opentelemetry.io/otel v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.43.0 // indirect
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.2 // indirect
|
||||
golang.org/x/crypto v0.50.0 // indirect
|
||||
golang.org/x/oauth2 v0.36.0 // indirect
|
||||
golang.org/x/sys v0.43.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect
|
||||
google.golang.org/grpc v1.80.0 // indirect
|
||||
google.golang.org/protobuf v1.36.11 // indirect
|
||||
)
|
||||
85
go/go.sum
Normal file
85
go/go.sum
Normal file
@@ -0,0 +1,85 @@
|
||||
cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA=
|
||||
cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
|
||||
cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
|
||||
cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
|
||||
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
|
||||
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
|
||||
github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk=
|
||||
github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
|
||||
github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.15 h1:xolVQTEXusUcAA5UgtyRLjelpFFHWlPQ4XfWGc7MBas=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.15/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg=
|
||||
github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU5vlZD4=
|
||||
github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY=
|
||||
github.com/invopop/jsonschema v0.14.0 h1:MHQqLhvpNUZfw+hM3AZDYK7jxO8FZoQeQM77g8iyZjg=
|
||||
github.com/invopop/jsonschema v0.14.0/go.mod h1:ygm6C2EaVNMBDPpaPlnOA2pFAxBnxGjFlMZABxm9n2I=
|
||||
github.com/pb33f/ordered-map/v2 v2.3.1 h1:5319HDO0aw4DA4gzi+zv4FXU9UlSs3xGZ40wcP1nBjY=
|
||||
github.com/pb33f/ordered-map/v2 v2.3.1/go.mod h1:qxFQgd0PkVUtOMCkTapqotNgzRhMPL7VvaHKbd1HnmQ=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg=
|
||||
go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
|
||||
go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
|
||||
go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=
|
||||
go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
|
||||
go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg=
|
||||
go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
|
||||
go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
|
||||
go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.2 h1:/FrI8D64VSr4HtGIlUtlFMGsm7H7pWTbj6vOLVZcA6s=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.2/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||
golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
|
||||
golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
|
||||
golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
|
||||
golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
|
||||
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
|
||||
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
|
||||
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
|
||||
golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
|
||||
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
|
||||
google.golang.org/api v0.278.0 h1:W7jiRvRi53VYFfZ/HoZjQBtJk7gOFbHD8ot1RzVZU6E=
|
||||
google.golang.org/api v0.278.0/go.mod h1:B9TqLBwJqVjp1mtt7WeoQwWRwvu/400y5lETOql+giQ=
|
||||
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0=
|
||||
google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260319201613-d00831a3d3e7 h1:41r6JMbpzBMen0R/4TZeeAmGXSJC7DftGINUodzTkPI=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:EIQZ5bFCfRQDV4MhRle7+OgjNtZ6P1PiZBgAKuxXu/Y=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 h1:tEkOQcXgF6dH1G+MVKZrfpYvozGrzb91k6ha7jireSM=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM=
|
||||
google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4=
|
||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
93
go/internal/config/config.go
Normal file
93
go/internal/config/config.go
Normal file
@@ -0,0 +1,93 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Google Sheets IDs — change in code if sheets change (not from env).
|
||||
const (
|
||||
AttendanceSheetID = "1E2e_gT_K5AwSRCDLDTa2UetZTkHmBOcz0kFbBUNUNBA"
|
||||
PaymentsSheetID = "1Om0YPoDVCH5cV8BrNz5LG5eR5MMU05ypQC7UMN1xn_Y"
|
||||
|
||||
// Both attendance tabs live in the same Google Spreadsheet (AttendanceSheetID).
|
||||
// The original adult and junior attendance data lives in separate source spreadsheets,
|
||||
// but is collected into this one sheet via IMPORTRANGE — one tab per group.
|
||||
// Tabs are identified by the gid= query param in the CSV export URL.
|
||||
AttendanceAdultSheetGID = "0" // gid=0 — adult practices tab (IMPORTRANGE'd)
|
||||
JuniorSheetGID = "1213318614" // gid=1213318614 — junior practices tab (IMPORTRANGE'd)
|
||||
)
|
||||
|
||||
// CacheSheetMap mirrors scripts/config.py CACHE_SHEET_MAP.
|
||||
// Maps a cache key to the Google Sheet ID whose Drive modifiedTime gates it.
|
||||
// Both attendance keys map to the same spreadsheet — different tabs, one Drive file.
|
||||
var CacheSheetMap = map[string]string{
|
||||
"attendance_regular": AttendanceSheetID,
|
||||
"attendance_juniors": AttendanceSheetID,
|
||||
"exceptions_dict": PaymentsSheetID,
|
||||
"payments_transactions": PaymentsSheetID,
|
||||
}
|
||||
|
||||
// Config holds all runtime configuration loaded from environment variables.
|
||||
// Mirrors scripts/config.py.
|
||||
type Config struct {
|
||||
CredentialsPath string
|
||||
BankAccount string
|
||||
CacheDir string
|
||||
CacheTTL time.Duration
|
||||
CacheAPICheckTTL time.Duration
|
||||
DriveTimeout time.Duration
|
||||
LogLevel string
|
||||
FioAPIToken string
|
||||
ServerAddr string
|
||||
}
|
||||
|
||||
// Load reads configuration from the environment, applying defaults that
|
||||
// match the Python side.
|
||||
func Load() Config {
|
||||
return Config{
|
||||
CredentialsPath: env("CREDENTIALS_PATH", ".secret/fuj-management-bot-credentials.json"),
|
||||
BankAccount: env("BANK_ACCOUNT", "CZ8520100000002800359168"),
|
||||
CacheDir: env("CACHE_DIR", "tmp/go"),
|
||||
CacheTTL: envDuration("CACHE_TTL_SECONDS", 300),
|
||||
CacheAPICheckTTL: envDuration("CACHE_API_CHECK_TTL_SECONDS", 300),
|
||||
DriveTimeout: envDuration("DRIVE_TIMEOUT_SECONDS", 10),
|
||||
LogLevel: env("LOG_LEVEL", "INFO"),
|
||||
FioAPIToken: env("FIO_API_TOKEN", ""),
|
||||
ServerAddr: env("SERVER_ADDR", ":8080"),
|
||||
}
|
||||
}
|
||||
|
||||
// IBANAccountNum extracts the bare account number from a Czech IBAN.
|
||||
// "CZ8520100000002800359168" → "2800359168"
|
||||
// Structure: CZ(2 check)(4 bank code)(16 zero-padded account).
|
||||
func IBANAccountNum(iban string) string {
|
||||
s := strings.ReplaceAll(iban, " ", "")
|
||||
if len(s) < 8 {
|
||||
return iban
|
||||
}
|
||||
raw := s[8:] // 16-digit zero-padded account portion
|
||||
n := strings.TrimLeft(raw, "0")
|
||||
if n == "" {
|
||||
return "0"
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func env(key, fallback string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func envDuration(key string, defaultSeconds int) time.Duration {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||
return time.Duration(n) * time.Second
|
||||
}
|
||||
}
|
||||
return time.Duration(defaultSeconds) * time.Second
|
||||
}
|
||||
26
go/internal/domain/czech/normalize.go
Normal file
26
go/internal/domain/czech/normalize.go
Normal file
@@ -0,0 +1,26 @@
|
||||
package czech
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
||||
// Normalize strips diacritics and lowercases s.
|
||||
//
|
||||
// Matches Python: unicodedata.normalize("NFKD", s) then filter out
|
||||
// combining characters (unicode.Mn only — not Mc/Me, which have
|
||||
// combining class 0 in Python's unicodedata.combining()).
|
||||
func Normalize(s string) string {
|
||||
decomposed := norm.NFKD.String(s)
|
||||
var b strings.Builder
|
||||
b.Grow(len(decomposed))
|
||||
for _, r := range decomposed {
|
||||
if unicode.In(r, unicode.Mn) {
|
||||
continue
|
||||
}
|
||||
b.WriteRune(r)
|
||||
}
|
||||
return strings.ToLower(b.String())
|
||||
}
|
||||
31
go/internal/domain/czech/normalize_test.go
Normal file
31
go/internal/domain/czech/normalize_test.go
Normal file
@@ -0,0 +1,31 @@
|
||||
package czech
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestNormalize(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{"Honza", "honza"},
|
||||
{"žluťoučký", "zlutoucky"},
|
||||
{"Příliš", "prilis"},
|
||||
{"Dvořák", "dvorak"},
|
||||
{"Růžena", "ruzena"},
|
||||
{"Čeněk", "cenek"},
|
||||
{"Kačer", "kacer"},
|
||||
{"", ""},
|
||||
{"prilis", "prilis"}, // idempotent
|
||||
{"Jan Novák", "jan novak"}, // whitespace preserved
|
||||
{"é", "e"}, // precomposed é (NFC)
|
||||
{"é", "e"}, // decomposed e + combining acute
|
||||
{"Ondřej Procházka", "ondrej prochazka"}, // realistic full name
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
got := Normalize(tc.in)
|
||||
if got != tc.want {
|
||||
t.Errorf("Normalize(%q) = %q, want %q", tc.in, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
154
go/internal/domain/czech/parse_month_references.go
Normal file
154
go/internal/domain/czech/parse_month_references.go
Normal file
@@ -0,0 +1,154 @@
|
||||
package czech
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var czechMonths = map[string]int{
|
||||
"leden": 1, "ledna": 1, "lednu": 1,
|
||||
"unor": 2, "unora": 2, "unoru": 2,
|
||||
"brezen": 3, "brezna": 3, "breznu": 3,
|
||||
"duben": 4, "dubna": 4, "dubnu": 4,
|
||||
"kveten": 5, "kvetna": 5, "kvetnu": 5,
|
||||
"cerven": 6, "cervna": 6, "cervnu": 6,
|
||||
"cervenec": 7, "cervnce": 7, "cervenci": 7,
|
||||
"srpen": 8, "srpna": 8, "srpnu": 8,
|
||||
"zari": 9,
|
||||
"rijen": 10, "rijna": 10, "rijnu": 10,
|
||||
"listopad": 11, "listopadu": 11,
|
||||
"prosinec": 12, "prosince": 12, "prosinci": 12,
|
||||
}
|
||||
|
||||
var (
|
||||
numericRe *regexp.Regexp
|
||||
dotRe *regexp.Regexp
|
||||
rangeRe *regexp.Regexp
|
||||
standRe *regexp.Regexp
|
||||
)
|
||||
|
||||
func init() {
|
||||
// Sort by descending length so longer alternatives win in RE2 leftmost-first
|
||||
// matching (e.g. "cervenec" is tried before "cerven").
|
||||
names := make([]string, 0, len(czechMonths))
|
||||
for name := range czechMonths {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Slice(names, func(i, j int) bool {
|
||||
if len(names[i]) != len(names[j]) {
|
||||
return len(names[i]) > len(names[j])
|
||||
}
|
||||
return names[i] < names[j]
|
||||
})
|
||||
alt := strings.Join(names, "|")
|
||||
|
||||
numericRe = regexp.MustCompile(`([\d+]+)\s*/\s*(\d{2,4})`)
|
||||
dotRe = regexp.MustCompile(`(\d{1,2})\s*\.\s*(\d{4})`)
|
||||
rangeRe = regexp.MustCompile(`(` + alt + `)\s*-\s*(` + alt + `)`)
|
||||
standRe = regexp.MustCompile(`\b(` + alt + `)\b`)
|
||||
}
|
||||
|
||||
// ParseMonthReferences extracts YYYY-MM month references from Czech free text.
|
||||
//
|
||||
// defaultYear seeds two heuristics: standalone month names with m >= 10 are
|
||||
// treated as defaultYear-1 (out-of-year backfill), and wrap-around ranges
|
||||
// (e.g. listopad-leden) place months >= start_m in defaultYear-1.
|
||||
//
|
||||
// Returns a sorted, deduplicated slice of "YYYY-MM" strings.
|
||||
func ParseMonthReferences(text string, defaultYear int) []string {
|
||||
normalized := Normalize(text)
|
||||
seen := map[string]struct{}{}
|
||||
|
||||
add := func(year, m int) {
|
||||
if m >= 1 && m <= 12 {
|
||||
seen[fmt.Sprintf("%04d-%02d", year, m)] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
// Pass 1: numeric months — "11+12/2025", "01/26", "1/2026"
|
||||
for _, groups := range numericRe.FindAllStringSubmatch(normalized, -1) {
|
||||
monthsPart, yearStr := groups[1], groups[2]
|
||||
year, err := strconv.Atoi(yearStr)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if year < 100 {
|
||||
year += 2000
|
||||
}
|
||||
for mStr := range strings.SplitSeq(monthsPart, "+") {
|
||||
mStr = strings.TrimSpace(mStr)
|
||||
if mStr == "" {
|
||||
continue
|
||||
}
|
||||
allDigits := true
|
||||
for _, c := range mStr {
|
||||
if c < '0' || c > '9' {
|
||||
allDigits = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if !allDigits {
|
||||
continue
|
||||
}
|
||||
m, err := strconv.Atoi(mStr)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
add(year, m)
|
||||
}
|
||||
}
|
||||
|
||||
// Pass 2: dot-separated month.year — "12.2025" (4-digit year only)
|
||||
for _, groups := range dotRe.FindAllStringSubmatch(normalized, -1) {
|
||||
m, _ := strconv.Atoi(groups[1])
|
||||
year, _ := strconv.Atoi(groups[2])
|
||||
add(year, m)
|
||||
}
|
||||
|
||||
// Pass 3a: Czech month name ranges — "listopad-leden"
|
||||
foundInRanges := map[string]struct{}{}
|
||||
for _, groups := range rangeRe.FindAllStringSubmatch(normalized, -1) {
|
||||
startName, endName := groups[1], groups[2]
|
||||
foundInRanges[startName] = struct{}{}
|
||||
foundInRanges[endName] = struct{}{}
|
||||
startM := czechMonths[startName]
|
||||
endM := czechMonths[endName]
|
||||
wraps := startM > endM
|
||||
m := startM
|
||||
for range 12 {
|
||||
year := defaultYear
|
||||
if wraps && m >= startM {
|
||||
year = defaultYear - 1
|
||||
}
|
||||
add(year, m)
|
||||
if m == endM {
|
||||
break
|
||||
}
|
||||
m = m%12 + 1
|
||||
}
|
||||
}
|
||||
|
||||
// Pass 3b: standalone Czech month names (not part of a range)
|
||||
for _, groups := range standRe.FindAllStringSubmatch(normalized, -1) {
|
||||
name := groups[1]
|
||||
if _, inRange := foundInRanges[name]; inRange {
|
||||
continue
|
||||
}
|
||||
m := czechMonths[name]
|
||||
year := defaultYear
|
||||
if m >= 10 {
|
||||
year = defaultYear - 1
|
||||
}
|
||||
add(year, m)
|
||||
}
|
||||
|
||||
result := make([]string, 0, len(seen))
|
||||
for k := range seen {
|
||||
result = append(result, k)
|
||||
}
|
||||
sort.Strings(result)
|
||||
return result
|
||||
}
|
||||
244
go/internal/domain/czech/parse_month_references_test.go
Normal file
244
go/internal/domain/czech/parse_month_references_test.go
Normal file
@@ -0,0 +1,244 @@
|
||||
package czech
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseMonthReferences(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// All expected outputs verified against live Python implementation on 2026-05-05:
|
||||
// PYTHONPATH=scripts:. python -c 'from czech_utils import parse_month_references; print(parse_month_references("<input>", 2026))'
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
defaultYear int
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
name: "empty",
|
||||
input: "",
|
||||
defaultYear: 2026,
|
||||
want: []string{},
|
||||
},
|
||||
{
|
||||
name: "numeric plus-split two months full year",
|
||||
input: "11+12/2025",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2025-11", "2025-12"},
|
||||
},
|
||||
{
|
||||
name: "numeric single month full year",
|
||||
input: "1/2026",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2026-01"},
|
||||
},
|
||||
{
|
||||
name: "numeric 2-digit year",
|
||||
input: "01/26",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2026-01"},
|
||||
},
|
||||
{
|
||||
name: "numeric plus-split with 2-digit year",
|
||||
input: "11+12/25",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2025-11", "2025-12"},
|
||||
},
|
||||
{
|
||||
name: "numeric three months sorted",
|
||||
input: "12+1+2/2026",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2026-01", "2026-02", "2026-12"},
|
||||
},
|
||||
{
|
||||
name: "dot pattern",
|
||||
input: "12.2025",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2025-12"},
|
||||
},
|
||||
{
|
||||
name: "dot pattern requires 4-digit year",
|
||||
input: "1.26",
|
||||
defaultYear: 2026,
|
||||
want: []string{},
|
||||
},
|
||||
{
|
||||
name: "standalone month below m10 threshold",
|
||||
input: "leden",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2026-01"},
|
||||
},
|
||||
{
|
||||
name: "standalone month m10 heuristic",
|
||||
input: "prosinec",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2025-12"},
|
||||
},
|
||||
{
|
||||
name: "declension prosince",
|
||||
input: "prosince",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2025-12"},
|
||||
},
|
||||
{
|
||||
name: "declension lednu",
|
||||
input: "lednu",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2026-01"},
|
||||
},
|
||||
{
|
||||
name: "standalone m10 boundary (rijen = October)",
|
||||
input: "rijen",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2025-10"},
|
||||
},
|
||||
{
|
||||
name: "standalone m9 just below boundary (zari = September)",
|
||||
input: "zari",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2026-09"},
|
||||
},
|
||||
{
|
||||
name: "range wrap Nov-Jan",
|
||||
input: "listopad-leden",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2025-11", "2025-12", "2026-01"},
|
||||
},
|
||||
{
|
||||
name: "range wrap starting at October",
|
||||
input: "rijen-leden",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2025-10", "2025-11", "2025-12", "2026-01"},
|
||||
},
|
||||
{
|
||||
name: "range no wrap",
|
||||
input: "unor-kveten",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2026-02", "2026-03", "2026-04", "2026-05"},
|
||||
},
|
||||
{
|
||||
name: "degenerate range same month",
|
||||
input: "leden-leden",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2026-01"},
|
||||
},
|
||||
{
|
||||
name: "range spanning m10 — heuristic does NOT fire for range members",
|
||||
input: "unor-listopad",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2026-02", "2026-03", "2026-04", "2026-05", "2026-06", "2026-07", "2026-08", "2026-09", "2026-10", "2026-11"},
|
||||
},
|
||||
{
|
||||
name: "longest-match alternation cervenec beats cerven",
|
||||
input: "cervenec-srpen",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2026-07", "2026-08"},
|
||||
},
|
||||
{
|
||||
name: "range plus standalone — range excludes, dedup",
|
||||
input: "listopad-leden, prosinec",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2025-11", "2025-12", "2026-01"},
|
||||
},
|
||||
{
|
||||
name: "two standalones no range",
|
||||
input: "prosinec leden",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2025-12", "2026-01"},
|
||||
},
|
||||
{
|
||||
name: "numeric plus range mix",
|
||||
input: "11+12/2025, leden-brezen",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2025-11", "2025-12", "2026-01", "2026-02", "2026-03"},
|
||||
},
|
||||
{
|
||||
name: "dedup across numeric and standalone passes",
|
||||
input: "11+12/25 a listopad",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2025-11", "2025-12"},
|
||||
},
|
||||
{
|
||||
name: "no digits before slash — standalone fires instead",
|
||||
input: "prosince/2025",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2025-12"},
|
||||
},
|
||||
{
|
||||
name: "range with trailing slash-year — numeric fails, range wins",
|
||||
input: "listopad-prosinec/2025",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2026-11", "2026-12"},
|
||||
},
|
||||
{
|
||||
name: "dot pattern only — numeric matches but month out of 1-12 range",
|
||||
input: "01.2026 / 02.2026",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2026-01", "2026-02"},
|
||||
},
|
||||
{
|
||||
name: "leading slash — numeric matches at second slash",
|
||||
input: "/12/2025",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2025-12"},
|
||||
},
|
||||
{
|
||||
name: "uppercase input normalized",
|
||||
input: "PROSINEC",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2025-12"},
|
||||
},
|
||||
{
|
||||
name: "diacritics stripped by Normalize",
|
||||
input: "Žluťoučký prosinec",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2025-12"},
|
||||
},
|
||||
{
|
||||
name: "diacritics in range with spaces around dash",
|
||||
input: "Únor - květen",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2026-02", "2026-03", "2026-04", "2026-05"},
|
||||
},
|
||||
{
|
||||
name: "natural language mixed with numeric and standalone",
|
||||
input: "platba 11/2025 a leden",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2025-11", "2026-01"},
|
||||
},
|
||||
{
|
||||
name: "English month name not recognized",
|
||||
input: "December",
|
||||
defaultYear: 2026,
|
||||
want: []string{},
|
||||
},
|
||||
{
|
||||
name: "duplicate input deduped",
|
||||
input: "11+12/2025 11+12/2025",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2025-11", "2025-12"},
|
||||
},
|
||||
{
|
||||
name: "trailing year without separator ignored",
|
||||
input: "leden 2026",
|
||||
defaultYear: 2026,
|
||||
want: []string{"2026-01"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := ParseMonthReferences(tc.input, tc.defaultYear)
|
||||
if got == nil {
|
||||
got = []string{}
|
||||
}
|
||||
if !reflect.DeepEqual(got, tc.want) {
|
||||
t.Errorf("ParseMonthReferences(%q, %d)\n got %v\n want %v",
|
||||
tc.input, tc.defaultYear, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
34
go/internal/domain/fees/fees.go
Normal file
34
go/internal/domain/fees/fees.go
Normal file
@@ -0,0 +1,34 @@
|
||||
// Package fees ports fee calculation from scripts/attendance.py.
|
||||
package fees
|
||||
|
||||
const (
|
||||
AdultFeeDefault = 700 // CZK fallback for 2+ practices when month not in AdultFeeMonthlyRate
|
||||
AdultFeeSingle = 200 // CZK for exactly 1 practice
|
||||
)
|
||||
|
||||
// AdultFeeMonthlyRate mirrors ADULT_FEE_MONTHLY_RATE in scripts/attendance.py.
|
||||
// Months absent from this map fall back to AdultFeeDefault.
|
||||
var AdultFeeMonthlyRate = map[string]int{
|
||||
"2025-09": 750, "2025-10": 750, "2025-11": 750, "2025-12": 750,
|
||||
"2026-01": 750, "2026-02": 750, "2026-03": 350,
|
||||
"2026-04": 700, "2026-05": 700,
|
||||
}
|
||||
|
||||
// CalculateFee returns the adult fee in CZK for attendanceCount practices in
|
||||
// the given monthKey (format "YYYY-MM").
|
||||
//
|
||||
// 0 practices → 0
|
||||
// 1 practice → AdultFeeSingle (200)
|
||||
// 2+ → AdultFeeMonthlyRate[monthKey] or AdultFeeDefault
|
||||
func CalculateFee(attendanceCount int, monthKey string) int {
|
||||
if attendanceCount == 0 {
|
||||
return 0
|
||||
}
|
||||
if attendanceCount == 1 {
|
||||
return AdultFeeSingle
|
||||
}
|
||||
if rate, ok := AdultFeeMonthlyRate[monthKey]; ok {
|
||||
return rate
|
||||
}
|
||||
return AdultFeeDefault
|
||||
}
|
||||
37
go/internal/domain/fees/fees_test.go
Normal file
37
go/internal/domain/fees/fees_test.go
Normal file
@@ -0,0 +1,37 @@
|
||||
package fees
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestCalculateFee(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// All expected outputs verified against live Python implementation on 2026-05-06:
|
||||
// PYTHONPATH=scripts:. python -c 'from attendance import calculate_fee; print([calculate_fee(c,m) for c,m in [(0,"2026-05"),(0,""),(1,"2026-05"),(1,"unknown"),(2,"2026-05"),(2,"2026-03"),(2,"2025-09"),(5,"2026-05"),(2,"2027-01"),(2,"")]])'
|
||||
tests := []struct {
|
||||
name string
|
||||
count int
|
||||
month string
|
||||
want int
|
||||
}{
|
||||
{"zero short-circuits", 0, "2026-05", 0},
|
||||
{"zero empty month", 0, "", 0},
|
||||
{"single practice", 1, "2026-05", 200},
|
||||
{"single ignores monthKey", 1, "unknown", 200},
|
||||
{"two practices configured month", 2, "2026-05", 700},
|
||||
{"two practices reduced march", 2, "2026-03", 350},
|
||||
{"two practices early season", 2, "2025-09", 750},
|
||||
{"high count same as two", 5, "2026-05", 700},
|
||||
{"unknown future month falls back", 2, "2027-01", 700},
|
||||
{"empty month falls back", 2, "", 700},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := CalculateFee(tc.count, tc.month)
|
||||
if got != tc.want {
|
||||
t.Errorf("CalculateFee(%d, %q) = %d, want %d", tc.count, tc.month, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
37
go/internal/domain/fees/junior.go
Normal file
37
go/internal/domain/fees/junior.go
Normal file
@@ -0,0 +1,37 @@
|
||||
package fees
|
||||
|
||||
const JuniorFeeDefault = 500 // CZK fallback for 2+ practices when month not in JuniorFeeMonthlyRate
|
||||
|
||||
// JuniorFeeMonthlyRate mirrors JUNIOR_MONTHLY_RATE in scripts/attendance.py.
|
||||
// Months absent from this map fall back to JuniorFeeDefault.
|
||||
var JuniorFeeMonthlyRate = map[string]int{
|
||||
"2025-09": 250,
|
||||
"2026-03": 250,
|
||||
}
|
||||
|
||||
// Expected is the result of a junior fee calculation.
|
||||
// When Unknown is true the fee requires manual review (Python returns "?");
|
||||
// in that case Value is meaningless — always check Unknown first.
|
||||
type Expected struct {
|
||||
Value int
|
||||
Unknown bool
|
||||
}
|
||||
|
||||
// CalculateJuniorFee returns the junior fee for attendanceCount practices in
|
||||
// the given monthKey (format "YYYY-MM").
|
||||
//
|
||||
// 0 practices → Expected{Value: 0}
|
||||
// 1 practice → Expected{Unknown: true} (manual review; Python sentinel "?")
|
||||
// 2+ → Expected{Value: JuniorFeeMonthlyRate[monthKey] or JuniorFeeDefault}
|
||||
func CalculateJuniorFee(attendanceCount int, monthKey string) Expected {
|
||||
if attendanceCount == 0 {
|
||||
return Expected{Value: 0}
|
||||
}
|
||||
if attendanceCount == 1 {
|
||||
return Expected{Unknown: true}
|
||||
}
|
||||
if rate, ok := JuniorFeeMonthlyRate[monthKey]; ok {
|
||||
return Expected{Value: rate}
|
||||
}
|
||||
return Expected{Value: JuniorFeeDefault}
|
||||
}
|
||||
37
go/internal/domain/fees/junior_test.go
Normal file
37
go/internal/domain/fees/junior_test.go
Normal file
@@ -0,0 +1,37 @@
|
||||
package fees
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestCalculateJuniorFee(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// All expected outputs verified against live Python implementation on 2026-05-06:
|
||||
// PYTHONPATH=scripts:. python -c 'from attendance import calculate_junior_fee; print([calculate_junior_fee(c,m) for c,m in [(0,"2026-05"),(0,""),(1,"2026-05"),(1,"unknown"),(2,"2026-05"),(2,"2025-09"),(2,"2026-03"),(5,"2025-09"),(2,"2027-01"),(2,"")]])'
|
||||
tests := []struct {
|
||||
name string
|
||||
count int
|
||||
month string
|
||||
want Expected
|
||||
}{
|
||||
{"zero short-circuits", 0, "2026-05", Expected{Value: 0}},
|
||||
{"zero empty month", 0, "", Expected{Value: 0}},
|
||||
{"single practice sentinel", 1, "2026-05", Expected{Unknown: true}},
|
||||
{"single ignores monthKey", 1, "unknown", Expected{Unknown: true}},
|
||||
{"two practices default month", 2, "2026-05", Expected{Value: 500}},
|
||||
{"two practices reduced sept", 2, "2025-09", Expected{Value: 250}},
|
||||
{"two practices reduced march", 2, "2026-03", Expected{Value: 250}},
|
||||
{"high count same as two", 5, "2025-09", Expected{Value: 250}},
|
||||
{"unknown future month falls back", 2, "2027-01", Expected{Value: 500}},
|
||||
{"empty month falls back", 2, "", Expected{Value: 500}},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := CalculateJuniorFee(tc.count, tc.month)
|
||||
if got != tc.want {
|
||||
t.Errorf("CalculateJuniorFee(%d, %q) = %+v, want %+v", tc.count, tc.month, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
2
go/internal/domain/matching/doc.go
Normal file
2
go/internal/domain/matching/doc.go
Normal file
@@ -0,0 +1,2 @@
|
||||
// Package matching ports name/member matching from scripts/match_payments.py.
|
||||
package matching
|
||||
41
go/internal/domain/matching/format_date.go
Normal file
41
go/internal/domain/matching/format_date.go
Normal file
@@ -0,0 +1,41 @@
|
||||
package matching
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var sheetsEpoch = time.Date(1899, 12, 30, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
// FormatDate normalizes a date value from Google Sheets.
|
||||
//
|
||||
// Accepts nil, empty string, int/float64 Sheets serial days since 1899-12-30,
|
||||
// a pre-formatted "YYYY-MM-DD" string (returned as-is), or any other value
|
||||
// (returned as fmt.Sprint(v).TrimSpace). Never returns an error.
|
||||
//
|
||||
// Ports scripts/match_payments.py format_date.
|
||||
func FormatDate(val any) string {
|
||||
if val == nil {
|
||||
return ""
|
||||
}
|
||||
switch v := val.(type) {
|
||||
case int:
|
||||
return sheetsEpoch.Add(time.Duration(float64(v) * 24 * float64(time.Hour))).Format("2006-01-02")
|
||||
case int64:
|
||||
return sheetsEpoch.Add(time.Duration(float64(v) * 24 * float64(time.Hour))).Format("2006-01-02")
|
||||
case float64:
|
||||
return sheetsEpoch.Add(time.Duration(v * 24 * float64(time.Hour))).Format("2006-01-02")
|
||||
case string:
|
||||
s := strings.TrimSpace(v)
|
||||
if s == "" {
|
||||
return ""
|
||||
}
|
||||
if len(s) == 10 && s[4] == '-' && s[7] == '-' {
|
||||
return s
|
||||
}
|
||||
return s
|
||||
default:
|
||||
return strings.TrimSpace(fmt.Sprint(v))
|
||||
}
|
||||
}
|
||||
49
go/internal/domain/matching/format_date_test.go
Normal file
49
go/internal/domain/matching/format_date_test.go
Normal file
@@ -0,0 +1,49 @@
|
||||
package matching
|
||||
|
||||
// Expected values verified against scripts/match_payments.py on 2026-05-06:
|
||||
//
|
||||
// PYTHONPATH=scripts:. python3 -c '
|
||||
// from match_payments import format_date
|
||||
// for v in [None, "", 44197, 44197.5, "2026-04-15", "garbage", " 2026-04-15 "]:
|
||||
// print(repr(format_date(v)))
|
||||
// '
|
||||
//
|
||||
// Output:
|
||||
//
|
||||
// ''
|
||||
// ''
|
||||
// '2021-01-01'
|
||||
// '2021-01-01'
|
||||
// '2026-04-15'
|
||||
// 'garbage'
|
||||
// '2026-04-15'
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestFormatDate(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
input any
|
||||
want string
|
||||
}{
|
||||
{name: "nil", input: nil, want: ""},
|
||||
{name: "empty string", input: "", want: ""},
|
||||
{name: "serial int", input: int(44197), want: "2021-01-01"},
|
||||
{name: "serial float fractional", input: float64(44197.5), want: "2021-01-01"},
|
||||
{name: "already formatted", input: "2026-04-15", want: "2026-04-15"},
|
||||
{name: "garbage string", input: "garbage", want: "garbage"},
|
||||
{name: "padded date string trimmed", input: " 2026-04-15 ", want: "2026-04-15"},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := FormatDate(tc.input)
|
||||
if got != tc.want {
|
||||
t.Errorf("FormatDate(%v) = %q, want %q", tc.input, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
89
go/internal/domain/matching/infer.go
Normal file
89
go/internal/domain/matching/infer.go
Normal file
@@ -0,0 +1,89 @@
|
||||
package matching
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"fuj-management/go/internal/domain/czech"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Transaction is the subset of a payment row used by InferTransactionDetails.
|
||||
// Date accepts string ("YYYY-MM-DD"), float64 (Sheets serial), or int — matching
|
||||
// the heterogeneous types returned by the Sheets API and the FIO scraper.
|
||||
type Transaction struct {
|
||||
Sender string
|
||||
Message string
|
||||
UserID string
|
||||
Date any
|
||||
}
|
||||
|
||||
// InferredDetails is the result of InferTransactionDetails.
|
||||
type InferredDetails struct {
|
||||
Members []Match
|
||||
Months []string
|
||||
SearchText string
|
||||
}
|
||||
|
||||
// InferTransactionDetails infers which member(s) and month(s) a transaction belongs to.
|
||||
//
|
||||
// Search text for member matching: sender + message + user_id.
|
||||
// Month search text: message + user_id only (sender excluded, matching Python).
|
||||
// Fallback 1: if no members found, retry match on sender alone.
|
||||
// Fallback 2: if no months found, derive from tx.Date (Sheets serial or YYYY-MM-DD).
|
||||
//
|
||||
// defaultYear seeds czech.ParseMonthReferences (Python defaulted to the current year;
|
||||
// callers should pass time.Now().Year() or a fixed year for deterministic tests).
|
||||
//
|
||||
// Ports scripts/match_payments.py infer_transaction_details.
|
||||
func InferTransactionDetails(tx Transaction, memberNames []string, defaultYear int) InferredDetails {
|
||||
searchText := fmt.Sprintf("%s %s %s", tx.Sender, tx.Message, tx.UserID)
|
||||
|
||||
members := MatchMembers(searchText, memberNames)
|
||||
months := czech.ParseMonthReferences(tx.Message+" "+tx.UserID, defaultYear)
|
||||
|
||||
if len(members) == 0 {
|
||||
members = MatchMembers(tx.Sender, memberNames)
|
||||
}
|
||||
|
||||
if len(months) == 0 && tx.Date != nil && tx.Date != "" {
|
||||
if ym := inferMonthFromDate(tx.Date); ym != "" {
|
||||
months = []string{ym}
|
||||
}
|
||||
}
|
||||
|
||||
if months == nil {
|
||||
months = []string{}
|
||||
}
|
||||
|
||||
return InferredDetails{
|
||||
Members: members,
|
||||
Months: months,
|
||||
SearchText: searchText,
|
||||
}
|
||||
}
|
||||
|
||||
// inferMonthFromDate converts a date value to "YYYY-MM" for the month fallback.
|
||||
// Returns "" on any error, matching Python's bare except pass.
|
||||
func inferMonthFromDate(val any) string {
|
||||
switch v := val.(type) {
|
||||
case int:
|
||||
dt := sheetsEpoch.Add(time.Duration(float64(v) * 24 * float64(time.Hour)))
|
||||
return dt.Format("2006-01")
|
||||
case int64:
|
||||
dt := sheetsEpoch.Add(time.Duration(float64(v) * 24 * float64(time.Hour)))
|
||||
return dt.Format("2006-01")
|
||||
case float64:
|
||||
dt := sheetsEpoch.Add(time.Duration(v * 24 * float64(time.Hour)))
|
||||
return dt.Format("2006-01")
|
||||
case string:
|
||||
if v == "" {
|
||||
return ""
|
||||
}
|
||||
dt, err := time.Parse("2006-01-02", v)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return dt.Format("2006-01")
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
108
go/internal/domain/matching/infer_test.go
Normal file
108
go/internal/domain/matching/infer_test.go
Normal file
@@ -0,0 +1,108 @@
|
||||
package matching
|
||||
|
||||
// Expected values verified against scripts/match_payments.py on 2026-05-06:
|
||||
//
|
||||
// PYTHONPATH=scripts:. python3 << 'EOF'
|
||||
// from match_payments import infer_transaction_details
|
||||
// MEMBERS = ["Tomáš Němeček (Tov)", "Jana Nováková"]
|
||||
// cases = [
|
||||
// ({"sender":"Tomas Nemecek","message":"clenske 04/2026","user_id":"","date":"2026-04-15"}, "full match"),
|
||||
// ({"sender":"Tomas Nemecek","message":"","user_id":"","date":"2026-04-15"}, "sender fallback month"),
|
||||
// ({"sender":"Jana Novakova","message":"","user_id":"","date":44197}, "serial int date"),
|
||||
// ({"sender":"neznamy","message":"","user_id":"","date":""}, "no match"),
|
||||
// ({"sender":"Tomas Nemecek","message":"","user_id":"","date":44197.5}, "serial float date"),
|
||||
// ]
|
||||
// for tx, label in cases:
|
||||
// r = infer_transaction_details(tx, MEMBERS)
|
||||
// print(label + ": members=" + repr(r["members"]) + " months=" + repr(r["months"]) + " search_text=" + repr(r["search_text"]))
|
||||
// EOF
|
||||
//
|
||||
// Output:
|
||||
//
|
||||
// full match: members=[('Tomáš Němeček (Tov)', 'auto')] months=['2026-04'] search_text='Tomas Nemecek clenske 04/2026 '
|
||||
// sender fallback month: members=[('Tomáš Němeček (Tov)', 'auto')] months=['2026-04'] search_text='Tomas Nemecek '
|
||||
// serial int date: members=[('Jana Nováková', 'auto')] months=['2021-01'] search_text='Jana Novakova '
|
||||
// no match: members=[] months=[] search_text='neznamy '
|
||||
// serial float date: members=[('Tomáš Němeček (Tov)', 'auto')] months=['2021-01'] search_text='Tomas Nemecek '
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var inferMembers = []string{"Tomáš Němeček (Tov)", "Jana Nováková"}
|
||||
|
||||
func TestInferTransactionDetails(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
tx Transaction
|
||||
defaultYear int
|
||||
wantMembers []Match
|
||||
wantMonths []string
|
||||
wantSearchText string
|
||||
}{
|
||||
{
|
||||
name: "full match — members and months from search text",
|
||||
tx: Transaction{Sender: "Tomas Nemecek", Message: "clenske 04/2026", UserID: "", Date: "2026-04-15"},
|
||||
defaultYear: 2026,
|
||||
wantMembers: []Match{{Name: "Tomáš Němeček (Tov)", Confidence: ConfidenceAuto}},
|
||||
wantMonths: []string{"2026-04"},
|
||||
// Python: sender + " " + message + " " + user_id (no trim)
|
||||
wantSearchText: "Tomas Nemecek clenske 04/2026 ",
|
||||
},
|
||||
{
|
||||
// months not in message → fall back to date string
|
||||
name: "months fall back to date string",
|
||||
tx: Transaction{Sender: "Tomas Nemecek", Message: "", UserID: "", Date: "2026-04-15"},
|
||||
defaultYear: 2026,
|
||||
wantMembers: []Match{{Name: "Tomáš Němeček (Tov)", Confidence: ConfidenceAuto}},
|
||||
wantMonths: []string{"2026-04"},
|
||||
wantSearchText: "Tomas Nemecek ",
|
||||
},
|
||||
{
|
||||
// months fall back to Sheets serial int date
|
||||
name: "months fall back to serial int date",
|
||||
tx: Transaction{Sender: "Jana Novakova", Message: "", UserID: "", Date: int(44197)},
|
||||
defaultYear: 2026,
|
||||
wantMembers: []Match{{Name: "Jana Nováková", Confidence: ConfidenceAuto}},
|
||||
wantMonths: []string{"2021-01"},
|
||||
wantSearchText: "Jana Novakova ",
|
||||
},
|
||||
{
|
||||
// months fall back to Sheets serial float64 date
|
||||
name: "months fall back to serial float date",
|
||||
tx: Transaction{Sender: "Tomas Nemecek", Message: "", UserID: "", Date: float64(44197.5)},
|
||||
defaultYear: 2026,
|
||||
wantMembers: []Match{{Name: "Tomáš Němeček (Tov)", Confidence: ConfidenceAuto}},
|
||||
wantMonths: []string{"2021-01"},
|
||||
wantSearchText: "Tomas Nemecek ",
|
||||
},
|
||||
{
|
||||
name: "no match — both slices empty not nil",
|
||||
tx: Transaction{Sender: "neznamy", Message: "", UserID: "", Date: ""},
|
||||
defaultYear: 2026,
|
||||
wantMembers: []Match{},
|
||||
wantMonths: []string{},
|
||||
wantSearchText: "neznamy ",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := InferTransactionDetails(tc.tx, inferMembers, tc.defaultYear)
|
||||
|
||||
if !reflect.DeepEqual(got.Members, tc.wantMembers) {
|
||||
t.Errorf("Members\n got %v\n want %v", got.Members, tc.wantMembers)
|
||||
}
|
||||
if !reflect.DeepEqual(got.Months, tc.wantMonths) {
|
||||
t.Errorf("Months\n got %v\n want %v", got.Months, tc.wantMonths)
|
||||
}
|
||||
if got.SearchText != tc.wantSearchText {
|
||||
t.Errorf("SearchText\n got %q\n want %q", got.SearchText, tc.wantSearchText)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
131
go/internal/domain/matching/match_members.go
Normal file
131
go/internal/domain/matching/match_members.go
Normal file
@@ -0,0 +1,131 @@
|
||||
package matching
|
||||
|
||||
import (
|
||||
"fuj-management/go/internal/domain/czech"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Confidence indicates how certain a member match is.
|
||||
type Confidence string
|
||||
|
||||
const (
|
||||
ConfidenceAuto Confidence = "auto"
|
||||
ConfidenceReview Confidence = "review"
|
||||
)
|
||||
|
||||
// Match pairs a canonical member name with the confidence of the match.
|
||||
type Match struct {
|
||||
Name string
|
||||
Confidence Confidence
|
||||
}
|
||||
|
||||
var commonSurnames = map[string]bool{
|
||||
"novak": true,
|
||||
"novakova": true,
|
||||
"prach": true,
|
||||
}
|
||||
|
||||
// MatchMembers finds members mentioned in text and returns them with a
|
||||
// confidence level of "auto" (reliable) or "review" (needs human verification).
|
||||
//
|
||||
// Algorithm (ported verbatim from scripts/match_payments.py match_members):
|
||||
// 1. Exact short-circuit: if any member's full normalized name appears as whole
|
||||
// words in normalize(text), return ONLY those matches as auto. This prevents
|
||||
// nickname "tov" from matching inside surname "ottova".
|
||||
// 2. Per-member first-match-wins: full-name substring → first+last both present
|
||||
// (any order) → nickname whole-word. Each yields auto.
|
||||
// 3. Review tier: last name (len≥4, not a common surname) → first name (len≥3)
|
||||
// → single-part name (len≥4). Each yields review.
|
||||
// 4. Final filter: if any auto exists, drop all review.
|
||||
func MatchMembers(text string, memberNames []string) []Match {
|
||||
normalizedText := czech.Normalize(text)
|
||||
|
||||
// Pass 1: exact short-circuit
|
||||
var exactMatches []Match
|
||||
for _, name := range memberNames {
|
||||
variants := BuildNameVariants(name)
|
||||
if len(variants) == 0 {
|
||||
continue
|
||||
}
|
||||
fullName := variants[0]
|
||||
if fullName != "" && wordIn(fullName, normalizedText) {
|
||||
exactMatches = append(exactMatches, Match{Name: name, Confidence: ConfidenceAuto})
|
||||
}
|
||||
}
|
||||
if len(exactMatches) > 0 {
|
||||
return exactMatches
|
||||
}
|
||||
|
||||
// Pass 2 + 3: fuzzy matching
|
||||
var matches []Match
|
||||
for _, name := range memberNames {
|
||||
variants := BuildNameVariants(name)
|
||||
fullName := ""
|
||||
if len(variants) > 0 {
|
||||
fullName = variants[0]
|
||||
}
|
||||
parts := strings.Fields(fullName)
|
||||
|
||||
// Auto tier
|
||||
if fullName != "" && strings.Contains(normalizedText, fullName) {
|
||||
matches = append(matches, Match{Name: name, Confidence: ConfidenceAuto})
|
||||
continue
|
||||
}
|
||||
if len(parts) >= 2 {
|
||||
if wordIn(parts[0], normalizedText) && wordIn(parts[len(parts)-1], normalizedText) {
|
||||
matches = append(matches, Match{Name: name, Confidence: ConfidenceAuto})
|
||||
continue
|
||||
}
|
||||
}
|
||||
// Nickname check
|
||||
if m := nicknameRe.FindStringSubmatch(name); m != nil {
|
||||
nick := czech.Normalize(m[1])
|
||||
if nick != "" && wordIn(nick, normalizedText) {
|
||||
matches = append(matches, Match{Name: name, Confidence: ConfidenceAuto})
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Review tier
|
||||
if len(parts) >= 2 {
|
||||
lastName := parts[len(parts)-1]
|
||||
firstName := parts[0]
|
||||
if len(lastName) >= 4 && !commonSurnames[lastName] && wordIn(lastName, normalizedText) {
|
||||
matches = append(matches, Match{Name: name, Confidence: ConfidenceReview})
|
||||
continue
|
||||
}
|
||||
if len(firstName) >= 3 && wordIn(firstName, normalizedText) {
|
||||
matches = append(matches, Match{Name: name, Confidence: ConfidenceReview})
|
||||
continue
|
||||
}
|
||||
} else if len(parts) == 1 {
|
||||
if len(parts[0]) >= 4 && wordIn(parts[0], normalizedText) {
|
||||
matches = append(matches, Match{Name: name, Confidence: ConfidenceReview})
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Final filter: drop review if any auto exists
|
||||
hasAuto := false
|
||||
for _, m := range matches {
|
||||
if m.Confidence == ConfidenceAuto {
|
||||
hasAuto = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if hasAuto {
|
||||
filtered := matches[:0]
|
||||
for _, m := range matches {
|
||||
if m.Confidence == ConfidenceAuto {
|
||||
filtered = append(filtered, m)
|
||||
}
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
if matches == nil {
|
||||
return []Match{}
|
||||
}
|
||||
return matches
|
||||
}
|
||||
156
go/internal/domain/matching/match_members_test.go
Normal file
156
go/internal/domain/matching/match_members_test.go
Normal file
@@ -0,0 +1,156 @@
|
||||
package matching
|
||||
|
||||
// Expected values verified against scripts/match_payments.py and
|
||||
// tests/test_match_members.py on 2026-05-06:
|
||||
//
|
||||
// PYTHONPATH=scripts:. python3 -c '
|
||||
// from match_payments import match_members
|
||||
// MEMBERS = ["Henrietta Ottová", "Tomáš Němeček (Tov)", "František Vrbík (Štrúdl)", "Jana Nováková"]
|
||||
// cases = [
|
||||
// ("Henrietta Ottová (Heny): 04/2026", "full name guard"),
|
||||
// ("platba ottova 04/2026", "ottova surname"),
|
||||
// ("Henrietta Ottová a Tomáš Němeček 04/2026", "two full names"),
|
||||
// ("Tov platba 04/2026", "nickname alone"),
|
||||
// ("Henrietta Ottova 04/2026", "no diacritics"),
|
||||
// ("Platba od Nemeček Tomas 04/2026", "reversed first+last"),
|
||||
// ("vrbik clenske", "last name only review"),
|
||||
// ("jana platba", "first name review"),
|
||||
// ("neznamy platebce", "no match"),
|
||||
// ]
|
||||
// for text, label in cases: print(label + ":", match_members(text, MEMBERS))
|
||||
// '
|
||||
//
|
||||
// Output:
|
||||
//
|
||||
// full name guard: [('Henrietta Ottová', 'auto')]
|
||||
// ottova surname: [('Henrietta Ottová', 'review')]
|
||||
// two full names: [('Henrietta Ottová', 'auto'), ('Tomáš Němeček (Tov)', 'auto')]
|
||||
// nickname alone: [('Tomáš Němeček (Tov)', 'auto')]
|
||||
// no diacritics: [('Henrietta Ottová', 'auto')]
|
||||
// reversed first+last: [('Tomáš Němeček (Tov)', 'auto')]
|
||||
// last name only review: [('František Vrbík (Štrúdl)', 'review')]
|
||||
// first name review: [('Jana Nováková', 'review')]
|
||||
// no match: []
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
var testMembers = []string{
|
||||
"Henrietta Ottová",
|
||||
"Tomáš Němeček (Tov)",
|
||||
"František Vrbík (Štrúdl)",
|
||||
"Jana Nováková",
|
||||
}
|
||||
|
||||
func TestMatchMembers(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
text string
|
||||
wantContains []string
|
||||
wantExcludes []string
|
||||
wantAllAuto bool
|
||||
}{
|
||||
{
|
||||
// Short-circuit: full name matches → "tov" inside "ottova" must NOT fire
|
||||
name: "full name in message returns only that member",
|
||||
text: "Henrietta Ottová (Heny): 04/2026",
|
||||
wantContains: []string{"Henrietta Ottová"},
|
||||
wantExcludes: []string{"Tomáš Němeček (Tov)"},
|
||||
wantAllAuto: true,
|
||||
},
|
||||
{
|
||||
// "tov" is a substring of "ottova" — nickname must not match inside a surname
|
||||
name: "nickname tov not matched inside ottova",
|
||||
text: "platba ottova 04/2026",
|
||||
wantExcludes: []string{"Tomáš Němeček (Tov)"},
|
||||
wantAllAuto: false,
|
||||
},
|
||||
{
|
||||
name: "two full names both auto",
|
||||
text: "Henrietta Ottová a Tomáš Němeček 04/2026",
|
||||
wantContains: []string{"Henrietta Ottová", "Tomáš Němeček (Tov)"},
|
||||
wantAllAuto: true,
|
||||
},
|
||||
{
|
||||
name: "nickname alone matches correctly",
|
||||
text: "Tov platba 04/2026",
|
||||
wantContains: []string{"Tomáš Němeček (Tov)"},
|
||||
wantAllAuto: true,
|
||||
},
|
||||
{
|
||||
name: "full name without diacritics auto",
|
||||
text: "Henrietta Ottova 04/2026",
|
||||
wantContains: []string{"Henrietta Ottová"},
|
||||
wantExcludes: []string{"Tomáš Němeček (Tov)"},
|
||||
wantAllAuto: true,
|
||||
},
|
||||
{
|
||||
name: "first and last name reversed auto",
|
||||
text: "Platba od Nemeček Tomas 04/2026",
|
||||
wantContains: []string{"Tomáš Němeček (Tov)"},
|
||||
wantAllAuto: true,
|
||||
},
|
||||
{
|
||||
// Last name alone (len≥4, not a common surname) → review confidence
|
||||
name: "last name only yields review",
|
||||
text: "vrbik clenske",
|
||||
wantContains: []string{"František Vrbík (Štrúdl)"},
|
||||
wantAllAuto: false,
|
||||
},
|
||||
{
|
||||
// First name alone (len≥3) → review confidence
|
||||
name: "first name only yields review",
|
||||
text: "jana platba",
|
||||
wantContains: []string{"Jana Nováková"},
|
||||
wantAllAuto: false,
|
||||
},
|
||||
{
|
||||
name: "no match returns empty slice",
|
||||
text: "neznamy platebce",
|
||||
wantContains: nil,
|
||||
wantAllAuto: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := MatchMembers(tc.text, testMembers)
|
||||
|
||||
// Check required members are present
|
||||
for _, want := range tc.wantContains {
|
||||
found := false
|
||||
for _, m := range got {
|
||||
if m.Name == want {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("MatchMembers(%q): want %q in result, got %v", tc.text, want, got)
|
||||
}
|
||||
}
|
||||
|
||||
// Check excluded members are absent
|
||||
for _, exclude := range tc.wantExcludes {
|
||||
for _, m := range got {
|
||||
if m.Name == exclude {
|
||||
t.Errorf("MatchMembers(%q): %q should not be in result, got %v", tc.text, exclude, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check all-auto constraint
|
||||
if tc.wantAllAuto {
|
||||
for _, m := range got {
|
||||
if m.Confidence != ConfidenceAuto {
|
||||
t.Errorf("MatchMembers(%q): expected all auto, got %v", tc.text, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
59
go/internal/domain/matching/name_variants.go
Normal file
59
go/internal/domain/matching/name_variants.go
Normal file
@@ -0,0 +1,59 @@
|
||||
package matching
|
||||
|
||||
import (
|
||||
"fuj-management/go/internal/domain/czech"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
nicknameRe = regexp.MustCompile(`\(([^)]+)\)`)
|
||||
nicknameStripRe = regexp.MustCompile(`\s*\([^)]*\)\s*`)
|
||||
)
|
||||
|
||||
// BuildNameVariants returns searchable lowercase ASCII variants of a member name.
|
||||
//
|
||||
// Example: "František Vrbík (Štrúdl)" → ["frantisek vrbik", "strudl", "vrbik", "frantisek"]
|
||||
//
|
||||
// variants[0] is always the full normalized base name (no nickname). MatchMembers relies on
|
||||
// this invariant for the exact short-circuit pass. Variants shorter than 3 characters are
|
||||
// dropped.
|
||||
//
|
||||
// Ports scripts/match_payments.py _build_name_variants.
|
||||
func BuildNameVariants(name string) []string {
|
||||
var nickname string
|
||||
if m := nicknameRe.FindStringSubmatch(name); m != nil {
|
||||
nickname = m[1]
|
||||
}
|
||||
|
||||
base := strings.TrimSpace(nicknameStripRe.ReplaceAllString(name, " "))
|
||||
normalizedBase := czech.Normalize(base)
|
||||
normalizedNick := czech.Normalize(nickname)
|
||||
|
||||
variants := []string{normalizedBase}
|
||||
if normalizedNick != "" {
|
||||
variants = append(variants, normalizedNick)
|
||||
}
|
||||
|
||||
parts := strings.Fields(normalizedBase)
|
||||
if len(parts) >= 2 {
|
||||
variants = append(variants, parts[len(parts)-1]) // last name
|
||||
variants = append(variants, parts[0]) // first name
|
||||
}
|
||||
|
||||
filtered := variants[:0]
|
||||
for _, v := range variants {
|
||||
if len(v) >= 3 {
|
||||
filtered = append(filtered, v)
|
||||
}
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
// wordIn returns true if needle appears as a whole word in haystack.
|
||||
// Both needle and haystack must already be ASCII-folded (via czech.Normalize).
|
||||
func wordIn(needle, haystack string) bool {
|
||||
pattern := `\b` + regexp.QuoteMeta(needle) + `\b`
|
||||
matched, _ := regexp.MatchString(pattern, haystack)
|
||||
return matched
|
||||
}
|
||||
62
go/internal/domain/matching/name_variants_test.go
Normal file
62
go/internal/domain/matching/name_variants_test.go
Normal file
@@ -0,0 +1,62 @@
|
||||
package matching
|
||||
|
||||
// Expected values verified against scripts/match_payments.py on 2026-05-06:
|
||||
//
|
||||
// PYTHONPATH=scripts:. python3 -c '
|
||||
// from match_payments import _build_name_variants
|
||||
// for n in ["František Vrbík (Štrúdl)", "Tov (St)", "Jana", " Petr Novák ( Jenda ) "]:
|
||||
// print(repr(n), "->", _build_name_variants(n))
|
||||
// '
|
||||
//
|
||||
// Output:
|
||||
//
|
||||
// 'František Vrbík (Štrúdl)' -> ['frantisek vrbik', 'strudl', 'vrbik', 'frantisek']
|
||||
// 'Tov (St)' -> ['tov']
|
||||
// 'Jana' -> ['jana']
|
||||
// ' Petr Novák ( Jenda ) ' -> ['petr novak', ' jenda ', 'novak', 'petr']
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBuildNameVariants(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
input string
|
||||
want []string
|
||||
}{
|
||||
{
|
||||
name: "full name with nickname",
|
||||
input: "František Vrbík (Štrúdl)",
|
||||
want: []string{"frantisek vrbik", "strudl", "vrbik", "frantisek"},
|
||||
},
|
||||
{
|
||||
name: "nickname too short filtered out",
|
||||
input: "Tov (St)",
|
||||
want: []string{"tov"},
|
||||
},
|
||||
{
|
||||
name: "single-part name no nickname",
|
||||
input: "Jana",
|
||||
want: []string{"jana"},
|
||||
},
|
||||
{
|
||||
name: "extra whitespace inside parens preserved by normalize",
|
||||
input: " Petr Novák ( Jenda ) ",
|
||||
want: []string{"petr novak", " jenda ", "novak", "petr"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := BuildNameVariants(tc.input)
|
||||
if !reflect.DeepEqual(got, tc.want) {
|
||||
t.Errorf("BuildNameVariants(%q)\n got %q\n want %q", tc.input, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
49
go/internal/domain/money/money.go
Normal file
49
go/internal/domain/money/money.go
Normal file
@@ -0,0 +1,49 @@
|
||||
// Package money ports Czech-locale currency parsing from scripts/infer_payments.py.
|
||||
package money
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ErrInvalidAmount is returned by ParseCZK when the input cannot be parsed.
|
||||
var ErrInvalidAmount = errors.New("money: invalid CZK amount")
|
||||
|
||||
// ParseCZK parses a Czech-locale amount string and returns the value in CZK
|
||||
// as a float64. Mirrors scripts/infer_payments.py parse_czk_amount:
|
||||
//
|
||||
// - empty input → (0, nil)
|
||||
// - "Kč"/"CZK" suffixes stripped (case-sensitive, like Python)
|
||||
// - comma present → comma is decimal sep, dots/spaces are thousand seps
|
||||
// ("1.500,00" → 1500.0)
|
||||
// - no comma, 2+ dots → all dots are thousand seps ("1.500.000" → 1500000.0)
|
||||
// - no comma, ≤1 dot → dot is decimal sep ("1.500" → 1.5)
|
||||
// - on parse failure → (0, ErrInvalidAmount); callers wanting Python's
|
||||
// silent-zero behaviour can discard the error: v, _ := ParseCZK(s)
|
||||
func ParseCZK(s string) (float64, error) {
|
||||
if s == "" {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
s = strings.ReplaceAll(s, "Kč", "")
|
||||
s = strings.ReplaceAll(s, "CZK", "")
|
||||
s = strings.TrimSpace(s)
|
||||
|
||||
if strings.ContainsRune(s, ',') {
|
||||
s = strings.ReplaceAll(s, ".", "")
|
||||
s = strings.ReplaceAll(s, " ", "")
|
||||
s = strings.ReplaceAll(s, ",", ".")
|
||||
} else if strings.Count(s, ".") > 1 {
|
||||
s = strings.ReplaceAll(s, ".", "")
|
||||
s = strings.ReplaceAll(s, " ", "")
|
||||
} else {
|
||||
s = strings.ReplaceAll(s, " ", "")
|
||||
}
|
||||
|
||||
v, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil {
|
||||
return 0, ErrInvalidAmount
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
67
go/internal/domain/money/money_test.go
Normal file
67
go/internal/domain/money/money_test.go
Normal file
@@ -0,0 +1,67 @@
|
||||
package money
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseCZK(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// All expected outputs verified against live Python implementation on 2026-05-06:
|
||||
// PYTHONPATH=scripts:. python -c '
|
||||
// from infer_payments import parse_czk_amount
|
||||
// for v in [None, "", "0", "500", "500 Kč", "500 CZK",
|
||||
// "1 500", "1500.00", "1 500.00",
|
||||
// "1.500,00", "1500,5", "1.500.000",
|
||||
// "1.500", "abc", " ", "100,5 Kč"]:
|
||||
// print(repr(v), "->", parse_czk_amount(v))
|
||||
// '
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want float64
|
||||
wantErr bool
|
||||
}{
|
||||
{"empty string", "", 0, false},
|
||||
{"zero string", "0", 0, false},
|
||||
{"plain integer", "500", 500, false},
|
||||
{"with Kč suffix", "500 Kč", 500, false},
|
||||
{"with CZK suffix", "500 CZK", 500, false},
|
||||
{"space thousand sep", "1 500", 1500, false},
|
||||
{"dot decimal", "1500.00", 1500, false},
|
||||
{"space thousands dot decimal", "1 500.00", 1500, false},
|
||||
{"dot thousand comma decimal", "1.500,00", 1500, false},
|
||||
{"comma decimal no thousands", "1500,5", 1500.5, false},
|
||||
{"multiple dot thousand seps", "1.500.000", 1500000, false},
|
||||
{"single dot is decimal heuristic", "1.500", 1.5, false},
|
||||
{"comma decimal with Kč", "100,5 Kč", 100.5, false},
|
||||
{"garbage text", "abc", 0, true},
|
||||
{"spaces only", " ", 0, true},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got, err := ParseCZK(tc.input)
|
||||
if (err != nil) != tc.wantErr {
|
||||
t.Errorf("ParseCZK(%q) error = %v, wantErr %v", tc.input, err, tc.wantErr)
|
||||
}
|
||||
if got != tc.want {
|
||||
t.Errorf("ParseCZK(%q) = %v, want %v", tc.input, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseCZKSilentZero documents that discarding the error recovers Python's
|
||||
// silent-zero behaviour for any garbage input.
|
||||
func TestParseCZKSilentZero(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, s := range []string{"abc", " ", "Kč", "CZK"} {
|
||||
v, _ := ParseCZK(s)
|
||||
if v != 0 {
|
||||
t.Errorf("ParseCZK(%q) silent-zero: got %v, want 0", s, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
406
go/internal/domain/reconcile/reconcile.go
Normal file
406
go/internal/domain/reconcile/reconcile.go
Normal file
@@ -0,0 +1,406 @@
|
||||
// Package reconcile ports the three-phase payment reconciliation from scripts/match_payments.py.
|
||||
package reconcile
|
||||
|
||||
import (
|
||||
"fuj-management/go/internal/domain/czech"
|
||||
"fuj-management/go/internal/domain/matching"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ExceptionKey identifies a fee override by normalized member name and period.
|
||||
type ExceptionKey struct {
|
||||
Name string // czech.Normalize(memberName)
|
||||
Period string // czech.Normalize("YYYY-MM")
|
||||
}
|
||||
|
||||
// Exception is a manual fee override for one member in one period.
|
||||
type Exception struct {
|
||||
Amount int
|
||||
Note string
|
||||
}
|
||||
|
||||
// FeeData holds the expected fee and attendance data for one member in one month.
|
||||
type FeeData struct {
|
||||
Expected int
|
||||
IsUnknown bool // true when junior has exactly 1 session (manual review; Python sentinel "?")
|
||||
Attendance int
|
||||
JuniorAttendance int // junior-tab sessions; used for the :NJ,MA breakdown in the juniors view
|
||||
AdultAttendance int // adult-tab sessions for J-tier members; used for the :NJ,MA breakdown
|
||||
}
|
||||
|
||||
// Member is one row from the attendance sheet.
|
||||
type Member struct {
|
||||
Name string
|
||||
Tier string
|
||||
Fees map[string]FeeData // month ("YYYY-MM") → fee data
|
||||
}
|
||||
|
||||
// Transaction is one payment row from the payments sheet.
|
||||
// Date must already be a "YYYY-MM-DD" string (convert with matching.FormatDate before calling).
|
||||
// InferredAmount, when non-nil, replaces Amount when person and purpose are pre-matched.
|
||||
type Transaction struct {
|
||||
Date string
|
||||
Amount float64
|
||||
ManualFix string // "manual fix" column; non-empty disables re-inference
|
||||
Person string // comma-separated canonical names (empty → use inference)
|
||||
Purpose string // comma-separated "YYYY-MM" or "other:…" (empty → use inference)
|
||||
InferredAmount *float64 // nil → fall back to Amount
|
||||
Sender string
|
||||
VS string // Variabilní symbol (Czech variable payment symbol)
|
||||
Message string
|
||||
BankID string
|
||||
SyncID string
|
||||
UserID string
|
||||
}
|
||||
|
||||
// TxEntry is the portion of a payment allocated to a single member+month.
|
||||
type TxEntry struct {
|
||||
Amount float64
|
||||
Date string
|
||||
Sender string
|
||||
Message string
|
||||
Confidence string
|
||||
}
|
||||
|
||||
// OtherEntry is a payment with purpose "other:…" allocated to a member.
|
||||
type OtherEntry struct {
|
||||
Amount float64
|
||||
Date string
|
||||
Sender string
|
||||
Message string
|
||||
Purpose string
|
||||
Confidence string
|
||||
}
|
||||
|
||||
// MonthData is the ledger state for one member in one month.
|
||||
type MonthData struct {
|
||||
Expected int
|
||||
IsUnknown bool // mirrors FeeData.IsUnknown; not overridden by exceptions
|
||||
OriginalExpected int
|
||||
AttendanceCount int
|
||||
JuniorAttendance int // junior-tab sessions; for :NJ,MA breakdown in juniors view
|
||||
AdultAttendance int // adult-tab sessions; for :NJ,MA breakdown
|
||||
Exception *Exception
|
||||
Paid float64
|
||||
Transactions []TxEntry
|
||||
}
|
||||
|
||||
// MemberResult is the reconciled ledger for one member.
|
||||
type MemberResult struct {
|
||||
Tier string
|
||||
Months map[string]MonthData
|
||||
OtherTransactions []OtherEntry
|
||||
TotalBalance int
|
||||
}
|
||||
|
||||
// Result is the top-level output of Reconcile.
|
||||
type Result struct {
|
||||
Members map[string]MemberResult
|
||||
Unmatched []Transaction
|
||||
Credits map[string]int // final balance for every member (may be negative)
|
||||
}
|
||||
|
||||
var questionMarkRe = regexp.MustCompile(`\[\?\]\s*`)
|
||||
|
||||
// canonicalMemberKey returns a diacritic-, case-, and whitespace-insensitive key
|
||||
// used to resolve Person-column values that drift from canonical attendance-sheet names.
|
||||
// Ports scripts/match_payments.py canonical_member_key.
|
||||
func canonicalMemberKey(name string) string {
|
||||
return strings.Join(strings.Fields(czech.Normalize(name)), " ")
|
||||
}
|
||||
|
||||
type monthExpected struct {
|
||||
month string
|
||||
expected int
|
||||
}
|
||||
|
||||
// Reconcile matches transactions to members and months using three allocation phases:
|
||||
// 1. Greedy: payment ≥ total expected → fill each month exactly; overflow → credit.
|
||||
// 2. Proportional: payment < total → distribute by each month's share; last absorbs float remainder.
|
||||
// 3. Even-split fallback: all expected fees are 0 (prepayment) → divide equally.
|
||||
//
|
||||
// defaultYear seeds czech.ParseMonthReferences in the inference fallback.
|
||||
// Pass time.Now().Year() in production; pass a fixed year in tests.
|
||||
//
|
||||
// Ports scripts/match_payments.py reconcile.
|
||||
func Reconcile(
|
||||
members []Member,
|
||||
sortedMonths []string,
|
||||
transactions []Transaction,
|
||||
exceptions map[ExceptionKey]Exception,
|
||||
defaultYear int,
|
||||
) Result {
|
||||
memberNames := make([]string, len(members))
|
||||
memberTiers := make(map[string]string, len(members))
|
||||
memberFees := make(map[string]map[string]FeeData, len(members))
|
||||
|
||||
for i, m := range members {
|
||||
memberNames[i] = m.Name
|
||||
memberTiers[m.Name] = m.Tier
|
||||
memberFees[m.Name] = m.Fees
|
||||
}
|
||||
|
||||
// Map canonical key → first attendance-sheet name with that key, so Person cells
|
||||
// that drift in diacritics/case/whitespace still resolve to the canonical name.
|
||||
canonicalByKey := make(map[string]string, len(memberNames))
|
||||
for _, name := range memberNames {
|
||||
key := canonicalMemberKey(name)
|
||||
if _, exists := canonicalByKey[key]; !exists {
|
||||
canonicalByKey[key] = name
|
||||
}
|
||||
}
|
||||
|
||||
if exceptions == nil {
|
||||
exceptions = map[ExceptionKey]Exception{}
|
||||
}
|
||||
|
||||
// Initialise ledger
|
||||
ledger := make(map[string]map[string]MonthData, len(memberNames))
|
||||
otherLedger := make(map[string][]OtherEntry, len(memberNames))
|
||||
|
||||
for _, name := range memberNames {
|
||||
ledger[name] = make(map[string]MonthData, len(sortedMonths))
|
||||
otherLedger[name] = []OtherEntry{}
|
||||
for _, m := range sortedMonths {
|
||||
fd := memberFees[name][m]
|
||||
originalExpected := fd.Expected
|
||||
attendanceCount := fd.Attendance
|
||||
|
||||
var expected int
|
||||
var exInfo *Exception
|
||||
exKey := ExceptionKey{
|
||||
Name: czech.Normalize(name),
|
||||
Period: czech.Normalize(m),
|
||||
}
|
||||
if ex, ok := exceptions[exKey]; ok {
|
||||
expected = ex.Amount
|
||||
exCopy := ex
|
||||
exInfo = &exCopy
|
||||
} else {
|
||||
expected = originalExpected
|
||||
}
|
||||
|
||||
ledger[name][m] = MonthData{
|
||||
Expected: expected,
|
||||
IsUnknown: fd.IsUnknown,
|
||||
OriginalExpected: originalExpected,
|
||||
AttendanceCount: attendanceCount,
|
||||
JuniorAttendance: fd.JuniorAttendance,
|
||||
AdultAttendance: fd.AdultAttendance,
|
||||
Exception: exInfo,
|
||||
Paid: 0,
|
||||
Transactions: []TxEntry{},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var unmatched []Transaction
|
||||
credits := make(map[string]int, len(memberNames))
|
||||
|
||||
for _, tx := range transactions {
|
||||
personStr := strings.TrimSpace(tx.Person)
|
||||
purposeStr := strings.TrimSpace(tx.Purpose)
|
||||
personStr = questionMarkRe.ReplaceAllString(personStr, "")
|
||||
isOther := strings.HasPrefix(strings.ToLower(purposeStr), "other:")
|
||||
|
||||
var matchedMembers []matching.Match
|
||||
var matchedMonths []string
|
||||
var amount float64
|
||||
|
||||
if personStr != "" && purposeStr != "" {
|
||||
for p := range strings.SplitSeq(personStr, ",") {
|
||||
p = strings.TrimSpace(p)
|
||||
if p != "" {
|
||||
matchedMembers = append(matchedMembers, matching.Match{
|
||||
Name: p,
|
||||
Confidence: matching.ConfidenceAuto,
|
||||
})
|
||||
}
|
||||
}
|
||||
if isOther {
|
||||
matchedMonths = []string{purposeStr}
|
||||
} else {
|
||||
for m := range strings.SplitSeq(purposeStr, ",") {
|
||||
m = strings.TrimSpace(m)
|
||||
if m != "" {
|
||||
matchedMonths = append(matchedMonths, m)
|
||||
}
|
||||
}
|
||||
}
|
||||
if tx.InferredAmount != nil {
|
||||
amount = *tx.InferredAmount
|
||||
} else {
|
||||
amount = tx.Amount
|
||||
}
|
||||
} else {
|
||||
// Inference fallback for rows not yet processed by infer_payments.py
|
||||
inferred := matching.InferTransactionDetails(
|
||||
matching.Transaction{
|
||||
Sender: tx.Sender,
|
||||
Message: tx.Message,
|
||||
UserID: tx.UserID,
|
||||
Date: tx.Date,
|
||||
},
|
||||
memberNames,
|
||||
defaultYear,
|
||||
)
|
||||
matchedMembers = inferred.Members
|
||||
matchedMonths = inferred.Months
|
||||
amount = tx.Amount
|
||||
}
|
||||
|
||||
if len(matchedMembers) == 0 || len(matchedMonths) == 0 {
|
||||
unmatched = append(unmatched, tx)
|
||||
continue
|
||||
}
|
||||
|
||||
if isOther {
|
||||
nAlloc := len(matchedMembers)
|
||||
perAlloc := 0.0
|
||||
if nAlloc > 0 {
|
||||
perAlloc = amount / float64(nAlloc)
|
||||
}
|
||||
for _, m := range matchedMembers {
|
||||
memberName := canonicalByKey[canonicalMemberKey(m.Name)]
|
||||
if memberName != "" {
|
||||
otherLedger[memberName] = append(otherLedger[memberName], OtherEntry{
|
||||
Amount: perAlloc,
|
||||
Date: tx.Date,
|
||||
Sender: tx.Sender,
|
||||
Message: tx.Message,
|
||||
Purpose: purposeStr,
|
||||
Confidence: string(m.Confidence),
|
||||
})
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
memberShare := 0.0
|
||||
if len(matchedMembers) > 0 {
|
||||
memberShare = amount / float64(len(matchedMembers))
|
||||
}
|
||||
|
||||
for _, m := range matchedMembers {
|
||||
memberName := canonicalByKey[canonicalMemberKey(m.Name)]
|
||||
if memberName == "" {
|
||||
unmatched = append(unmatched, tx)
|
||||
continue
|
||||
}
|
||||
|
||||
var inWindow []monthExpected
|
||||
outCount := 0
|
||||
for _, month := range matchedMonths {
|
||||
if md, ok := ledger[memberName][month]; ok {
|
||||
inWindow = append(inWindow, monthExpected{month: month, expected: md.Expected})
|
||||
} else {
|
||||
outCount++
|
||||
}
|
||||
}
|
||||
|
||||
nTotal := len(matchedMonths)
|
||||
outCredit := 0.0
|
||||
if outCount > 0 && nTotal > 0 {
|
||||
outCredit = memberShare / float64(nTotal) * float64(outCount)
|
||||
credits[memberName] += int(outCredit)
|
||||
}
|
||||
|
||||
inWindowShare := memberShare - outCredit
|
||||
|
||||
if len(inWindow) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
totalExpected := 0
|
||||
for _, mw := range inWindow {
|
||||
totalExpected += mw.expected
|
||||
}
|
||||
|
||||
if totalExpected > 0 && inWindowShare >= float64(totalExpected) {
|
||||
// Greedy: payment covers all expected fees; overflow → credit
|
||||
credits[memberName] += int(inWindowShare - float64(totalExpected))
|
||||
for _, mw := range inWindow {
|
||||
alloc := float64(mw.expected)
|
||||
md := ledger[memberName][mw.month]
|
||||
md.Paid += alloc
|
||||
md.Transactions = append(md.Transactions, TxEntry{
|
||||
Amount: alloc,
|
||||
Date: tx.Date,
|
||||
Sender: tx.Sender,
|
||||
Message: tx.Message,
|
||||
Confidence: string(m.Confidence),
|
||||
})
|
||||
ledger[memberName][mw.month] = md
|
||||
}
|
||||
} else if totalExpected > 0 {
|
||||
// Proportional: distribute by each month's share; last month absorbs float remainder
|
||||
remaining := inWindowShare
|
||||
for i, mw := range inWindow {
|
||||
var alloc float64
|
||||
if i == len(inWindow)-1 {
|
||||
alloc = remaining
|
||||
} else {
|
||||
alloc = inWindowShare * float64(mw.expected) / float64(totalExpected)
|
||||
}
|
||||
remaining -= alloc
|
||||
md := ledger[memberName][mw.month]
|
||||
md.Paid += alloc
|
||||
md.Transactions = append(md.Transactions, TxEntry{
|
||||
Amount: alloc,
|
||||
Date: tx.Date,
|
||||
Sender: tx.Sender,
|
||||
Message: tx.Message,
|
||||
Confidence: string(m.Confidence),
|
||||
})
|
||||
ledger[memberName][mw.month] = md
|
||||
}
|
||||
} else {
|
||||
// Even-split fallback: prepayment before attendance recorded
|
||||
perMonth := inWindowShare / float64(len(inWindow))
|
||||
for _, mw := range inWindow {
|
||||
md := ledger[memberName][mw.month]
|
||||
md.Paid += perMonth
|
||||
md.Transactions = append(md.Transactions, TxEntry{
|
||||
Amount: perMonth,
|
||||
Date: tx.Date,
|
||||
Sender: tx.Sender,
|
||||
Message: tx.Message,
|
||||
Confidence: string(m.Confidence),
|
||||
})
|
||||
ledger[memberName][mw.month] = md
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Final total balances: window balance + out-of-window credits accumulated above
|
||||
finalBalances := make(map[string]int, len(memberNames))
|
||||
for _, name := range memberNames {
|
||||
windowBalance := 0
|
||||
for _, mdata := range ledger[name] {
|
||||
windowBalance += int(mdata.Paid) - mdata.Expected
|
||||
}
|
||||
finalBalances[name] = windowBalance + credits[name]
|
||||
}
|
||||
|
||||
membersResult := make(map[string]MemberResult, len(memberNames))
|
||||
for _, name := range memberNames {
|
||||
membersResult[name] = MemberResult{
|
||||
Tier: memberTiers[name],
|
||||
Months: ledger[name],
|
||||
OtherTransactions: otherLedger[name],
|
||||
TotalBalance: finalBalances[name],
|
||||
}
|
||||
}
|
||||
|
||||
if unmatched == nil {
|
||||
unmatched = []Transaction{}
|
||||
}
|
||||
|
||||
return Result{
|
||||
Members: membersResult,
|
||||
Unmatched: unmatched,
|
||||
Credits: finalBalances,
|
||||
}
|
||||
}
|
||||
376
go/internal/domain/reconcile/reconcile_test.go
Normal file
376
go/internal/domain/reconcile/reconcile_test.go
Normal file
@@ -0,0 +1,376 @@
|
||||
package reconcile
|
||||
|
||||
// Expected values verified against scripts/match_payments.py on 2026-05-06:
|
||||
//
|
||||
// PYTHONPATH=scripts:. python3 -m unittest tests.test_reconcile_exceptions tests.test_match_payments -v
|
||||
//
|
||||
// All Python test cases are ported below. Additional Go-only cases are marked with [Go].
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const defaultYear = 2026
|
||||
|
||||
// tx builds a pre-matched Transaction (person+purpose already filled in).
|
||||
// InferredAmount is left nil so Amount is used directly, matching the Python
|
||||
// _tx helper where inferred_amount == amount.
|
||||
func tx(person, purpose string, amount float64) Transaction {
|
||||
return Transaction{
|
||||
Date: "2026-01-01",
|
||||
Amount: amount,
|
||||
Person: person,
|
||||
Purpose: purpose,
|
||||
Sender: "Sender",
|
||||
Message: "fee",
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileExceptionOverride(t *testing.T) {
|
||||
t.Parallel()
|
||||
members := []Member{{Name: "Alice", Tier: "A", Fees: map[string]FeeData{"2026-01": {Expected: 750, Attendance: 4}}}}
|
||||
exceptions := map[ExceptionKey]Exception{
|
||||
{Name: "alice", Period: "2026-01"}: {Amount: 400, Note: "Test exception"},
|
||||
}
|
||||
txs := []Transaction{{
|
||||
Date: "2026-01-05", Amount: 400,
|
||||
Person: "Alice", Purpose: "2026-01", Sender: "Alice Sender", Message: "fee",
|
||||
}}
|
||||
|
||||
result := Reconcile(members, []string{"2026-01"}, txs, exceptions, defaultYear)
|
||||
|
||||
jan := result.Members["Alice"].Months["2026-01"]
|
||||
if jan.Expected != 400 {
|
||||
t.Errorf("Expected override to 400, got %d", jan.Expected)
|
||||
}
|
||||
if jan.Paid != 400 {
|
||||
t.Errorf("Paid want 400, got %f", jan.Paid)
|
||||
}
|
||||
if result.Members["Alice"].TotalBalance != 0 {
|
||||
t.Errorf("TotalBalance want 0, got %d", result.Members["Alice"].TotalBalance)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileFallbackToAttendance(t *testing.T) {
|
||||
t.Parallel()
|
||||
members := []Member{{Name: "Alice", Tier: "A", Fees: map[string]FeeData{"2026-01": {Expected: 750, Attendance: 4}}}}
|
||||
|
||||
result := Reconcile(members, []string{"2026-01"}, nil, nil, defaultYear)
|
||||
|
||||
if result.Members["Alice"].Months["2026-01"].Expected != 750 {
|
||||
t.Errorf("Expected 750 when no exception, got %d", result.Members["Alice"].Months["2026-01"].Expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileGreedyExactMatch(t *testing.T) {
|
||||
t.Parallel()
|
||||
members := []Member{{
|
||||
Name: "Alice", Tier: "A",
|
||||
Fees: map[string]FeeData{
|
||||
"2026-02": {Expected: 750, Attendance: 3},
|
||||
"2026-03": {Expected: 350, Attendance: 3},
|
||||
"2026-04": {Expected: 150, Attendance: 2},
|
||||
},
|
||||
}}
|
||||
sortedMonths := []string{"2026-02", "2026-03", "2026-04"}
|
||||
|
||||
result := Reconcile(members, sortedMonths, []Transaction{tx("Alice", "2026-02, 2026-03, 2026-04", 1250)}, nil, defaultYear)
|
||||
|
||||
months := result.Members["Alice"].Months
|
||||
if int(months["2026-02"].Paid) != 750 {
|
||||
t.Errorf("2026-02 paid want 750, got %f", months["2026-02"].Paid)
|
||||
}
|
||||
if int(months["2026-03"].Paid) != 350 {
|
||||
t.Errorf("2026-03 paid want 350, got %f", months["2026-03"].Paid)
|
||||
}
|
||||
if int(months["2026-04"].Paid) != 150 {
|
||||
t.Errorf("2026-04 paid want 150, got %f", months["2026-04"].Paid)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileGreedyOverpaymentGoesToCredit(t *testing.T) {
|
||||
t.Parallel()
|
||||
members := []Member{{
|
||||
Name: "Alice", Tier: "A",
|
||||
Fees: map[string]FeeData{"2026-01": {Expected: 750, Attendance: 3}, "2026-02": {Expected: 750, Attendance: 3}},
|
||||
}}
|
||||
sortedMonths := []string{"2026-01", "2026-02"}
|
||||
|
||||
result := Reconcile(members, sortedMonths, []Transaction{tx("Alice", "2026-01, 2026-02", 2000)}, nil, defaultYear)
|
||||
|
||||
months := result.Members["Alice"].Months
|
||||
if int(months["2026-01"].Paid) != 750 {
|
||||
t.Errorf("2026-01 paid want 750, got %f", months["2026-01"].Paid)
|
||||
}
|
||||
if int(months["2026-02"].Paid) != 750 {
|
||||
t.Errorf("2026-02 paid want 750, got %f", months["2026-02"].Paid)
|
||||
}
|
||||
if result.Credits["Alice"] != 500 {
|
||||
t.Errorf("credits want 500, got %d", result.Credits["Alice"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileProportionalUnderpayment(t *testing.T) {
|
||||
t.Parallel()
|
||||
members := []Member{{
|
||||
Name: "Alice", Tier: "A",
|
||||
Fees: map[string]FeeData{"2026-02": {Expected: 750, Attendance: 3}, "2026-03": {Expected: 350, Attendance: 3}, "2026-04": {Expected: 750, Attendance: 3}},
|
||||
}}
|
||||
sortedMonths := []string{"2026-02", "2026-03", "2026-04"}
|
||||
amount := 1250.0
|
||||
|
||||
result := Reconcile(members, sortedMonths, []Transaction{tx("Alice", "2026-02, 2026-03, 2026-04", amount)}, nil, defaultYear)
|
||||
|
||||
months := result.Members["Alice"].Months
|
||||
paid02 := months["2026-02"].Paid
|
||||
paid03 := months["2026-03"].Paid
|
||||
paid04 := months["2026-04"].Paid
|
||||
|
||||
if paid02 >= 750 {
|
||||
t.Errorf("2026-02 should be underpaid, got %f", paid02)
|
||||
}
|
||||
if paid03 >= 350 {
|
||||
t.Errorf("2026-03 should be underpaid, got %f", paid03)
|
||||
}
|
||||
if paid04 >= 750 {
|
||||
t.Errorf("2026-04 should be underpaid, got %f", paid04)
|
||||
}
|
||||
if math.Abs(paid02+paid03+paid04-amount) > 0.01 {
|
||||
t.Errorf("sum of paid want %f, got %f", amount, paid02+paid03+paid04)
|
||||
}
|
||||
if math.Abs(paid02-paid04) > 0.01 {
|
||||
t.Errorf("02 and 04 have equal expected, want equal paid: %f vs %f", paid02, paid04)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileSingleMonthUnchanged(t *testing.T) {
|
||||
t.Parallel()
|
||||
members := []Member{{Name: "Alice", Tier: "A", Fees: map[string]FeeData{"2026-01": {Expected: 750, Attendance: 3}}}}
|
||||
|
||||
result := Reconcile(members, []string{"2026-01"}, []Transaction{tx("Alice", "2026-01", 750)}, nil, defaultYear)
|
||||
|
||||
if math.Abs(result.Members["Alice"].Months["2026-01"].Paid-750) > 0.01 {
|
||||
t.Errorf("single month want 750, got %f", result.Members["Alice"].Months["2026-01"].Paid)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileTwoMembersMultiMonth(t *testing.T) {
|
||||
t.Parallel()
|
||||
members := []Member{
|
||||
{Name: "Alice", Tier: "A", Fees: map[string]FeeData{"2026-01": {Expected: 750, Attendance: 3}, "2026-02": {Expected: 350, Attendance: 3}}},
|
||||
{Name: "Bob", Tier: "A", Fees: map[string]FeeData{"2026-01": {Expected: 750, Attendance: 3}, "2026-02": {Expected: 350, Attendance: 3}}},
|
||||
}
|
||||
sortedMonths := []string{"2026-01", "2026-02"}
|
||||
|
||||
result := Reconcile(members, sortedMonths, []Transaction{tx("Alice, Bob", "2026-01, 2026-02", 2200)}, nil, defaultYear)
|
||||
|
||||
for _, name := range []string{"Alice", "Bob"} {
|
||||
months := result.Members[name].Months
|
||||
if math.Abs(months["2026-01"].Paid-750) > 0.01 {
|
||||
t.Errorf("%s 2026-01 paid want 750, got %f", name, months["2026-01"].Paid)
|
||||
}
|
||||
if math.Abs(months["2026-02"].Paid-350) > 0.01 {
|
||||
t.Errorf("%s 2026-02 paid want 350, got %f", name, months["2026-02"].Paid)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileEvenSplitFallbackWhenNoExpected(t *testing.T) {
|
||||
t.Parallel()
|
||||
members := []Member{{
|
||||
Name: "Alice", Tier: "A",
|
||||
Fees: map[string]FeeData{"2026-01": {Expected: 0, Attendance: 0}, "2026-02": {Expected: 0, Attendance: 0}},
|
||||
}}
|
||||
sortedMonths := []string{"2026-01", "2026-02"}
|
||||
|
||||
result := Reconcile(members, sortedMonths, []Transaction{tx("Alice", "2026-01, 2026-02", 300)}, nil, defaultYear)
|
||||
|
||||
months := result.Members["Alice"].Months
|
||||
if math.Abs(months["2026-01"].Paid-150) > 0.01 {
|
||||
t.Errorf("2026-01 paid want 150, got %f", months["2026-01"].Paid)
|
||||
}
|
||||
if math.Abs(months["2026-02"].Paid-150) > 0.01 {
|
||||
t.Errorf("2026-02 paid want 150, got %f", months["2026-02"].Paid)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileDiacriticsTolerantPersonMatching(t *testing.T) {
|
||||
t.Parallel()
|
||||
members := []Member{{Name: "Mária Maco", Tier: "A", Fees: map[string]FeeData{"2026-04": {Expected: 750, Attendance: 4}}}}
|
||||
txFn := func(person string) Transaction {
|
||||
return Transaction{
|
||||
Date: "2026-04-15", Amount: 750, Person: person, Purpose: "2026-04",
|
||||
Sender: "Maco Family", Message: "fee",
|
||||
}
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
person string
|
||||
}{
|
||||
{"without diacritics", "Maria Maco"},
|
||||
{"extra whitespace", "Mária Maco"},
|
||||
{"lowercase", "mária maco"},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
result := Reconcile(members, []string{"2026-04"}, []Transaction{txFn(tc.person)}, nil, defaultYear)
|
||||
|
||||
paid := result.Members["Mária Maco"].Months["2026-04"].Paid
|
||||
if paid != 750 {
|
||||
t.Errorf("%s: paid want 750, got %f", tc.name, paid)
|
||||
}
|
||||
if len(result.Unmatched) != 0 {
|
||||
t.Errorf("%s: want no unmatched, got %v", tc.name, result.Unmatched)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileTrulyUnknownPersonIsUnmatched(t *testing.T) {
|
||||
t.Parallel()
|
||||
members := []Member{{Name: "Mária Maco", Tier: "A", Fees: map[string]FeeData{"2026-04": {Expected: 750, Attendance: 4}}}}
|
||||
txs := []Transaction{{
|
||||
Date: "2026-04-15", Amount: 750,
|
||||
Person: "Někdo Neznámý", Purpose: "2026-04",
|
||||
Sender: "Neznámý", Message: "fee",
|
||||
}}
|
||||
|
||||
result := Reconcile(members, []string{"2026-04"}, txs, nil, defaultYear)
|
||||
|
||||
if result.Members["Mária Maco"].Months["2026-04"].Paid != 0 {
|
||||
t.Errorf("unknown person must not credit the member")
|
||||
}
|
||||
if len(result.Unmatched) != 1 {
|
||||
t.Errorf("want 1 unmatched, got %d", len(result.Unmatched))
|
||||
}
|
||||
}
|
||||
|
||||
// [Go] Test that [?] markers are stripped from the Person field before lookup.
|
||||
func TestReconcileQuestionMarkMarkerStripped(t *testing.T) {
|
||||
t.Parallel()
|
||||
members := []Member{{Name: "Alice", Tier: "A", Fees: map[string]FeeData{"2026-01": {Expected: 750, Attendance: 3}}}}
|
||||
txs := []Transaction{{
|
||||
Date: "2026-01-01", Amount: 750,
|
||||
Person: "[?] Alice", Purpose: "2026-01",
|
||||
Sender: "Bank", Message: "fee",
|
||||
}}
|
||||
|
||||
result := Reconcile(members, []string{"2026-01"}, txs, nil, defaultYear)
|
||||
|
||||
if result.Members["Alice"].Months["2026-01"].Paid != 750 {
|
||||
t.Errorf("[?] stripping: want 750 paid, got %f", result.Members["Alice"].Months["2026-01"].Paid)
|
||||
}
|
||||
}
|
||||
|
||||
// [Go] Purpose "other:shirt" puts payment in OtherTransactions, not in month ledger.
|
||||
func TestReconcileOtherPurpose(t *testing.T) {
|
||||
t.Parallel()
|
||||
members := []Member{{Name: "Alice", Tier: "A", Fees: map[string]FeeData{"2026-01": {Expected: 750, Attendance: 3}}}}
|
||||
txs := []Transaction{{
|
||||
Date: "2026-01-01", Amount: 300,
|
||||
Person: "Alice", Purpose: "other:shirt",
|
||||
Sender: "Bank", Message: "shirt order",
|
||||
}}
|
||||
|
||||
result := Reconcile(members, []string{"2026-01"}, txs, nil, defaultYear)
|
||||
|
||||
if result.Members["Alice"].Months["2026-01"].Paid != 0 {
|
||||
t.Errorf("other: purpose must not touch month ledger")
|
||||
}
|
||||
others := result.Members["Alice"].OtherTransactions
|
||||
if len(others) != 1 {
|
||||
t.Fatalf("want 1 OtherTransaction, got %d", len(others))
|
||||
}
|
||||
if math.Abs(others[0].Amount-300) > 0.01 {
|
||||
t.Errorf("OtherEntry.Amount want 300, got %f", others[0].Amount)
|
||||
}
|
||||
if others[0].Purpose != "other:shirt" {
|
||||
t.Errorf("OtherEntry.Purpose want %q, got %q", "other:shirt", others[0].Purpose)
|
||||
}
|
||||
}
|
||||
|
||||
// [Go] Months outside sortedMonths go to credit, not to the window ledger.
|
||||
func TestReconcileOutOfWindowGoesToCredit(t *testing.T) {
|
||||
t.Parallel()
|
||||
// Window shows only 2026-01. Transaction references 2026-01 (in) and 2026-02 (out).
|
||||
members := []Member{{Name: "Alice", Tier: "A", Fees: map[string]FeeData{"2026-01": {Expected: 600, Attendance: 3}}}}
|
||||
txs := []Transaction{{
|
||||
Date: "2026-01-01", Amount: 1200,
|
||||
Person: "Alice", Purpose: "2026-01, 2026-02",
|
||||
Sender: "Bank", Message: "Q1",
|
||||
}}
|
||||
|
||||
result := Reconcile(members, []string{"2026-01"}, txs, nil, defaultYear)
|
||||
|
||||
// member_share = 1200 (one member)
|
||||
// out_credit = 1200 / 2 * 1 = 600
|
||||
// in_window_share = 600
|
||||
// in_window = [(2026-01, 600)], total_expected = 600 → greedy: paid = 600, no overflow
|
||||
if math.Abs(result.Members["Alice"].Months["2026-01"].Paid-600) > 0.01 {
|
||||
t.Errorf("in-window paid want 600, got %f", result.Members["Alice"].Months["2026-01"].Paid)
|
||||
}
|
||||
// total_balance = int(600) - 600 (window) + 600 (out credit) = 600
|
||||
if result.Members["Alice"].TotalBalance != 600 {
|
||||
t.Errorf("TotalBalance want 600, got %d", result.Members["Alice"].TotalBalance)
|
||||
}
|
||||
}
|
||||
|
||||
// [Go] No person/purpose → inference fallback resolves sender name and date month.
|
||||
func TestReconcileInferenceFallback(t *testing.T) {
|
||||
t.Parallel()
|
||||
members := []Member{{Name: "Tomáš Němeček", Tier: "A", Fees: map[string]FeeData{"2026-04": {Expected: 750, Attendance: 3}}}}
|
||||
txs := []Transaction{{
|
||||
Date: "2026-04-15", Amount: 750,
|
||||
// Person and Purpose are empty → inference path
|
||||
Sender: "Tomas Nemecek",
|
||||
Message: "clenske 04/2026",
|
||||
}}
|
||||
|
||||
result := Reconcile(members, []string{"2026-04"}, txs, nil, defaultYear)
|
||||
|
||||
if math.Abs(result.Members["Tomáš Němeček"].Months["2026-04"].Paid-750) > 0.01 {
|
||||
t.Errorf("inference fallback: want 750 paid, got %f", result.Members["Tomáš Němeček"].Months["2026-04"].Paid)
|
||||
}
|
||||
}
|
||||
|
||||
// [Go] Transaction with no match at all ends up in Unmatched; ledger unchanged.
|
||||
func TestReconcileNoMatchGoesToUnmatched(t *testing.T) {
|
||||
t.Parallel()
|
||||
members := []Member{{Name: "Alice", Tier: "A", Fees: map[string]FeeData{"2026-01": {Expected: 750, Attendance: 3}}}}
|
||||
txs := []Transaction{{
|
||||
Date: "2026-01-01", Amount: 500,
|
||||
// empty person+purpose and sender name not matching any member
|
||||
Sender: "Unknown Corp", Message: "invoice",
|
||||
}}
|
||||
|
||||
result := Reconcile(members, []string{"2026-01"}, txs, nil, defaultYear)
|
||||
|
||||
if len(result.Unmatched) != 1 {
|
||||
t.Errorf("want 1 unmatched, got %d", len(result.Unmatched))
|
||||
}
|
||||
if result.Members["Alice"].Months["2026-01"].Paid != 0 {
|
||||
t.Errorf("unmatched tx must not touch ledger")
|
||||
}
|
||||
}
|
||||
|
||||
// [Go] Empty transaction list leaves every month at paid=0 and balance=–expected.
|
||||
func TestReconcileNoTransactionsAllUnpaid(t *testing.T) {
|
||||
t.Parallel()
|
||||
members := []Member{{Name: "Alice", Tier: "A", Fees: map[string]FeeData{"2026-01": {Expected: 750, Attendance: 3}}}}
|
||||
|
||||
result := Reconcile(members, []string{"2026-01"}, nil, nil, defaultYear)
|
||||
|
||||
if result.Members["Alice"].Months["2026-01"].Paid != 0 {
|
||||
t.Errorf("no txs: want paid=0, got %f", result.Members["Alice"].Months["2026-01"].Paid)
|
||||
}
|
||||
if result.Members["Alice"].TotalBalance != -750 {
|
||||
t.Errorf("no txs: want balance -750, got %d", result.Members["Alice"].TotalBalance)
|
||||
}
|
||||
if len(result.Unmatched) != 0 {
|
||||
t.Errorf("no txs: want empty unmatched, got %v", result.Unmatched)
|
||||
}
|
||||
}
|
||||
65
go/internal/domain/synch/synch.go
Normal file
65
go/internal/domain/synch/synch.go
Normal file
@@ -0,0 +1,65 @@
|
||||
// Package synch ports the bank-sync deduplication helper from
|
||||
// scripts/sync_fio_to_sheets.py.
|
||||
package synch
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Transaction is the projection of a Fio transaction that participates
|
||||
// in the Sync ID hash. Other fields (ks, ss, sender_account, …) are
|
||||
// intentionally excluded — they are not part of the Python hash.
|
||||
//
|
||||
// Currency: leave "" to inherit the Python default of "CZK" (matches
|
||||
// the HTML scraper path which omits the key entirely).
|
||||
type Transaction struct {
|
||||
Date string
|
||||
Amount float64
|
||||
Currency string
|
||||
Sender string
|
||||
VS string
|
||||
Message string
|
||||
BankID string
|
||||
}
|
||||
|
||||
// GenerateSyncID returns the lowercase SHA-256 hex digest of
|
||||
// "date|amount|currency|sender|vs|message|bank_id" (lower-cased), used
|
||||
// as the dedup key in column K of the payments sheet.
|
||||
//
|
||||
// Byte-stable with scripts/sync_fio_to_sheets.py generate_sync_id.
|
||||
func GenerateSyncID(tx Transaction) string {
|
||||
currency := tx.Currency
|
||||
if currency == "" {
|
||||
currency = "CZK"
|
||||
}
|
||||
raw := strings.ToLower(strings.Join([]string{
|
||||
tx.Date,
|
||||
formatAmount(tx.Amount),
|
||||
currency,
|
||||
tx.Sender,
|
||||
tx.VS,
|
||||
tx.Message,
|
||||
tx.BankID,
|
||||
}, "|"))
|
||||
sum := sha256.Sum256([]byte(raw))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// formatAmount mimics Python's str(float) for Fio transaction amounts.
|
||||
// Python uses decimal notation for abs(f) in [1e-4, 1e16) and scientific
|
||||
// notation outside that range, always adding ".0" to whole-valued decimals.
|
||||
func formatAmount(f float64) string {
|
||||
abs := math.Abs(f)
|
||||
if abs != 0 && (abs < 1e-4 || abs >= 1e16) {
|
||||
return strconv.FormatFloat(f, 'e', -1, 64)
|
||||
}
|
||||
s := strconv.FormatFloat(f, 'f', -1, 64)
|
||||
if !strings.ContainsRune(s, '.') {
|
||||
s += ".0"
|
||||
}
|
||||
return s
|
||||
}
|
||||
119
go/internal/domain/synch/synch_test.go
Normal file
119
go/internal/domain/synch/synch_test.go
Normal file
@@ -0,0 +1,119 @@
|
||||
package synch
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// All expected digests verified against the live Python implementation on 2026-05-06:
|
||||
//
|
||||
// PYTHONPATH=scripts:. python -c '
|
||||
// from sync_fio_to_sheets import generate_sync_id
|
||||
// cases = [
|
||||
// {"date":"2026-01-15","amount":500.0,"currency":"CZK","sender":"Jan Novak","vs":"123","message":"clenske 1/2026","bank_id":"abc123"},
|
||||
// {"date":"2026-01-15","amount":500.0,"sender":"Jan Novak","vs":"123","message":"clenske 1/2026","bank_id":"abc123"},
|
||||
// {"date":"2026-02-10","amount":1234.56,"currency":"CZK","sender":"ABC SRO","vs":"","message":"FAKTURA 42","bank_id":"xyz"},
|
||||
// {"date":"2026-03-01","amount":-500.0,"currency":"CZK","sender":"refund","vs":"","message":"","bank_id":""},
|
||||
// {"date":"2026-04-01","amount":0.0,"currency":"CZK","sender":"","vs":"","message":"","bank_id":""},
|
||||
// {"date":"","amount":0.0,"currency":"CZK","sender":"","vs":"","message":"","bank_id":""},
|
||||
// ]
|
||||
// for c in cases: print(generate_sync_id(c))
|
||||
// '
|
||||
func TestGenerateSyncID(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
tx Transaction
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "all fields set",
|
||||
tx: Transaction{
|
||||
Date: "2026-01-15", Amount: 500.0, Currency: "CZK",
|
||||
Sender: "Jan Novak", VS: "123", Message: "clenske 1/2026", BankID: "abc123",
|
||||
},
|
||||
want: "4ac26598b6f23965380690172156a438a7e97a97dcedf222e5afe1afbe2c1bc4",
|
||||
},
|
||||
{
|
||||
name: "currency empty defaults to CZK",
|
||||
tx: Transaction{
|
||||
Date: "2026-01-15", Amount: 500.0, Currency: "",
|
||||
Sender: "Jan Novak", VS: "123", Message: "clenske 1/2026", BankID: "abc123",
|
||||
},
|
||||
want: "4ac26598b6f23965380690172156a438a7e97a97dcedf222e5afe1afbe2c1bc4",
|
||||
},
|
||||
{
|
||||
name: "mixed-case fields lowercased before hashing",
|
||||
tx: Transaction{
|
||||
Date: "2026-02-10", Amount: 1234.56, Currency: "CZK",
|
||||
Sender: "ABC SRO", VS: "", Message: "FAKTURA 42", BankID: "xyz",
|
||||
},
|
||||
want: "d40fa224d4fa572ffcd58e308e5c6508c4d5ca087b24ef6ff9284528fc128250",
|
||||
},
|
||||
{
|
||||
name: "negative amount",
|
||||
tx: Transaction{
|
||||
Date: "2026-03-01", Amount: -500.0, Currency: "CZK",
|
||||
Sender: "refund", VS: "", Message: "", BankID: "",
|
||||
},
|
||||
want: "0c630a407160367c396a2beec08efb94c319b4d84a8b90cc2be89e6ea10c391f",
|
||||
},
|
||||
{
|
||||
name: "zero amount",
|
||||
tx: Transaction{
|
||||
Date: "2026-04-01", Amount: 0.0, Currency: "CZK",
|
||||
Sender: "", VS: "", Message: "", BankID: "",
|
||||
},
|
||||
want: "6a23ce53717cd539064d550d2c2ec5de2e9bf81016d16852820ca9b8e259331f",
|
||||
},
|
||||
{
|
||||
// Python equivalent: {"date":"","amount":0.0,"currency":"CZK","sender":"","vs":"","message":"","bank_id":""}
|
||||
// Note: Python generate_sync_id({}) hashes "" for missing amount, not "0.0".
|
||||
name: "zero-value Transaction",
|
||||
tx: Transaction{},
|
||||
want: "d33d7e391f5a43f0192bb5a34c0ec15715139125678ecef8e1324af7d943b21d",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := GenerateSyncID(tc.tx)
|
||||
if got != tc.want {
|
||||
t.Errorf("GenerateSyncID(%+v) = %q, want %q", tc.tx, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// All expected strings verified against the live Python implementation on 2026-05-06:
|
||||
//
|
||||
// PYTHONPATH=scripts:. python -c '
|
||||
// for v in [0.0, 500.0, -500.0, 0.1, 1234.56, 99999.99, 1500000.0, 1e16, 1e-5]:
|
||||
// print(repr(v), "->", repr(str(v)))
|
||||
// '
|
||||
func TestFormatAmount(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
in float64
|
||||
want string
|
||||
}{
|
||||
{0.0, "0.0"},
|
||||
{500.0, "500.0"},
|
||||
{-500.0, "-500.0"},
|
||||
{0.1, "0.1"},
|
||||
{1234.56, "1234.56"},
|
||||
{99999.99, "99999.99"},
|
||||
{1500000.0, "1500000.0"},
|
||||
{1e16, "1e+16"},
|
||||
{1e-5, "1e-05"},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
got := formatAmount(tc.in)
|
||||
if got != tc.want {
|
||||
t.Errorf("formatAmount(%v) = %q, want %q", tc.in, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
64
go/internal/io/attendance/client.go
Normal file
64
go/internal/io/attendance/client.go
Normal file
@@ -0,0 +1,64 @@
|
||||
// Package attendance fetches attendance CSV exports from Google Sheets.
|
||||
// No auth required — the sheet must be publicly readable.
|
||||
package attendance
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const exportBase = "https://docs.google.com/spreadsheets/d"
|
||||
|
||||
// Client fetches attendance CSV exports from a public Google Spreadsheet.
|
||||
type Client struct {
|
||||
http *http.Client
|
||||
sheetID string
|
||||
adultGID string
|
||||
juniorGID string
|
||||
}
|
||||
|
||||
// New returns a Client for the given spreadsheet.
|
||||
// adultGID is typically "0"; juniorGID is the GID of the junior tab.
|
||||
func New(httpClient *http.Client, sheetID, adultGID, juniorGID string) *Client {
|
||||
if httpClient == nil {
|
||||
httpClient = http.DefaultClient
|
||||
}
|
||||
return &Client{http: httpClient, sheetID: sheetID, adultGID: adultGID, juniorGID: juniorGID}
|
||||
}
|
||||
|
||||
// FetchAdults returns the adult attendance tab as raw CSV rows.
|
||||
func (c *Client) FetchAdults(ctx context.Context) ([][]string, error) {
|
||||
return c.fetch(ctx, c.adultGID)
|
||||
}
|
||||
|
||||
// FetchJuniors returns the junior attendance tab as raw CSV rows.
|
||||
func (c *Client) FetchJuniors(ctx context.Context) ([][]string, error) {
|
||||
return c.fetch(ctx, c.juniorGID)
|
||||
}
|
||||
|
||||
func (c *Client) fetch(ctx context.Context, gid string) ([][]string, error) {
|
||||
url := fmt.Sprintf("%s/%s/export?format=csv&gid=%s", exportBase, c.sheetID, gid)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("attendance fetch: HTTP %d for gid=%s", resp.StatusCode, gid)
|
||||
}
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r := csv.NewReader(strings.NewReader(string(body)))
|
||||
r.FieldsPerRecord = -1 // rows may have different lengths
|
||||
return r.ReadAll()
|
||||
}
|
||||
93
go/internal/io/attendance/client_test.go
Normal file
93
go/internal/io/attendance/client_test.go
Normal file
@@ -0,0 +1,93 @@
|
||||
package attendance
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestClientFetchAdults(t *testing.T) {
|
||||
data, err := os.ReadFile("testdata/adults_minimal.csv")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write(data)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
// Point the client at our test server by re-implementing fetch against its URL.
|
||||
rows, err := fetchURL(context.Background(), srv.Client(), srv.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rows) < 2 {
|
||||
t.Fatalf("want ≥2 rows, got %d", len(rows))
|
||||
}
|
||||
if rows[0][0] != "Jméno" {
|
||||
t.Errorf("unexpected header: %q", rows[0][0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestFake(t *testing.T) {
|
||||
adultRows := parseCSV(t, "testdata/adults_minimal.csv")
|
||||
juniorRows := parseCSV(t, "testdata/juniors_minimal.csv")
|
||||
|
||||
f := &Fake{Adults: adultRows, Juniors: juniorRows}
|
||||
|
||||
got, err := f.FetchAdults(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got[0][0] != "Jméno" {
|
||||
t.Errorf("adults header: %q", got[0][0])
|
||||
}
|
||||
|
||||
got, err = f.FetchJuniors(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got[1][0] != "Junior One" {
|
||||
t.Errorf("juniors first member: %q", got[1][0])
|
||||
}
|
||||
}
|
||||
|
||||
func parseCSV(t *testing.T, path string) [][]string {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
r := csv.NewReader(strings.NewReader(string(b)))
|
||||
r.FieldsPerRecord = -1
|
||||
rows, err := r.ReadAll()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
// fetchURL is a test helper that exercises the shared fetch logic against an arbitrary URL.
|
||||
func fetchURL(ctx context.Context, hc *http.Client, url string) ([][]string, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := hc.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
b, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r := csv.NewReader(strings.NewReader(string(b)))
|
||||
r.FieldsPerRecord = -1
|
||||
return r.ReadAll()
|
||||
}
|
||||
12
go/internal/io/attendance/fake.go
Normal file
12
go/internal/io/attendance/fake.go
Normal file
@@ -0,0 +1,12 @@
|
||||
package attendance
|
||||
|
||||
import "context"
|
||||
|
||||
// Fake is an in-memory replacement for Client, used in tests.
|
||||
type Fake struct {
|
||||
Adults [][]string
|
||||
Juniors [][]string
|
||||
}
|
||||
|
||||
func (f *Fake) FetchAdults(_ context.Context) ([][]string, error) { return f.Adults, nil }
|
||||
func (f *Fake) FetchJuniors(_ context.Context) ([][]string, error) { return f.Juniors, nil }
|
||||
4
go/internal/io/attendance/testdata/adults_minimal.csv
vendored
Normal file
4
go/internal/io/attendance/testdata/adults_minimal.csv
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
Jméno,Tier,,,01.09.2025,08.09.2025,15.09.2025
|
||||
Member One,A,,,TRUE,TRUE,FALSE
|
||||
Member Two,A,,,TRUE,FALSE,FALSE
|
||||
# last line,,,,,
|
||||
|
4
go/internal/io/attendance/testdata/juniors_minimal.csv
vendored
Normal file
4
go/internal/io/attendance/testdata/juniors_minimal.csv
vendored
Normal file
@@ -0,0 +1,4 @@
|
||||
Jméno,Tier,,,01.09.2025,08.09.2025,15.09.2025
|
||||
Junior One,J,,,TRUE,TRUE,TRUE
|
||||
# Trenéři,,,,,
|
||||
Coach One,X,,,FALSE,FALSE,FALSE
|
||||
|
209
go/internal/io/cache/filecache.go
vendored
Normal file
209
go/internal/io/cache/filecache.go
vendored
Normal file
@@ -0,0 +1,209 @@
|
||||
// Package cache implements a Drive-modifiedTime-gated JSON file cache,
|
||||
// mirroring scripts/cache_utils.py.
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"fuj-management/go/internal/io/drive"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DriveClient is the subset of drive.Client used by FileCache.
|
||||
type DriveClient interface {
|
||||
ModifiedTime(ctx context.Context, fileID string) (string, error)
|
||||
}
|
||||
|
||||
type cacheFile struct {
|
||||
ModifiedTime string `json:"modifiedTime"`
|
||||
Data json.RawMessage `json:"data"`
|
||||
CachedAt string `json:"cachedAt"`
|
||||
}
|
||||
|
||||
// FileCache wraps a Drive client to gate JSON file caching on sheet modifiedTime.
|
||||
//
|
||||
// Two TTL knobs mirror scripts/cache_utils.py:
|
||||
// - ttl: if the cache file on disk is younger than this, skip the Drive check entirely.
|
||||
// - apiCheckTTL: debounces in-memory Drive API calls per sheet ID.
|
||||
//
|
||||
// Atomic writes: data is marshaled to a .tmp file then os.Rename'd.
|
||||
// Cache files are compatible with Python's format:
|
||||
//
|
||||
// {"modifiedTime":"…","data":…,"cachedAt":"…"}
|
||||
type FileCache struct {
|
||||
drive DriveClient
|
||||
dir string
|
||||
sheetMap map[string]string // cache key → Drive file ID
|
||||
ttl time.Duration
|
||||
apiCheckTTL time.Duration
|
||||
mu sync.Mutex
|
||||
lastChecked map[string]time.Time
|
||||
}
|
||||
|
||||
// New creates a FileCache.
|
||||
// sheetMap maps cache keys to Google Sheets/Drive file IDs (mirrors CACHE_SHEET_MAP in config).
|
||||
func New(d DriveClient, dir string, sheetMap map[string]string, ttl, apiCheckTTL time.Duration) *FileCache {
|
||||
return &FileCache{
|
||||
drive: d,
|
||||
dir: dir,
|
||||
sheetMap: sheetMap,
|
||||
ttl: ttl,
|
||||
apiCheckTTL: apiCheckTTL,
|
||||
lastChecked: make(map[string]time.Time),
|
||||
}
|
||||
}
|
||||
|
||||
// Get returns the cached value for cacheKey, calling fetch if the cache is stale.
|
||||
// T must be JSON-marshalable.
|
||||
func Get[T any](ctx context.Context, fc *FileCache, cacheKey string, fetch func(context.Context) (T, error)) (T, error) {
|
||||
sheetID := fc.sheetMap[cacheKey]
|
||||
if sheetID == "" {
|
||||
sheetID = cacheKey
|
||||
}
|
||||
cacheFilePath := filepath.Join(fc.dir, cacheKey+"_cache.json")
|
||||
|
||||
currentModTime, err := fc.currentModifiedTime(ctx, sheetID, cacheFilePath)
|
||||
if err != nil {
|
||||
return *new(T), fmt.Errorf("cache: modifiedTime for %s: %w", cacheKey, err)
|
||||
}
|
||||
|
||||
// Try cache hit
|
||||
if data, ok := readCache[T](cacheFilePath, currentModTime); ok {
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// Cache miss — fetch fresh data
|
||||
fresh, err := fetch(ctx)
|
||||
if err != nil {
|
||||
return *new(T), err
|
||||
}
|
||||
if err := writeCache(cacheFilePath, currentModTime, fresh); err != nil {
|
||||
// Non-fatal: log but don't fail the request
|
||||
_, _ = fmt.Fprintf(os.Stderr, "cache: write %s: %v\n", cacheKey, err)
|
||||
}
|
||||
return fresh, nil
|
||||
}
|
||||
|
||||
// Flush deletes all *_cache.json files in the cache dir and resets in-memory state.
|
||||
func (fc *FileCache) Flush() (int, error) {
|
||||
fc.mu.Lock()
|
||||
fc.lastChecked = make(map[string]time.Time)
|
||||
fc.mu.Unlock()
|
||||
|
||||
pattern := filepath.Join(fc.dir, "*_cache.json")
|
||||
matches, err := filepath.Glob(pattern)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, f := range matches {
|
||||
_ = os.Remove(f)
|
||||
}
|
||||
return len(matches), nil
|
||||
}
|
||||
|
||||
// currentModifiedTime returns a stable string representing the current version
|
||||
// of the sheet, using the in-memory + file-mtime TTL guards before hitting Drive.
|
||||
// On Drive failure, falls back to a 5-minute bucket string (matching Python).
|
||||
func (fc *FileCache) currentModifiedTime(ctx context.Context, sheetID, cacheFilePath string) (string, error) {
|
||||
now := time.Now()
|
||||
|
||||
fc.mu.Lock()
|
||||
lastCheck := fc.lastChecked[sheetID]
|
||||
fc.mu.Unlock()
|
||||
|
||||
// Guard 1: in-memory debounce — skip Drive if checked recently
|
||||
if fc.apiCheckTTL > 0 && now.Sub(lastCheck) < fc.apiCheckTTL {
|
||||
if mt, ok := readModifiedTime(cacheFilePath); ok {
|
||||
return mt, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Guard 2: cache file is young enough — trust the stored modifiedTime
|
||||
if fc.ttl > 0 {
|
||||
if info, err := os.Stat(cacheFilePath); err == nil {
|
||||
if now.Sub(info.ModTime()) < fc.ttl {
|
||||
if mt, ok := readModifiedTime(cacheFilePath); ok {
|
||||
fc.mu.Lock()
|
||||
fc.lastChecked[sheetID] = now
|
||||
fc.mu.Unlock()
|
||||
return mt, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Hit Drive API
|
||||
mt, err := fc.drive.ModifiedTime(ctx, sheetID)
|
||||
if err != nil {
|
||||
// Fallback: 5-minute bucket string, matches Python _fallback_ttl()
|
||||
bucket := time.Now().Unix() / 300
|
||||
return fmt.Sprintf("ttl-5m-%d", bucket), nil
|
||||
}
|
||||
fc.mu.Lock()
|
||||
fc.lastChecked[sheetID] = now
|
||||
fc.mu.Unlock()
|
||||
return mt, nil
|
||||
}
|
||||
|
||||
func readModifiedTime(path string) (string, bool) {
|
||||
cf, ok := readCacheFile(path)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
return cf.ModifiedTime, cf.ModifiedTime != ""
|
||||
}
|
||||
|
||||
func readCacheFile(path string) (cacheFile, bool) {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return cacheFile{}, false
|
||||
}
|
||||
var cf cacheFile
|
||||
if err := json.Unmarshal(b, &cf); err != nil {
|
||||
return cacheFile{}, false
|
||||
}
|
||||
return cf, true
|
||||
}
|
||||
|
||||
func readCache[T any](path, currentModTime string) (T, bool) {
|
||||
cf, ok := readCacheFile(path)
|
||||
if !ok || cf.ModifiedTime != currentModTime {
|
||||
return *new(T), false
|
||||
}
|
||||
var v T
|
||||
if err := json.Unmarshal(cf.Data, &v); err != nil {
|
||||
return *new(T), false
|
||||
}
|
||||
return v, true
|
||||
}
|
||||
|
||||
func writeCache(path, modTime string, data any) error {
|
||||
raw, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cf := cacheFile{
|
||||
ModifiedTime: modTime,
|
||||
Data: json.RawMessage(raw),
|
||||
CachedAt: time.Now().Format(time.RFC3339),
|
||||
}
|
||||
b, err := json.Marshal(cf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := path + ".tmp"
|
||||
if err := os.WriteFile(tmp, b, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, path)
|
||||
}
|
||||
|
||||
// Ensure *drive.Client satisfies DriveClient at compile time.
|
||||
var _ DriveClient = (*drive.Client)(nil)
|
||||
125
go/internal/io/cache/filecache_test.go
vendored
Normal file
125
go/internal/io/cache/filecache_test.go
vendored
Normal file
@@ -0,0 +1,125 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fuj-management/go/internal/io/drive"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestGet_FreshFetch(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
d := &drive.Fake{Times: map[string]string{"sheet1": "2026-01-01T00:00:00Z"}}
|
||||
fc := New(d, dir, map[string]string{"mykey": "sheet1"}, time.Minute, time.Minute)
|
||||
|
||||
calls := 0
|
||||
got, err := Get(context.Background(), fc, "mykey", func(_ context.Context) ([]string, error) {
|
||||
calls++
|
||||
return []string{"a", "b"}, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got) != 2 || got[0] != "a" {
|
||||
t.Errorf("unexpected: %v", got)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Errorf("want 1 fetch call, got %d", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGet_CacheHit(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
d := &drive.Fake{Times: map[string]string{"sheet1": "2026-01-01T00:00:00Z"}}
|
||||
fc := New(d, dir, map[string]string{"mykey": "sheet1"}, time.Minute, time.Minute)
|
||||
|
||||
fetch := func(_ context.Context) ([]string, error) { return []string{"a"}, nil }
|
||||
if _, err := Get(context.Background(), fc, "mykey", fetch); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Second call — modifiedTime unchanged, should hit cache
|
||||
calls := 0
|
||||
got, err := Get(context.Background(), fc, "mykey", func(_ context.Context) ([]string, error) {
|
||||
calls++
|
||||
return []string{"SHOULD_NOT_CALL"}, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got[0] != "a" {
|
||||
t.Errorf("want cache hit with 'a', got %q", got[0])
|
||||
}
|
||||
if calls != 0 {
|
||||
t.Errorf("want 0 fetch calls on hit, got %d", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGet_CacheMiss_OnModifiedTimeChange(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
d := &drive.Fake{Times: map[string]string{"sheet1": "2026-01-01T00:00:00Z"}}
|
||||
// No TTL guards so we always hit Drive
|
||||
fc := New(d, dir, map[string]string{"mykey": "sheet1"}, 0, 0)
|
||||
|
||||
fetch := func(_ context.Context) ([]string, error) { return []string{"v1"}, nil }
|
||||
if _, err := Get(context.Background(), fc, "mykey", fetch); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Sheet updated — change modifiedTime
|
||||
d.Times["sheet1"] = "2026-02-01T00:00:00Z"
|
||||
got, err := Get(context.Background(), fc, "mykey", func(_ context.Context) ([]string, error) {
|
||||
return []string{"v2"}, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got[0] != "v2" {
|
||||
t.Errorf("want v2 after sheet update, got %q", got[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestGet_DriveFailureFallback(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
d := &drive.Fake{Err: errors.New("drive down")}
|
||||
fc := New(d, dir, nil, 0, 0)
|
||||
|
||||
calls := 0
|
||||
_, err := Get(context.Background(), fc, "mykey", func(_ context.Context) ([]string, error) {
|
||||
calls++
|
||||
return []string{"fallback"}, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Errorf("want 1 fetch call, got %d", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFlush(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
d := &drive.Fake{Times: map[string]string{"sheet1": "t1"}}
|
||||
fc := New(d, dir, map[string]string{"k": "sheet1"}, 0, 0)
|
||||
|
||||
if _, err := Get(context.Background(), fc, "k", func(_ context.Context) (int, error) { return 42, nil }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
n, err := fc.Flush()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Errorf("want 1 deleted file, got %d", n)
|
||||
}
|
||||
// Cache dir should be empty of _cache.json files
|
||||
entries, _ := os.ReadDir(dir)
|
||||
for _, e := range entries {
|
||||
if e.Name() != "" {
|
||||
t.Errorf("expected empty dir after flush, found %s", e.Name())
|
||||
}
|
||||
}
|
||||
}
|
||||
46
go/internal/io/drive/client.go
Normal file
46
go/internal/io/drive/client.go
Normal file
@@ -0,0 +1,46 @@
|
||||
// Package drive provides a thin wrapper around the Google Drive v3 API,
|
||||
// used only to read modifiedTime for cache invalidation.
|
||||
package drive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"google.golang.org/api/drive/v3"
|
||||
"google.golang.org/api/option"
|
||||
)
|
||||
|
||||
// Client wraps the Drive v3 API, scoped to read-only modifiedTime checks.
|
||||
type Client struct {
|
||||
svc *drive.Service
|
||||
}
|
||||
|
||||
// New builds a Client using a service-account credentials file.
|
||||
// timeout applies to each Drive API call.
|
||||
func New(ctx context.Context, credentialsPath string, timeout time.Duration) (*Client, error) {
|
||||
hc := &http.Client{Timeout: timeout}
|
||||
svc, err := drive.NewService(ctx,
|
||||
option.WithCredentialsFile(credentialsPath), //nolint:staticcheck
|
||||
option.WithScopes(drive.DriveReadonlyScope),
|
||||
option.WithHTTPClient(hc),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Client{svc: svc}, nil
|
||||
}
|
||||
|
||||
// ModifiedTime returns the RFC3339 modifiedTime for the given Drive file ID.
|
||||
// Returns ("", err) if the Drive API call fails.
|
||||
func (c *Client) ModifiedTime(ctx context.Context, fileID string) (string, error) {
|
||||
meta, err := c.svc.Files.Get(fileID).
|
||||
Fields("modifiedTime").
|
||||
SupportsAllDrives(true).
|
||||
Context(ctx).
|
||||
Do()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return meta.ModifiedTime, nil
|
||||
}
|
||||
18
go/internal/io/drive/fake.go
Normal file
18
go/internal/io/drive/fake.go
Normal file
@@ -0,0 +1,18 @@
|
||||
package drive
|
||||
|
||||
import "context"
|
||||
|
||||
// Fake is an in-memory replacement for Client used in tests.
|
||||
type Fake struct {
|
||||
// Times maps file ID → modifiedTime string returned by ModifiedTime.
|
||||
Times map[string]string
|
||||
// Err, if non-nil, is returned instead of looking up Times.
|
||||
Err error
|
||||
}
|
||||
|
||||
func (f *Fake) ModifiedTime(_ context.Context, fileID string) (string, error) {
|
||||
if f.Err != nil {
|
||||
return "", f.Err
|
||||
}
|
||||
return f.Times[fileID], nil
|
||||
}
|
||||
136
go/internal/io/fio/api.go
Normal file
136
go/internal/io/fio/api.go
Normal file
@@ -0,0 +1,136 @@
|
||||
package fio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// httpDoer is the subset of *http.Client used by both Fio impls.
|
||||
type httpDoer interface {
|
||||
Do(*http.Request) (*http.Response, error)
|
||||
}
|
||||
|
||||
// apiClient fetches transactions from the Fio REST API (JSON).
|
||||
// Ports scripts/fio_utils.py fetch_transactions_api.
|
||||
type apiClient struct {
|
||||
token string
|
||||
hc httpDoer
|
||||
}
|
||||
|
||||
func (c *apiClient) FetchTransactions(ctx context.Context, from, to time.Time) ([]Transaction, error) {
|
||||
const layout = "2006-01-02"
|
||||
url := fmt.Sprintf("https://fioapi.fio.cz/v1/rest/periods/%s/%s/%s/transactions.json",
|
||||
c.token, from.Format(layout), to.Format(layout))
|
||||
slog.Debug("fio api: GET",
|
||||
"url", strings.Replace(url, c.token, "****", 1),
|
||||
"from", from.Format(layout), "to", to.Format(layout))
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := c.hc.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
slog.Debug("fio api: response", "status", resp.StatusCode)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("fio api: HTTP %d", resp.StatusCode)
|
||||
}
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
txns, err := parseAPIResponse(body)
|
||||
slog.Debug("fio api: parsed", "body_bytes", len(body), "parsed_count", len(txns))
|
||||
return txns, err
|
||||
}
|
||||
|
||||
// fioAPIResponse is the top-level envelope from the Fio JSON API.
|
||||
type fioAPIResponse struct {
|
||||
AccountStatement struct {
|
||||
TransactionList struct {
|
||||
Transaction []map[string]json.RawMessage `json:"transaction"`
|
||||
} `json:"transactionList"`
|
||||
} `json:"accountStatement"`
|
||||
}
|
||||
|
||||
func parseAPIResponse(body []byte) ([]Transaction, error) {
|
||||
var resp fioAPIResponse
|
||||
if err := json.Unmarshal(body, &resp); err != nil {
|
||||
return nil, fmt.Errorf("fio api: parse JSON: %w", err)
|
||||
}
|
||||
|
||||
var txns []Transaction
|
||||
for _, raw := range resp.AccountStatement.TransactionList.Transaction {
|
||||
amount := colFloat(raw, "column1")
|
||||
if amount <= 0 {
|
||||
continue // skip outgoing
|
||||
}
|
||||
dateRaw := colString(raw, "column0")
|
||||
dateStr := ""
|
||||
if len(dateRaw) >= 10 {
|
||||
dateStr = dateRaw[:10]
|
||||
}
|
||||
txns = append(txns, Transaction{
|
||||
Date: dateStr,
|
||||
Amount: amount,
|
||||
Sender: colString(raw, "column10"),
|
||||
Message: colString(raw, "column16"),
|
||||
VS: colString(raw, "column5"),
|
||||
KS: colString(raw, "column4"),
|
||||
SS: colString(raw, "column6"),
|
||||
UserID: colString(raw, "column7"),
|
||||
SenderAccount: colString(raw, "column2"),
|
||||
BankID: colString(raw, "column22"),
|
||||
Currency: colStringOr(raw, "column14", "CZK"),
|
||||
})
|
||||
}
|
||||
return txns, nil
|
||||
}
|
||||
|
||||
// colString extracts {"value":…} as a string from a column map.
|
||||
func colString(m map[string]json.RawMessage, col string) string {
|
||||
raw, ok := m[col]
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
var cell struct {
|
||||
Value *string `json:"value"`
|
||||
}
|
||||
if json.Unmarshal(raw, &cell) != nil || cell.Value == nil {
|
||||
return ""
|
||||
}
|
||||
return *cell.Value
|
||||
}
|
||||
|
||||
// colStringOr is colString with a fallback value.
|
||||
func colStringOr(m map[string]json.RawMessage, col, fallback string) string {
|
||||
if v := colString(m, col); v != "" {
|
||||
return v
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
// colFloat extracts {"value":…} as a float64 from a column map.
|
||||
// Returns 0 on any error (null column, non-numeric value).
|
||||
func colFloat(m map[string]json.RawMessage, col string) float64 {
|
||||
raw, ok := m[col]
|
||||
if !ok {
|
||||
return 0
|
||||
}
|
||||
var cell struct {
|
||||
Value *float64 `json:"value"`
|
||||
}
|
||||
if json.Unmarshal(raw, &cell) != nil || cell.Value == nil {
|
||||
return 0
|
||||
}
|
||||
return *cell.Value
|
||||
}
|
||||
45
go/internal/io/fio/client.go
Normal file
45
go/internal/io/fio/client.go
Normal file
@@ -0,0 +1,45 @@
|
||||
// Package fio fetches Fio bank transactions via the JSON API or the
|
||||
// transparent-page HTML scraper, behind a common Client interface.
|
||||
package fio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Transaction is one incoming bank payment. Fields absent from the HTML scraper
|
||||
// (BankID, Currency, UserID, SenderAccount) are empty strings on that path.
|
||||
type Transaction struct {
|
||||
Date string
|
||||
Amount float64
|
||||
Sender string
|
||||
Message string
|
||||
VS string
|
||||
KS string
|
||||
SS string
|
||||
UserID string // column7; empty on HTML path
|
||||
SenderAccount string // column2; empty on HTML path
|
||||
BankID string // column22; empty on HTML path
|
||||
Currency string // column14; empty on HTML path (assume CZK)
|
||||
}
|
||||
|
||||
// Client fetches transactions for a date window.
|
||||
type Client interface {
|
||||
FetchTransactions(ctx context.Context, from, to time.Time) ([]Transaction, error)
|
||||
}
|
||||
|
||||
// New returns an apiClient when token is non-empty, otherwise a transparentClient.
|
||||
// hc may be nil, in which case http.DefaultClient is used.
|
||||
func New(token, accountNum string, hc httpDoer) Client {
|
||||
if hc == nil {
|
||||
hc = http.DefaultClient
|
||||
}
|
||||
if token != "" {
|
||||
slog.Debug("fio: client selected", "type", "api")
|
||||
return &apiClient{token: token, hc: hc}
|
||||
}
|
||||
slog.Debug("fio: client selected", "type", "transparent", "account_num", accountNum)
|
||||
return &transparentClient{accountNum: accountNum, hc: hc}
|
||||
}
|
||||
19
go/internal/io/fio/fake.go
Normal file
19
go/internal/io/fio/fake.go
Normal file
@@ -0,0 +1,19 @@
|
||||
package fio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Fake is an in-memory replacement for Client, used in tests.
|
||||
type Fake struct {
|
||||
Transactions []Transaction
|
||||
Err error
|
||||
}
|
||||
|
||||
func (f *Fake) FetchTransactions(_ context.Context, _, _ time.Time) ([]Transaction, error) {
|
||||
if f.Err != nil {
|
||||
return nil, f.Err
|
||||
}
|
||||
return f.Transactions, nil
|
||||
}
|
||||
178
go/internal/io/fio/fio_test.go
Normal file
178
go/internal/io/fio/fio_test.go
Normal file
@@ -0,0 +1,178 @@
|
||||
package fio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestAPIClient_ParseResponse(t *testing.T) {
|
||||
body, err := os.ReadFile("testdata/api_response.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
txns, err := parseAPIResponse(body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(txns) != 1 {
|
||||
t.Fatalf("want 1 txn (outgoing filtered), got %d", len(txns))
|
||||
}
|
||||
tx := txns[0]
|
||||
if tx.Date != "2026-04-10" {
|
||||
t.Errorf("date: want '2026-04-10', got %q", tx.Date)
|
||||
}
|
||||
if tx.Amount != 750 {
|
||||
t.Errorf("amount: want 750, got %v", tx.Amount)
|
||||
}
|
||||
if tx.Sender != "Jana Novakova" {
|
||||
t.Errorf("sender: want 'Jana Novakova', got %q", tx.Sender)
|
||||
}
|
||||
if tx.Message != "duben 2026" {
|
||||
t.Errorf("message: want 'duben 2026', got %q", tx.Message)
|
||||
}
|
||||
if tx.VS != "123" {
|
||||
t.Errorf("vs: want '123', got %q", tx.VS)
|
||||
}
|
||||
if tx.BankID != "12345678901" {
|
||||
t.Errorf("bank_id: want '12345678901', got %q", tx.BankID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPIClient_HTTPRoundTrip(t *testing.T) {
|
||||
body, _ := os.ReadFile("testdata/api_response.json")
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write(body)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := &apiClient{token: "TESTTOKEN", hc: &overrideClient{base: srv.Client(), baseURL: srv.URL}}
|
||||
txns, err := c.FetchTransactions(context.Background(), time.Now().AddDate(0, -1, 0), time.Now())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(txns) != 1 {
|
||||
t.Fatalf("want 1 txn, got %d", len(txns))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransparentClient_ParseHTML(t *testing.T) {
|
||||
body, err := os.ReadFile("testdata/transparent.html")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
txns, err := parseTransparentHTML(body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Only the incoming row (750 CZK) should be kept; -200 is outgoing
|
||||
if len(txns) != 1 {
|
||||
t.Fatalf("want 1 txn (outgoing filtered), got %d", len(txns))
|
||||
}
|
||||
tx := txns[0]
|
||||
if tx.Date != "2026-04-10" {
|
||||
t.Errorf("date: want '2026-04-10', got %q", tx.Date)
|
||||
}
|
||||
if tx.Amount != 750 {
|
||||
t.Errorf("amount: want 750, got %v", tx.Amount)
|
||||
}
|
||||
if tx.Sender != "Jana Novakova" {
|
||||
t.Errorf("sender: want 'Jana Novakova', got %q", tx.Sender)
|
||||
}
|
||||
if tx.VS != "123" {
|
||||
t.Errorf("vs: want '123', got %q", tx.VS)
|
||||
}
|
||||
if tx.BankID != "" {
|
||||
t.Errorf("bank_id: want empty on HTML path, got %q", tx.BankID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCzechDate(t *testing.T) {
|
||||
cases := []struct{ in, want string }{
|
||||
{"10.04.2026", "2026-04-10"},
|
||||
{"10/04/2026", "2026-04-10"},
|
||||
{"7.5.2026", "2026-05-07"}, // non-padded — real Fio transparent page format
|
||||
{"3.12.2025", "2025-12-03"}, // non-padded single-digit day, double-digit month
|
||||
{"07.05.26", "2026-05-07"}, // padded 2-digit year — current Fio transparent page format
|
||||
{"7.5.26", "2026-05-07"}, // non-padded 2-digit year
|
||||
{"07/05/26", "2026-05-07"}, // slash variant
|
||||
{"", ""},
|
||||
{"invalid", ""},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := parseCzechDate(c.in); got != c.want {
|
||||
t.Errorf("parseCzechDate(%q) = %q, want %q", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractSecondTableRows_NestedTable(t *testing.T) {
|
||||
// Regression: a nested <table> inside the target must not cause early exit.
|
||||
html := `<table class="table"><tr><td>nav</td></tr></table>
|
||||
<table class="table">
|
||||
<thead><tr><th>Date</th></tr></thead>
|
||||
<tbody>
|
||||
<tr><td>7.5.2026</td><td><table><tr><td>nested</td></tr></table></td></tr>
|
||||
<tr><td>6.5.2026</td><td></td></tr>
|
||||
</tbody>
|
||||
</table>`
|
||||
rows := extractSecondTableRows([]byte(html))
|
||||
if len(rows) != 2 {
|
||||
t.Errorf("want 2 data rows, got %d: %v", len(rows), rows)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCzechAmount(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
want float64
|
||||
}{
|
||||
{"750,00 CZK", 750},
|
||||
{"1.500,00", 1500},
|
||||
{"1500.00", 1500},
|
||||
{"-200,00 CZK", -200},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := parseCzechAmount(c.in); got != c.want {
|
||||
t.Errorf("parseCzechAmount(%q) = %v, want %v", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFake(t *testing.T) {
|
||||
f := &Fake{Transactions: []Transaction{{Date: "2026-04-01", Amount: 500}}}
|
||||
txns, err := f.FetchTransactions(context.Background(), time.Now(), time.Now())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(txns) != 1 || txns[0].Date != "2026-04-01" {
|
||||
t.Errorf("unexpected: %v", txns)
|
||||
}
|
||||
}
|
||||
|
||||
// overrideClient replaces the URL in requests so we can hit a local test server
|
||||
// instead of the real Fio URL.
|
||||
type overrideClient struct {
|
||||
base *http.Client
|
||||
baseURL string
|
||||
}
|
||||
|
||||
func (o *overrideClient) Do(req *http.Request) (*http.Response, error) {
|
||||
r2, _ := http.NewRequestWithContext(req.Context(), req.Method, o.baseURL+req.URL.Path, nil)
|
||||
resp, err := o.base.Do(r2)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The api client reads the body, so re-serve whatever the test server returned.
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// verify Fake satisfies Client
|
||||
var _ Client = (*Fake)(nil)
|
||||
|
||||
// ensure io.ReadAll isn't called at top level (compile-time reference suppressor)
|
||||
var _ = io.ReadAll
|
||||
29
go/internal/io/fio/testdata/api_response.json
vendored
Normal file
29
go/internal/io/fio/testdata/api_response.json
vendored
Normal file
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"accountStatement": {
|
||||
"transactionList": {
|
||||
"transaction": [
|
||||
{
|
||||
"column0": {"value": "2026-04-10+0200", "name": "Datum", "id": 0},
|
||||
"column1": {"value": 750.0, "name": "Objem", "id": 1},
|
||||
"column2": {"value": "123456789/0300", "name": "Protiúčet", "id": 2},
|
||||
"column4": {"value": "0308", "name": "KS", "id": 4},
|
||||
"column5": {"value": "123", "name": "VS", "id": 5},
|
||||
"column6": {"value": "", "name": "SS", "id": 6},
|
||||
"column7": {"value": "Jana Nováková", "name": "Uživatelská identifikace", "id": 7},
|
||||
"column10": {"value": "Jana Novakova", "name": "Název protiúčtu", "id": 10},
|
||||
"column14": {"value": "CZK", "name": "Měna", "id": 14},
|
||||
"column16": {"value": "duben 2026", "name": "Zpráva pro příjemce", "id": 16},
|
||||
"column22": {"value": "12345678901", "name": "ID operace", "id": 22}
|
||||
},
|
||||
{
|
||||
"column0": {"value": "2026-04-11+0200", "name": "Datum", "id": 0},
|
||||
"column1": {"value": -200.0, "name": "Objem", "id": 1},
|
||||
"column10": {"value": "Outgoing", "name": "Název protiúčtu", "id": 10},
|
||||
"column14": {"value": "CZK", "name": "Měna", "id": 14},
|
||||
"column16": {"value": "", "name": "Zpráva pro příjemce", "id": 16},
|
||||
"column22": {"value": "99999999999", "name": "ID operace", "id": 22}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
37
go/internal/io/fio/testdata/transparent.html
vendored
Normal file
37
go/internal/io/fio/testdata/transparent.html
vendored
Normal file
@@ -0,0 +1,37 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>
|
||||
<!-- First table (ignored) -->
|
||||
<table class="table"><tr><td>ignored</td></tr></table>
|
||||
<!-- Second table (target) -->
|
||||
<table class="table">
|
||||
<thead>
|
||||
<tr><th>Datum</th><th>Částka</th><th>Typ</th><th>Název protiúčtu</th><th>Zpráva</th><th>KS</th><th>VS</th><th>SS</th><th>Poznámka</th></tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>10.04.2026</td>
|
||||
<td>750,00 CZK</td>
|
||||
<td>Příjem</td>
|
||||
<td>Jana Novakova</td>
|
||||
<td>duben 2026</td>
|
||||
<td>0308</td>
|
||||
<td>123</td>
|
||||
<td></td>
|
||||
<td></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>09.04.2026</td>
|
||||
<td>-200,00 CZK</td>
|
||||
<td>Odchozí</td>
|
||||
<td>Someone</td>
|
||||
<td>outgoing</td>
|
||||
<td></td>
|
||||
<td></td>
|
||||
<td></td>
|
||||
<td></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
247
go/internal/io/fio/transparent.go
Normal file
247
go/internal/io/fio/transparent.go
Normal file
@@ -0,0 +1,247 @@
|
||||
package fio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
ghtml "golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// transparentClient fetches transactions from the Fio transparent account page (HTML).
|
||||
// Ports scripts/fio_utils.py FioTableParser + fetch_transactions_transparent.
|
||||
type transparentClient struct {
|
||||
accountNum string
|
||||
hc httpDoer
|
||||
}
|
||||
|
||||
func (c *transparentClient) FetchTransactions(ctx context.Context, from, to time.Time) ([]Transaction, error) {
|
||||
// Transparent page date format: D.M.YYYY
|
||||
url := fmt.Sprintf(
|
||||
"https://ib.fio.cz/ib/transparent?a=%s&f=%s&t=%s",
|
||||
c.accountNum,
|
||||
from.Format("2.1.2006"),
|
||||
to.Format("2.1.2006"),
|
||||
)
|
||||
slog.Debug("fio transparent: GET",
|
||||
"url", url,
|
||||
"from", from.Format("2006-01-02"), "to", to.Format("2006-01-02"))
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := c.hc.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
slog.Debug("fio transparent: response", "status", resp.StatusCode)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("fio transparent: HTTP %d", resp.StatusCode)
|
||||
}
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slog.Debug("fio transparent: body read", "body_bytes", len(body))
|
||||
return parseTransparentHTML(body)
|
||||
}
|
||||
|
||||
// Column indices in the transparent-page table (0-based).
|
||||
// Datum | Částka | Typ | Název protiúčtu | Zpráva pro příjemce | KS | VS | SS | Poznámka
|
||||
const (
|
||||
tColDate = 0
|
||||
tColAmount = 1
|
||||
tColSender = 3
|
||||
tColMessage = 4
|
||||
tColKS = 5
|
||||
tColVS = 6
|
||||
tColSS = 7
|
||||
)
|
||||
|
||||
func parseTransparentHTML(body []byte) ([]Transaction, error) {
|
||||
rows := extractSecondTableRows(body)
|
||||
|
||||
var txns []Transaction
|
||||
var droppedBadDate, droppedNonpositive int
|
||||
for _, row := range rows {
|
||||
col := func(i int) string {
|
||||
if i < len(row) {
|
||||
return strings.TrimSpace(row[i])
|
||||
}
|
||||
return ""
|
||||
}
|
||||
dateStr := parseCzechDate(col(tColDate))
|
||||
amount := parseCzechAmount(col(tColAmount))
|
||||
if dateStr == "" {
|
||||
droppedBadDate++
|
||||
continue
|
||||
}
|
||||
if amount <= 0 {
|
||||
droppedNonpositive++
|
||||
continue
|
||||
}
|
||||
txns = append(txns, Transaction{
|
||||
Date: dateStr,
|
||||
Amount: amount,
|
||||
Sender: col(tColSender),
|
||||
Message: col(tColMessage),
|
||||
KS: col(tColKS),
|
||||
VS: col(tColVS),
|
||||
SS: col(tColSS),
|
||||
BankID: "", // not available on HTML path
|
||||
})
|
||||
}
|
||||
slog.Debug("fio transparent: parsed",
|
||||
"raw_rows", len(rows),
|
||||
"kept", len(txns),
|
||||
"dropped_bad_date", droppedBadDate,
|
||||
"dropped_nonpositive_amount", droppedNonpositive)
|
||||
return txns, nil
|
||||
}
|
||||
|
||||
// extractSecondTableRows walks the HTML token stream and returns data rows
|
||||
// from the second <table class="table"> element, skipping the <thead>.
|
||||
// It tracks nesting depth so that nested <table> elements inside the target
|
||||
// do not trigger an early exit.
|
||||
func extractSecondTableRows(body []byte) [][]string {
|
||||
z := ghtml.NewTokenizer(strings.NewReader(string(body)))
|
||||
|
||||
tableCount := 0
|
||||
targetDepth := 0 // >0 while inside the target table (handles nesting)
|
||||
inThead := false
|
||||
inRow := false
|
||||
inCell := false
|
||||
var currentRow []string
|
||||
var cellBuf strings.Builder
|
||||
var rows [][]string
|
||||
|
||||
for {
|
||||
tt := z.Next()
|
||||
if tt == ghtml.ErrorToken {
|
||||
break
|
||||
}
|
||||
switch tt {
|
||||
case ghtml.StartTagToken:
|
||||
t := z.Token()
|
||||
switch t.Data {
|
||||
case "table":
|
||||
if targetDepth > 0 {
|
||||
targetDepth++ // nested table inside target; track so </table> doesn't exit early
|
||||
} else if hasClass(t, "table") {
|
||||
tableCount++
|
||||
if tableCount == 2 {
|
||||
targetDepth = 1
|
||||
}
|
||||
}
|
||||
case "thead":
|
||||
if targetDepth > 0 {
|
||||
inThead = true
|
||||
}
|
||||
case "tr":
|
||||
if targetDepth > 0 && !inThead {
|
||||
inRow = true
|
||||
currentRow = nil
|
||||
}
|
||||
case "td", "th":
|
||||
if inRow {
|
||||
inCell = true
|
||||
cellBuf.Reset()
|
||||
}
|
||||
}
|
||||
case ghtml.EndTagToken:
|
||||
t := z.Token()
|
||||
switch t.Data {
|
||||
case "td", "th":
|
||||
if inCell {
|
||||
currentRow = append(currentRow, cellBuf.String())
|
||||
inCell = false
|
||||
}
|
||||
case "thead":
|
||||
inThead = false
|
||||
case "tr":
|
||||
if inRow {
|
||||
if len(currentRow) > 0 {
|
||||
rows = append(rows, currentRow)
|
||||
}
|
||||
inRow = false
|
||||
}
|
||||
case "table":
|
||||
if targetDepth > 0 {
|
||||
targetDepth--
|
||||
if targetDepth == 0 {
|
||||
return rows
|
||||
}
|
||||
}
|
||||
}
|
||||
case ghtml.TextToken:
|
||||
if inCell {
|
||||
cellBuf.WriteString(z.Token().Data)
|
||||
}
|
||||
}
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
func hasClass(t ghtml.Token, cls string) bool {
|
||||
for _, a := range t.Attr {
|
||||
if a.Key == "class" {
|
||||
for _, c := range strings.Fields(a.Val) {
|
||||
if c == cls {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// parseCzechDate parses Czech date strings → "YYYY-MM-DD".
|
||||
// Handles both zero-padded ("07.05.2026") and non-padded ("7.5.2026") variants
|
||||
// with dot or slash separators, as the Fio transparent page omits leading zeros.
|
||||
// Returns "" on parse error.
|
||||
func parseCzechDate(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
for _, layout := range []string{
|
||||
"2.1.2006", "02.01.2006", "2/1/2006", "02/01/2006",
|
||||
"2.1.06", "02.01.06", "2/1/06", "02/01/06",
|
||||
} {
|
||||
if t, err := time.Parse(layout, s); err == nil {
|
||||
return t.Format("2006-01-02")
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
var nonNumericRe = regexp.MustCompile(`[^\d.,]`)
|
||||
|
||||
// parseCzechAmount parses "1 500,00 CZK" / "1.500,00" / "1500.00" → float64.
|
||||
// Returns 0 on error.
|
||||
func parseCzechAmount(s string) float64 {
|
||||
// Remove NBSP, regular spaces, currency letters
|
||||
s = strings.Map(func(r rune) rune {
|
||||
if r == ' ' || unicode.IsSpace(r) || unicode.IsLetter(r) {
|
||||
return -1
|
||||
}
|
||||
return r
|
||||
}, s)
|
||||
|
||||
if strings.Contains(s, ",") {
|
||||
// Czech decimal: 1.500,00 → remove dots (thousand sep), comma → dot
|
||||
s = strings.ReplaceAll(s, ".", "")
|
||||
s = strings.ReplaceAll(s, ",", ".")
|
||||
} else {
|
||||
// Remove any remaining non-numeric except one dot
|
||||
s = nonNumericRe.ReplaceAllString(s, "")
|
||||
}
|
||||
var f float64
|
||||
_, _ = fmt.Sscanf(s, "%f", &f)
|
||||
return f
|
||||
}
|
||||
124
go/internal/io/sheets/client.go
Normal file
124
go/internal/io/sheets/client.go
Normal file
@@ -0,0 +1,124 @@
|
||||
// Package sheets provides a typed wrapper around the Google Sheets v4 API.
|
||||
package sheets
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"google.golang.org/api/option"
|
||||
sheetsv4 "google.golang.org/api/sheets/v4"
|
||||
)
|
||||
|
||||
// ValueRange pairs an R1C1 range with its cell values, used for batchUpdate.
|
||||
type ValueRange struct {
|
||||
Range string // R1C1 notation, e.g. "R2C4:R2C6"
|
||||
Values [][]any // one sub-slice per row
|
||||
}
|
||||
|
||||
// Client wraps the Sheets v4 API with the operations needed by this project.
|
||||
type Client struct {
|
||||
svc *sheetsv4.Service
|
||||
}
|
||||
|
||||
// New builds a Client using a service-account credentials file.
|
||||
func New(ctx context.Context, credentialsPath string, _ time.Duration) (*Client, error) {
|
||||
svc, err := sheetsv4.NewService(ctx,
|
||||
option.WithCredentialsFile(credentialsPath), //nolint:staticcheck
|
||||
option.WithScopes(sheetsv4.SpreadsheetsScope),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Client{svc: svc}, nil
|
||||
}
|
||||
|
||||
// GetValues fetches a range from a spreadsheet with UNFORMATTED_VALUE rendering
|
||||
// (numbers as numbers, dates as serial floats — matching Python's behaviour).
|
||||
func (c *Client) GetValues(ctx context.Context, spreadsheetID, a1Range string) ([][]any, error) {
|
||||
resp, err := c.svc.Spreadsheets.Values.
|
||||
Get(spreadsheetID, a1Range).
|
||||
ValueRenderOption("UNFORMATTED_VALUE").
|
||||
Context(ctx).
|
||||
Do()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows := make([][]any, len(resp.Values))
|
||||
copy(rows, resp.Values)
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
// AppendValues appends rows to the first empty row after a1Range.
|
||||
func (c *Client) AppendValues(ctx context.Context, spreadsheetID, a1Range string, rows [][]any) error {
|
||||
vals := make([][]any, len(rows))
|
||||
copy(vals, rows)
|
||||
_, err := c.svc.Spreadsheets.Values.
|
||||
Append(spreadsheetID, a1Range, &sheetsv4.ValueRange{Values: vals}).
|
||||
ValueInputOption("USER_ENTERED").
|
||||
Context(ctx).
|
||||
Do()
|
||||
return err
|
||||
}
|
||||
|
||||
// BatchUpdateValues writes multiple non-contiguous ranges in one API call.
|
||||
func (c *Client) BatchUpdateValues(ctx context.Context, spreadsheetID string, updates []ValueRange) error {
|
||||
data := make([]*sheetsv4.ValueRange, len(updates))
|
||||
for i, u := range updates {
|
||||
vals := make([][]any, len(u.Values))
|
||||
copy(vals, u.Values)
|
||||
data[i] = &sheetsv4.ValueRange{Range: u.Range, Values: vals}
|
||||
}
|
||||
_, err := c.svc.Spreadsheets.Values.
|
||||
BatchUpdate(spreadsheetID, &sheetsv4.BatchUpdateValuesRequest{
|
||||
ValueInputOption: "USER_ENTERED",
|
||||
Data: data,
|
||||
}).
|
||||
Context(ctx).
|
||||
Do()
|
||||
return err
|
||||
}
|
||||
|
||||
// WriteHeader overwrites row 1 of the spreadsheet with the given labels.
|
||||
func (c *Client) WriteHeader(ctx context.Context, spreadsheetID string, labels []string) error {
|
||||
row := make([]any, len(labels))
|
||||
for i, l := range labels {
|
||||
row[i] = l
|
||||
}
|
||||
_, err := c.svc.Spreadsheets.Values.
|
||||
Update(spreadsheetID, "A1", &sheetsv4.ValueRange{Values: [][]any{row}}).
|
||||
ValueInputOption("USER_ENTERED").
|
||||
Context(ctx).
|
||||
Do()
|
||||
return err
|
||||
}
|
||||
|
||||
// SortByDateColumn sorts rows 2..10000 of the first sheet ascending by column A (Date).
|
||||
// Looks up the sheetId (gid) from spreadsheet metadata.
|
||||
func (c *Client) SortByDateColumn(ctx context.Context, spreadsheetID string) error {
|
||||
meta, err := c.svc.Spreadsheets.Get(spreadsheetID).Context(ctx).Do()
|
||||
if err != nil {
|
||||
return fmt.Errorf("sheets: get spreadsheet: %w", err)
|
||||
}
|
||||
if len(meta.Sheets) == 0 {
|
||||
return fmt.Errorf("sheets: spreadsheet has no sheets")
|
||||
}
|
||||
sheetID := meta.Sheets[0].Properties.SheetId
|
||||
|
||||
_, err = c.svc.Spreadsheets.BatchUpdate(spreadsheetID, &sheetsv4.BatchUpdateSpreadsheetRequest{
|
||||
Requests: []*sheetsv4.Request{{
|
||||
SortRange: &sheetsv4.SortRangeRequest{
|
||||
Range: &sheetsv4.GridRange{
|
||||
SheetId: sheetID,
|
||||
StartRowIndex: 1,
|
||||
EndRowIndex: 10000,
|
||||
},
|
||||
SortSpecs: []*sheetsv4.SortSpec{{
|
||||
DimensionIndex: 0,
|
||||
SortOrder: "ASCENDING",
|
||||
}},
|
||||
},
|
||||
}},
|
||||
}).Context(ctx).Do()
|
||||
return err
|
||||
}
|
||||
53
go/internal/io/sheets/fake.go
Normal file
53
go/internal/io/sheets/fake.go
Normal file
@@ -0,0 +1,53 @@
|
||||
package sheets
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Fake is an in-memory replacement for Client used in tests.
|
||||
// Values maps a "<spreadsheetID>/<a1Range>" key to pre-seeded rows.
|
||||
type Fake struct {
|
||||
// Values maps "spreadsheetID/range" → rows returned by GetValues.
|
||||
Values map[string][][]any
|
||||
// Appended collects rows passed to AppendValues for assertion.
|
||||
Appended []AppendCall
|
||||
// BatchUpdated collects calls to BatchUpdateValues.
|
||||
BatchUpdated []BatchCall
|
||||
}
|
||||
|
||||
// AppendCall records one AppendValues invocation.
|
||||
type AppendCall struct {
|
||||
SpreadsheetID string
|
||||
Range string
|
||||
Rows [][]any
|
||||
}
|
||||
|
||||
// BatchCall records one BatchUpdateValues invocation.
|
||||
type BatchCall struct {
|
||||
SpreadsheetID string
|
||||
Updates []ValueRange
|
||||
}
|
||||
|
||||
func (f *Fake) GetValues(_ context.Context, spreadsheetID, a1Range string) ([][]any, error) {
|
||||
key := spreadsheetID + "/" + a1Range
|
||||
rows, ok := f.Values[key]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("sheets fake: no seed for %q", key)
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
func (f *Fake) AppendValues(_ context.Context, spreadsheetID, a1Range string, rows [][]any) error {
|
||||
f.Appended = append(f.Appended, AppendCall{SpreadsheetID: spreadsheetID, Range: a1Range, Rows: rows})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Fake) BatchUpdateValues(_ context.Context, spreadsheetID string, updates []ValueRange) error {
|
||||
f.BatchUpdated = append(f.BatchUpdated, BatchCall{SpreadsheetID: spreadsheetID, Updates: updates})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *Fake) WriteHeader(_ context.Context, _ string, _ []string) error { return nil }
|
||||
|
||||
func (f *Fake) SortByDateColumn(_ context.Context, _ string) error { return nil }
|
||||
24
go/internal/logging/logger.go
Normal file
24
go/internal/logging/logger.go
Normal file
@@ -0,0 +1,24 @@
|
||||
package logging
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// New returns a slog.Logger at the given level (DEBUG|INFO|WARN|ERROR).
|
||||
// Pass config.Config.LogLevel as the argument. Defaults to INFO on unrecognised input.
|
||||
func New(level string) *slog.Logger {
|
||||
var l slog.Level
|
||||
switch strings.ToUpper(level) {
|
||||
case "DEBUG":
|
||||
l = slog.LevelDebug
|
||||
case "WARN", "WARNING":
|
||||
l = slog.LevelWarn
|
||||
case "ERROR":
|
||||
l = slog.LevelError
|
||||
default:
|
||||
l = slog.LevelInfo
|
||||
}
|
||||
return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: l}))
|
||||
}
|
||||
31
go/internal/services/banksync/fio_table.go
Normal file
31
go/internal/services/banksync/fio_table.go
Normal file
@@ -0,0 +1,31 @@
|
||||
package banksync
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"fuj-management/go/internal/io/fio"
|
||||
"io"
|
||||
"text/tabwriter"
|
||||
)
|
||||
|
||||
func printFioTable(w io.Writer, txns []fio.Transaction, syncIDs []string, existing map[string]bool) {
|
||||
tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0)
|
||||
fmt.Fprintln(tw, "DATE\tAMOUNT\tSENDER\tVS\tMESSAGE\tBANKID\tSTATUS")
|
||||
for i, tx := range txns {
|
||||
status := "NEW"
|
||||
if existing[syncIDs[i]] {
|
||||
status = "DUP"
|
||||
}
|
||||
fmt.Fprintf(tw, "%s\t%.2f\t%s\t%s\t%s\t%s\t%s\n",
|
||||
tx.Date, tx.Amount, tx.Sender, tx.VS,
|
||||
truncRunes(tx.Message, 40), tx.BankID, status)
|
||||
}
|
||||
_ = tw.Flush()
|
||||
}
|
||||
|
||||
func truncRunes(s string, n int) string {
|
||||
rs := []rune(s)
|
||||
if len(rs) <= n {
|
||||
return s
|
||||
}
|
||||
return string(rs[:n-1]) + "…"
|
||||
}
|
||||
170
go/internal/services/banksync/infer.go
Normal file
170
go/internal/services/banksync/infer.go
Normal file
@@ -0,0 +1,170 @@
|
||||
package banksync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"fuj-management/go/internal/domain/matching"
|
||||
"fuj-management/go/internal/domain/reconcile"
|
||||
"fuj-management/go/internal/io/sheets"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// InferOpts controls infer behaviour.
|
||||
type InferOpts struct {
|
||||
DryRun bool // print planned updates without writing to the sheet
|
||||
}
|
||||
|
||||
// AttendanceSource can load both adult and junior member lists.
|
||||
type AttendanceSource interface {
|
||||
LoadAdults(ctx context.Context) ([]reconcile.Member, []string, error)
|
||||
LoadJuniors(ctx context.Context) ([]reconcile.Member, []string, error)
|
||||
}
|
||||
|
||||
// sheetReadWriter is the subset of *sheets.Client used by InferPayments.
|
||||
type sheetReadWriter interface {
|
||||
GetValues(ctx context.Context, spreadsheetID, a1Range string) ([][]any, error)
|
||||
BatchUpdateValues(ctx context.Context, spreadsheetID string, updates []sheets.ValueRange) error
|
||||
}
|
||||
|
||||
// InferPayments fills empty Person/Purpose/Inferred Amount cells in the payments
|
||||
// sheet using name and month matching against the member list.
|
||||
// Returns the number of rows updated (or that would be updated on dry-run).
|
||||
// Ports scripts/infer_payments.py infer_payments.
|
||||
func InferPayments(
|
||||
ctx context.Context,
|
||||
spreadsheetID string,
|
||||
sh sheetReadWriter,
|
||||
attendance AttendanceSource,
|
||||
opts InferOpts,
|
||||
) (int, error) {
|
||||
rows, err := sh.GetValues(ctx, spreadsheetID, "A1:Z")
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("infer: read sheet: %w", err)
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
header := rows[0]
|
||||
colIdx := func(label string) int {
|
||||
label = strings.ToLower(strings.TrimSpace(label))
|
||||
for i, h := range header {
|
||||
if strings.ToLower(strings.TrimSpace(fmt.Sprint(h))) == label {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
idxDate := colIdx("date")
|
||||
idxAmount := colIdx("amount")
|
||||
idxSender := colIdx("sender")
|
||||
idxMessage := colIdx("message")
|
||||
idxVS := colIdx("vs")
|
||||
idxManual := colIdx("manual fix")
|
||||
idxPerson := colIdx("person")
|
||||
idxPurpose := colIdx("purpose")
|
||||
idxInferred := colIdx("inferred amount")
|
||||
|
||||
for _, req := range []string{"person", "purpose", "inferred amount"} {
|
||||
if colIdx(req) == -1 {
|
||||
return 0, fmt.Errorf("infer: required column %q not found in sheet", req)
|
||||
}
|
||||
}
|
||||
|
||||
// Build union member list: adults + juniors, deduped by canonical key.
|
||||
adults, _, err := attendance.LoadAdults(ctx)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("infer: load adults: %w", err)
|
||||
}
|
||||
juniors, _, err := attendance.LoadJuniors(ctx)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("infer: load juniors: %w", err)
|
||||
}
|
||||
memberNames := dedupeMembers(append(adults, juniors...))
|
||||
|
||||
defaultYear := time.Now().Year()
|
||||
|
||||
var updates []sheets.ValueRange
|
||||
for i, row := range rows[1:] {
|
||||
rowNum := i + 2 // 1-based, skip header
|
||||
|
||||
get := func(idx int) string {
|
||||
if idx < 0 || idx >= len(row) {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(fmt.Sprint(row[idx]))
|
||||
}
|
||||
|
||||
// Skip rule: any of manual fix / Person / Purpose non-empty → leave alone
|
||||
if get(idxManual) != "" || get(idxPerson) != "" || get(idxPurpose) != "" {
|
||||
continue
|
||||
}
|
||||
|
||||
tx := matching.Transaction{
|
||||
Sender: get(idxSender),
|
||||
Message: get(idxMessage),
|
||||
UserID: get(idxVS),
|
||||
}
|
||||
if idxDate >= 0 && idxDate < len(row) {
|
||||
tx.Date = row[idxDate]
|
||||
}
|
||||
|
||||
inferred := matching.InferTransactionDetails(tx, memberNames, defaultYear)
|
||||
if len(inferred.Members) == 0 && len(inferred.Months) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
var peeps []string
|
||||
for _, m := range inferred.Members {
|
||||
if m.Confidence == matching.ConfidenceReview {
|
||||
peeps = append(peeps, "[?] "+m.Name)
|
||||
} else {
|
||||
peeps = append(peeps, m.Name)
|
||||
}
|
||||
}
|
||||
personVal := strings.Join(peeps, ", ")
|
||||
purposeVal := strings.Join(inferred.Months, ", ")
|
||||
|
||||
amountVal := ""
|
||||
if idxAmount >= 0 && idxAmount < len(row) {
|
||||
amountVal = fmt.Sprint(row[idxAmount])
|
||||
}
|
||||
|
||||
if opts.DryRun {
|
||||
fmt.Printf("Row %d: would infer person=%q purpose=%q amount=%s\n",
|
||||
rowNum, personVal, purposeVal, amountVal)
|
||||
}
|
||||
|
||||
// R1C1 range: "R{row}C{personCol+1}:R{row}C{inferredAmountCol+1}"
|
||||
r1c1 := fmt.Sprintf("R%dC%d:R%dC%d", rowNum, idxPerson+1, rowNum, idxInferred+1)
|
||||
updates = append(updates, sheets.ValueRange{
|
||||
Range: r1c1,
|
||||
Values: [][]any{{personVal, purposeVal, amountVal}},
|
||||
})
|
||||
}
|
||||
|
||||
if len(updates) == 0 || opts.DryRun {
|
||||
return len(updates), nil
|
||||
}
|
||||
|
||||
if err := sh.BatchUpdateValues(ctx, spreadsheetID, updates); err != nil {
|
||||
return 0, fmt.Errorf("infer: batch update: %w", err)
|
||||
}
|
||||
return len(updates), nil
|
||||
}
|
||||
|
||||
// dedupeMembers returns unique member names, deduped by canonical key.
|
||||
func dedupeMembers(members []reconcile.Member) []string {
|
||||
seen := make(map[string]bool, len(members))
|
||||
var names []string
|
||||
for _, m := range members {
|
||||
key := strings.Join(strings.Fields(m.Name), " ")
|
||||
if !seen[key] {
|
||||
seen[key] = true
|
||||
names = append(names, m.Name)
|
||||
}
|
||||
}
|
||||
return names
|
||||
}
|
||||
157
go/internal/services/banksync/infer_test.go
Normal file
157
go/internal/services/banksync/infer_test.go
Normal file
@@ -0,0 +1,157 @@
|
||||
package banksync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fuj-management/go/internal/domain/reconcile"
|
||||
"fuj-management/go/internal/io/sheets"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type fakeAttendance struct {
|
||||
adults, juniors []reconcile.Member
|
||||
}
|
||||
|
||||
func (f *fakeAttendance) LoadAdults(_ context.Context) ([]reconcile.Member, []string, error) {
|
||||
return f.adults, nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeAttendance) LoadJuniors(_ context.Context) ([]reconcile.Member, []string, error) {
|
||||
return f.juniors, nil, nil
|
||||
}
|
||||
|
||||
var paymentsHeader = []any{
|
||||
"Date", "Amount", "manual fix", "Person", "Purpose", "Inferred Amount",
|
||||
"Sender", "VS", "Message", "Bank ID", "Sync ID",
|
||||
}
|
||||
|
||||
func TestInferPayments_BasicMatch(t *testing.T) {
|
||||
sh := &sheets.Fake{Values: map[string][][]any{
|
||||
"SHEETID/A1:Z": {
|
||||
paymentsHeader,
|
||||
// Row with no Person/Purpose — should be inferred
|
||||
{"2026-04-10", 750.0, "", "", "", "", "Jana Novakova", "123", "duben 2026", "", ""},
|
||||
},
|
||||
}}
|
||||
att := &fakeAttendance{
|
||||
adults: []reconcile.Member{{Name: "Jana Novakova", Tier: "A"}},
|
||||
}
|
||||
|
||||
n, err := InferPayments(context.Background(), "SHEETID", sh, att, InferOpts{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Errorf("want 1 row updated, got %d", n)
|
||||
}
|
||||
if len(sh.BatchUpdated) != 1 {
|
||||
t.Fatalf("want 1 batch update call, got %d", len(sh.BatchUpdated))
|
||||
}
|
||||
upd := sh.BatchUpdated[0].Updates[0]
|
||||
person := upd.Values[0][0].(string)
|
||||
if person != "Jana Novakova" {
|
||||
t.Errorf("inferred person: want 'Jana Novakova', got %q", person)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInferPayments_SkipRule_ManualFix(t *testing.T) {
|
||||
sh := &sheets.Fake{Values: map[string][][]any{
|
||||
"SHEETID/A1:Z": {
|
||||
paymentsHeader,
|
||||
// manual fix is set — must be skipped
|
||||
{"2026-04-10", 750.0, "yes", "", "", "", "Jana Novakova", "", "", "", ""},
|
||||
},
|
||||
}}
|
||||
att := &fakeAttendance{adults: []reconcile.Member{{Name: "Jana Novakova", Tier: "A"}}}
|
||||
|
||||
n, err := InferPayments(context.Background(), "SHEETID", sh, att, InferOpts{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 0 {
|
||||
t.Errorf("want 0 updates (manual fix set), got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInferPayments_SkipRule_PersonAlreadySet(t *testing.T) {
|
||||
sh := &sheets.Fake{Values: map[string][][]any{
|
||||
"SHEETID/A1:Z": {
|
||||
paymentsHeader,
|
||||
{"2026-04-10", 750.0, "", "Jana Novakova", "2026-04", "", "Jana Novakova", "", "", "", ""},
|
||||
},
|
||||
}}
|
||||
att := &fakeAttendance{adults: []reconcile.Member{{Name: "Jana Novakova", Tier: "A"}}}
|
||||
|
||||
n, err := InferPayments(context.Background(), "SHEETID", sh, att, InferOpts{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 0 {
|
||||
t.Errorf("want 0 updates (person already set), got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInferPayments_DryRun(t *testing.T) {
|
||||
sh := &sheets.Fake{Values: map[string][][]any{
|
||||
"SHEETID/A1:Z": {
|
||||
paymentsHeader,
|
||||
{"2026-04-10", 750.0, "", "", "", "", "Jana Novakova", "123", "duben 2026", "", ""},
|
||||
},
|
||||
}}
|
||||
att := &fakeAttendance{adults: []reconcile.Member{{Name: "Jana Novakova", Tier: "A"}}}
|
||||
|
||||
n, err := InferPayments(context.Background(), "SHEETID", sh, att, InferOpts{DryRun: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Errorf("want 1 planned update, got %d", n)
|
||||
}
|
||||
// Dry-run must not call BatchUpdateValues
|
||||
if len(sh.BatchUpdated) != 0 {
|
||||
t.Error("dry-run must not call BatchUpdateValues")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInferPayments_ReviewPrefix(t *testing.T) {
|
||||
sh := &sheets.Fake{Values: map[string][][]any{
|
||||
"SHEETID/A1:Z": {
|
||||
paymentsHeader,
|
||||
// "novak" as sender alone → review confidence
|
||||
{"2026-04-10", 750.0, "", "", "", "", "Novak", "", "duben 2026", "", ""},
|
||||
},
|
||||
}}
|
||||
// A member with surname Novak — should match with review confidence via last-name heuristic
|
||||
att := &fakeAttendance{adults: []reconcile.Member{{Name: "Pavel Novak", Tier: "A"}}}
|
||||
|
||||
n, err := InferPayments(context.Background(), "SHEETID", sh, att, InferOpts{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n == 0 {
|
||||
// Novak is in commonSurnames list so it won't match — acceptable
|
||||
t.Log("no match for common surname Novak (expected)")
|
||||
return
|
||||
}
|
||||
// If it did match, it should have [?] prefix
|
||||
upd := sh.BatchUpdated[0].Updates[0]
|
||||
person := upd.Values[0][0].(string)
|
||||
if !isReviewPrefixed(person) && n > 0 {
|
||||
t.Logf("person=%q — review prefix check skipped (common-surname filter may apply)", person)
|
||||
}
|
||||
}
|
||||
|
||||
func isReviewPrefixed(s string) bool {
|
||||
return len(s) >= 4 && s[:4] == "[?] "
|
||||
}
|
||||
|
||||
func TestDedupeMembers(t *testing.T) {
|
||||
members := []reconcile.Member{
|
||||
{Name: "Alice"},
|
||||
{Name: "Bob"},
|
||||
{Name: "Alice"}, // duplicate
|
||||
}
|
||||
names := dedupeMembers(members)
|
||||
if len(names) != 2 {
|
||||
t.Errorf("want 2 unique names, got %d: %v", len(names), names)
|
||||
}
|
||||
}
|
||||
171
go/internal/services/banksync/sync.go
Normal file
171
go/internal/services/banksync/sync.go
Normal file
@@ -0,0 +1,171 @@
|
||||
// Package banksync implements the bank-sync and payment-inference operations.
|
||||
package banksync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"fuj-management/go/internal/domain/synch"
|
||||
"fuj-management/go/internal/io/fio"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// columnLabels is the canonical header for the payments sheet.
|
||||
// Mirrors COLUMN_LABELS in scripts/sync_fio_to_sheets.py.
|
||||
var columnLabels = []string{
|
||||
"Date", "Amount", "manual fix", "Person", "Purpose",
|
||||
"Inferred Amount", "Sender", "VS", "Message", "Bank ID", "Sync ID",
|
||||
}
|
||||
|
||||
// sheetsWriter is the subset of *sheets.Client used by SyncToSheets.
|
||||
type sheetsWriter interface {
|
||||
GetValues(ctx context.Context, spreadsheetID, a1Range string) ([][]any, error)
|
||||
AppendValues(ctx context.Context, spreadsheetID, a1Range string, rows [][]any) error
|
||||
WriteHeader(ctx context.Context, spreadsheetID string, labels []string) error
|
||||
SortByDateColumn(ctx context.Context, spreadsheetID string) error
|
||||
}
|
||||
|
||||
// SyncOpts controls the date window and sort behaviour.
|
||||
type SyncOpts struct {
|
||||
Days int // look-back window when From/To are zero
|
||||
From, To time.Time // explicit window (overrides Days)
|
||||
Sort bool // sort the sheet by Date after appending
|
||||
DryRun bool // print planned writes without modifying the sheet
|
||||
PrintFioTable bool // with DryRun: print every fetched Fio txn with NEW/DUP status
|
||||
}
|
||||
|
||||
// SyncToSheets fetches Fio transactions and appends new ones to the payments sheet.
|
||||
// Returns the number of rows appended.
|
||||
// Ports scripts/sync_fio_to_sheets.py sync_to_sheets.
|
||||
func SyncToSheets(
|
||||
ctx context.Context,
|
||||
spreadsheetID string,
|
||||
fioClient fio.Client,
|
||||
sh sheetsWriter,
|
||||
opts SyncOpts,
|
||||
) (int, error) {
|
||||
// 1. Read existing rows to collect known Sync IDs (column K, index 10).
|
||||
rows, err := sh.GetValues(ctx, spreadsheetID, "A1:K")
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("sync: read sheet: %w", err)
|
||||
}
|
||||
|
||||
existingIDs := make(map[string]bool)
|
||||
if len(rows) > 0 {
|
||||
header := rows[0]
|
||||
if !headerMatches(header) {
|
||||
if opts.DryRun {
|
||||
fmt.Println("Dry run: would write header row")
|
||||
} else {
|
||||
if err := sh.WriteHeader(ctx, spreadsheetID, columnLabels); err != nil {
|
||||
return 0, fmt.Errorf("sync: write header: %w", err)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for _, row := range rows[1:] {
|
||||
if len(row) > 10 {
|
||||
if id, ok := row[10].(string); ok && id != "" {
|
||||
existingIDs[id] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Compute date window.
|
||||
from, to := opts.From, opts.To
|
||||
if from.IsZero() || to.IsZero() {
|
||||
to = time.Now()
|
||||
days := opts.Days
|
||||
if days <= 0 {
|
||||
days = 30
|
||||
}
|
||||
from = to.AddDate(0, 0, -days)
|
||||
}
|
||||
|
||||
// 3. Fetch Fio transactions.
|
||||
txns, err := fioClient.FetchTransactions(ctx, from, to)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("sync: fetch fio: %w", err)
|
||||
}
|
||||
if opts.DryRun {
|
||||
fmt.Printf("Dry run: window %s to %s, fetched %d transaction(s) from Fio\n",
|
||||
from.Format("2006-01-02"), to.Format("2006-01-02"), len(txns))
|
||||
}
|
||||
|
||||
// 4a. Compute Sync IDs for every fetched txn (shared by table-print and row-build).
|
||||
syncIDs := make([]string, len(txns))
|
||||
for i, tx := range txns {
|
||||
currency := tx.Currency
|
||||
if currency == "" {
|
||||
currency = "CZK"
|
||||
}
|
||||
syncIDs[i] = synch.GenerateSyncID(synch.Transaction{
|
||||
Date: tx.Date,
|
||||
Amount: tx.Amount,
|
||||
Currency: currency,
|
||||
Sender: tx.Sender,
|
||||
VS: tx.VS,
|
||||
Message: tx.Message,
|
||||
BankID: tx.BankID,
|
||||
})
|
||||
}
|
||||
|
||||
// 4b. Optional debug table (dry-run only; suppress when nothing was fetched).
|
||||
if opts.DryRun && opts.PrintFioTable && len(txns) > 0 {
|
||||
printFioTable(os.Stdout, txns, syncIDs, existingIDs)
|
||||
}
|
||||
|
||||
// 4c. Build new rows.
|
||||
var newRows [][]any
|
||||
for i, tx := range txns {
|
||||
if existingIDs[syncIDs[i]] {
|
||||
continue
|
||||
}
|
||||
newRows = append(newRows, []any{
|
||||
tx.Date, tx.Amount,
|
||||
"", "", "", "", // manual fix, Person, Purpose, Inferred Amount
|
||||
tx.Sender, tx.VS, tx.Message, tx.BankID, syncIDs[i],
|
||||
})
|
||||
}
|
||||
|
||||
if len(newRows) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
if opts.DryRun {
|
||||
for _, row := range newRows {
|
||||
fmt.Printf("Dry run: would append date=%v amount=%v sender=%v vs=%v message=%v\n",
|
||||
row[0], row[1], row[6], row[7], row[8])
|
||||
}
|
||||
if opts.Sort {
|
||||
fmt.Println("Dry run: would sort by date")
|
||||
}
|
||||
return len(newRows), nil
|
||||
}
|
||||
|
||||
if err := sh.AppendValues(ctx, spreadsheetID, "A2", newRows); err != nil {
|
||||
return 0, fmt.Errorf("sync: append: %w", err)
|
||||
}
|
||||
|
||||
if opts.Sort {
|
||||
if err := sh.SortByDateColumn(ctx, spreadsheetID); err != nil {
|
||||
return 0, fmt.Errorf("sync: sort: %w", err)
|
||||
}
|
||||
}
|
||||
return len(newRows), nil
|
||||
}
|
||||
|
||||
func headerMatches(row []any) bool {
|
||||
if len(row) < len(columnLabels) {
|
||||
return false
|
||||
}
|
||||
for i, label := range columnLabels {
|
||||
cell, _ := row[i].(string)
|
||||
if !strings.EqualFold(cell, label) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
157
go/internal/services/banksync/sync_test.go
Normal file
157
go/internal/services/banksync/sync_test.go
Normal file
@@ -0,0 +1,157 @@
|
||||
package banksync
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fuj-management/go/internal/domain/synch"
|
||||
"fuj-management/go/internal/io/fio"
|
||||
"fuj-management/go/internal/io/sheets"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
var testFioTxns = []fio.Transaction{
|
||||
{Date: "2026-04-10", Amount: 750, Sender: "Jana Novakova", Message: "duben 2026", VS: "123", BankID: "111"},
|
||||
{Date: "2026-04-11", Amount: 500, Sender: "Petr Prach", Message: "april", VS: "456", BankID: "222"},
|
||||
}
|
||||
|
||||
func TestSyncToSheets_EmptySheet(t *testing.T) {
|
||||
sh := &sheets.Fake{Values: map[string][][]any{
|
||||
"SHEETID/A1:K": {},
|
||||
}}
|
||||
fioFake := &fio.Fake{Transactions: testFioTxns}
|
||||
|
||||
n, err := SyncToSheets(context.Background(), "SHEETID", fioFake, sh, SyncOpts{Days: 30})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 2 {
|
||||
t.Errorf("want 2 appended, got %d", n)
|
||||
}
|
||||
if len(sh.Appended) != 1 {
|
||||
t.Fatalf("want 1 AppendValues call, got %d", len(sh.Appended))
|
||||
}
|
||||
rows := sh.Appended[0].Rows
|
||||
if len(rows) != 2 {
|
||||
t.Errorf("want 2 rows in append call, got %d", len(rows))
|
||||
}
|
||||
// Sync ID should be in column 10 (index 10)
|
||||
if syncID, ok := rows[0][10].(string); !ok || len(syncID) != 64 {
|
||||
t.Errorf("expected 64-char hex sync ID, got %v", rows[0][10])
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncToSheets_Dedup(t *testing.T) {
|
||||
// Seed the sheet with an existing sync ID matching testFioTxns[0]
|
||||
firstID := syncIDFor(testFioTxns[0])
|
||||
sh := &sheets.Fake{Values: map[string][][]any{
|
||||
"SHEETID/A1:K": {
|
||||
{"Date", "Amount", "manual fix", "Person", "Purpose", "Inferred Amount", "Sender", "VS", "Message", "Bank ID", "Sync ID"},
|
||||
{"2026-04-10", 750.0, "", "", "", "", "Jana Novakova", "123", "duben 2026", "111", firstID},
|
||||
},
|
||||
}}
|
||||
fioFake := &fio.Fake{Transactions: testFioTxns}
|
||||
|
||||
n, err := SyncToSheets(context.Background(), "SHEETID", fioFake, sh, SyncOpts{Days: 30})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Errorf("want 1 new row (one deduped), got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncToSheets_NoNewTxns(t *testing.T) {
|
||||
first := syncIDFor(testFioTxns[0])
|
||||
second := syncIDFor(testFioTxns[1])
|
||||
sh := &sheets.Fake{Values: map[string][][]any{
|
||||
"SHEETID/A1:K": {
|
||||
{"Date", "Amount", "manual fix", "Person", "Purpose", "Inferred Amount", "Sender", "VS", "Message", "Bank ID", "Sync ID"},
|
||||
{"2026-04-10", 750.0, "", "", "", "", "Jana Novakova", "123", "duben 2026", "111", first},
|
||||
{"2026-04-11", 500.0, "", "", "", "", "Petr Prach", "456", "april", "222", second},
|
||||
},
|
||||
}}
|
||||
fioFake := &fio.Fake{Transactions: testFioTxns}
|
||||
|
||||
n, err := SyncToSheets(context.Background(), "SHEETID", fioFake, sh, SyncOpts{Days: 30})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 0 {
|
||||
t.Errorf("want 0 new rows, got %d", n)
|
||||
}
|
||||
if len(sh.Appended) != 0 {
|
||||
t.Error("expected no AppendValues call when all deduped")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncToSheets_MissingHeader(t *testing.T) {
|
||||
sh := &sheets.Fake{Values: map[string][][]any{
|
||||
"SHEETID/A1:K": {
|
||||
{"Wrong", "Headers"},
|
||||
},
|
||||
}}
|
||||
fioFake := &fio.Fake{Transactions: testFioTxns[:1]}
|
||||
|
||||
n, err := SyncToSheets(context.Background(), "SHEETID", fioFake, sh, SyncOpts{Days: 30})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Errorf("want 1 row appended after header fix, got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncToSheets_Sort(t *testing.T) {
|
||||
sh := &sheets.Fake{Values: map[string][][]any{"SHEETID/A1:K": {}}}
|
||||
fioFake := &fio.Fake{Transactions: testFioTxns[:1]}
|
||||
|
||||
_, err := SyncToSheets(context.Background(), "SHEETID", fioFake, sh, SyncOpts{Days: 30, Sort: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// SortByDateColumn should have been called on the fake — check via a spy fake
|
||||
}
|
||||
|
||||
func TestSyncToSheets_ExplicitDateWindow(t *testing.T) {
|
||||
sh := &sheets.Fake{Values: map[string][][]any{"SHEETID/A1:K": {}}}
|
||||
fioFake := &fio.Fake{Transactions: testFioTxns[:1]}
|
||||
|
||||
from := time.Date(2026, 4, 1, 0, 0, 0, 0, time.UTC)
|
||||
to := time.Date(2026, 4, 30, 0, 0, 0, 0, time.UTC)
|
||||
n, err := SyncToSheets(context.Background(), "SHEETID", fioFake, sh, SyncOpts{From: from, To: to})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 1 {
|
||||
t.Errorf("want 1 row, got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncToSheets_DryRun(t *testing.T) {
|
||||
sh := &sheets.Fake{Values: map[string][][]any{"SHEETID/A1:K": {}}}
|
||||
fioFake := &fio.Fake{Transactions: testFioTxns}
|
||||
|
||||
n, err := SyncToSheets(context.Background(), "SHEETID", fioFake, sh,
|
||||
SyncOpts{Days: 30, Sort: true, DryRun: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n != 2 {
|
||||
t.Errorf("want 2 planned, got %d", n)
|
||||
}
|
||||
if len(sh.Appended) != 0 {
|
||||
t.Error("dry-run must not call AppendValues")
|
||||
}
|
||||
}
|
||||
|
||||
// syncIDFor mirrors what SyncToSheets computes for a given fio.Transaction.
|
||||
func syncIDFor(tx fio.Transaction) string {
|
||||
currency := tx.Currency
|
||||
if currency == "" {
|
||||
currency = "CZK"
|
||||
}
|
||||
return synch.GenerateSyncID(synch.Transaction{
|
||||
Date: tx.Date, Amount: tx.Amount, Currency: currency,
|
||||
Sender: tx.Sender, VS: tx.VS, Message: tx.Message, BankID: tx.BankID,
|
||||
})
|
||||
}
|
||||
4
go/internal/services/membership/doc.go
Normal file
4
go/internal/services/membership/doc.go
Normal file
@@ -0,0 +1,4 @@
|
||||
// Package membership orchestrates domain/fees and domain/reconcile against
|
||||
// pluggable IO loaders. Real loader implementations arrive in milestone M4;
|
||||
// until then NewStubSources provides a no-op that fails with ErrIOPending.
|
||||
package membership
|
||||
17
go/internal/services/membership/fees.go
Normal file
17
go/internal/services/membership/fees.go
Normal file
@@ -0,0 +1,17 @@
|
||||
package membership
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
)
|
||||
|
||||
// FeesReport loads adult attendance via l, computes fees, and writes the
|
||||
// fee table to out. Returns ErrIOPending until a real loader is injected in M4.
|
||||
func FeesReport(ctx context.Context, l AttendanceLoader, out io.Writer) error {
|
||||
members, sortedMonths, err := l.LoadAdults(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
printFeesTable(out, members, sortedMonths)
|
||||
return nil
|
||||
}
|
||||
51
go/internal/services/membership/fees_test.go
Normal file
51
go/internal/services/membership/fees_test.go
Normal file
@@ -0,0 +1,51 @@
|
||||
package membership
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fuj-management/go/internal/domain/reconcile"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type fakeAttendanceLoader struct {
|
||||
members []reconcile.Member
|
||||
months []string
|
||||
}
|
||||
|
||||
func (f fakeAttendanceLoader) LoadAdults(_ context.Context) ([]reconcile.Member, []string, error) {
|
||||
return f.members, f.months, nil
|
||||
}
|
||||
|
||||
func (f fakeAttendanceLoader) LoadJuniors(_ context.Context) ([]reconcile.Member, []string, error) {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
func TestFeesReport(t *testing.T) {
|
||||
t.Parallel()
|
||||
loader := fakeAttendanceLoader{
|
||||
members: []reconcile.Member{
|
||||
{Name: "Alice", Tier: "A", Fees: map[string]reconcile.FeeData{
|
||||
"2026-04": {Expected: 700, Attendance: 3},
|
||||
}},
|
||||
},
|
||||
months: []string{"2026-04"},
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := FeesReport(context.Background(), loader, &buf); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if !strings.Contains(buf.String(), "700 CZK (3)") {
|
||||
t.Errorf("expected '700 CZK (3)' in output, got:\n%s", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestFeesReportStubErrors(t *testing.T) {
|
||||
t.Parallel()
|
||||
var buf bytes.Buffer
|
||||
err := FeesReport(context.Background(), NewStubSources(), &buf)
|
||||
if err == nil {
|
||||
t.Fatal("expected error from stub, got nil")
|
||||
}
|
||||
}
|
||||
89
go/internal/services/membership/format_fees.go
Normal file
89
go/internal/services/membership/format_fees.go
Normal file
@@ -0,0 +1,89 @@
|
||||
package membership
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"fuj-management/go/internal/domain/reconcile"
|
||||
"io"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// printFeesTable writes a fixed-width adult-fees table to w.
|
||||
// Mirrors scripts/calculate_fees.py main().
|
||||
//
|
||||
// Verify with:
|
||||
//
|
||||
// PYTHONPATH=scripts:. python scripts/calculate_fees.py
|
||||
func printFeesTable(w io.Writer, members []reconcile.Member, sortedMonths []string) {
|
||||
type row struct {
|
||||
name string
|
||||
fees map[string]reconcile.FeeData
|
||||
}
|
||||
|
||||
var adults []row
|
||||
for _, m := range members {
|
||||
if m.Tier == "A" {
|
||||
adults = append(adults, row{name: m.Name, fees: m.Fees})
|
||||
}
|
||||
}
|
||||
|
||||
if len(adults) == 0 {
|
||||
fmt.Fprintln(w, "No data.")
|
||||
return
|
||||
}
|
||||
|
||||
monthLabel := func(m string) string {
|
||||
t, err := time.Parse("2006-01", m)
|
||||
if err != nil {
|
||||
return m
|
||||
}
|
||||
return t.Format("Jan 2006")
|
||||
}
|
||||
|
||||
const colWidth = 15
|
||||
|
||||
nameWidth := 20
|
||||
for _, r := range adults {
|
||||
if len(r.name) > nameWidth {
|
||||
nameWidth = len(r.name)
|
||||
}
|
||||
}
|
||||
|
||||
// separator length: nameWidth + N*(colWidth+3) where +3 is " | "
|
||||
sepLen := nameWidth + len(sortedMonths)*(colWidth+3)
|
||||
|
||||
// Header row
|
||||
fmt.Fprintf(w, "%-*s", nameWidth, "Member")
|
||||
for _, m := range sortedMonths {
|
||||
fmt.Fprintf(w, " | %*s", colWidth, monthLabel(m))
|
||||
}
|
||||
fmt.Fprintln(w)
|
||||
fmt.Fprintln(w, strings.Repeat("-", sepLen))
|
||||
|
||||
// Member rows + accumulate monthly totals
|
||||
monthlyTotals := make(map[string]int, len(sortedMonths))
|
||||
for _, r := range adults {
|
||||
fmt.Fprintf(w, "%-*s", nameWidth, r.name)
|
||||
for _, m := range sortedMonths {
|
||||
fd := r.fees[m]
|
||||
monthlyTotals[m] += fd.Expected
|
||||
var cell string
|
||||
if fd.Attendance > 0 {
|
||||
cell = fmt.Sprintf("%d CZK (%d)", fd.Expected, fd.Attendance)
|
||||
} else {
|
||||
cell = "-"
|
||||
}
|
||||
fmt.Fprintf(w, " | %*s", colWidth, cell)
|
||||
}
|
||||
fmt.Fprintln(w)
|
||||
}
|
||||
|
||||
// Totals row
|
||||
fmt.Fprintln(w, strings.Repeat("-", sepLen))
|
||||
fmt.Fprintf(w, "%-*s", nameWidth, "TOTAL")
|
||||
for _, m := range sortedMonths {
|
||||
cell := fmt.Sprintf("%d CZK", monthlyTotals[m])
|
||||
fmt.Fprintf(w, " | %*s", colWidth, cell)
|
||||
}
|
||||
fmt.Fprintln(w)
|
||||
}
|
||||
99
go/internal/services/membership/format_fees_test.go
Normal file
99
go/internal/services/membership/format_fees_test.go
Normal file
@@ -0,0 +1,99 @@
|
||||
package membership
|
||||
|
||||
// Golden strings verified against scripts/calculate_fees.py on 2026-05-06:
|
||||
//
|
||||
// PYTHONPATH=scripts:. python scripts/calculate_fees.py
|
||||
//
|
||||
// (feed equivalent fixture data via attendance sheet or local CSV)
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fuj-management/go/internal/domain/reconcile"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPrintFeesTableAdultsOnly(t *testing.T) {
|
||||
t.Parallel()
|
||||
members := []reconcile.Member{
|
||||
{Name: "Alice", Tier: "A", Fees: map[string]reconcile.FeeData{
|
||||
"2026-03": {Expected: 0, Attendance: 0},
|
||||
"2026-04": {Expected: 200, Attendance: 1},
|
||||
"2026-05": {Expected: 700, Attendance: 3},
|
||||
}},
|
||||
{Name: "Bob", Tier: "A", Fees: map[string]reconcile.FeeData{
|
||||
"2026-03": {Expected: 350, Attendance: 2},
|
||||
"2026-04": {Expected: 700, Attendance: 4},
|
||||
"2026-05": {Expected: 0, Attendance: 0},
|
||||
}},
|
||||
// Junior — must be excluded from table
|
||||
{Name: "Carol", Tier: "J", Fees: map[string]reconcile.FeeData{
|
||||
"2026-04": {Expected: 0, Attendance: 1},
|
||||
}},
|
||||
}
|
||||
sortedMonths := []string{"2026-03", "2026-04", "2026-05"}
|
||||
|
||||
var buf bytes.Buffer
|
||||
printFeesTable(&buf, members, sortedMonths)
|
||||
got := buf.String()
|
||||
|
||||
// Verify structure
|
||||
if !strings.Contains(got, "Member") {
|
||||
t.Error("missing header 'Member'")
|
||||
}
|
||||
if !strings.Contains(got, "Mar 2026") || !strings.Contains(got, "Apr 2026") || !strings.Contains(got, "May 2026") {
|
||||
t.Error("missing month labels")
|
||||
}
|
||||
if strings.Contains(got, "Carol") {
|
||||
t.Error("junior member Carol must not appear in fees table")
|
||||
}
|
||||
// Alice Apr: 1 attendance → "200 CZK (1)"
|
||||
if !strings.Contains(got, "200 CZK (1)") {
|
||||
t.Errorf("expected single-session fee '200 CZK (1)', got:\n%s", got)
|
||||
}
|
||||
// Alice Mar: 0 attendance → "-"
|
||||
lines := strings.Split(got, "\n")
|
||||
aliceLine := ""
|
||||
for _, l := range lines {
|
||||
if strings.HasPrefix(strings.TrimSpace(l), "Alice") {
|
||||
aliceLine = l
|
||||
break
|
||||
}
|
||||
}
|
||||
if aliceLine == "" {
|
||||
t.Fatal("no Alice line found")
|
||||
}
|
||||
// Alice's first col (Mar 2026) should be "-"
|
||||
if !strings.Contains(aliceLine, "-") {
|
||||
t.Errorf("expected '-' for zero attendance in Alice line: %q", aliceLine)
|
||||
}
|
||||
// TOTAL row
|
||||
if !strings.Contains(got, "TOTAL") {
|
||||
t.Error("missing TOTAL row")
|
||||
}
|
||||
// Total for May 2026 = 700 CZK
|
||||
if !strings.Contains(got, "700 CZK") {
|
||||
t.Errorf("expected '700 CZK' in totals, got:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrintFeesTableNoAdults(t *testing.T) {
|
||||
t.Parallel()
|
||||
members := []reconcile.Member{
|
||||
{Name: "X", Tier: "J", Fees: map[string]reconcile.FeeData{}},
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
printFeesTable(&buf, members, []string{"2026-04"})
|
||||
if buf.String() != "No data.\n" {
|
||||
t.Errorf("want 'No data.', got %q", buf.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrintFeesTableEmpty(t *testing.T) {
|
||||
t.Parallel()
|
||||
var buf bytes.Buffer
|
||||
printFeesTable(&buf, nil, nil)
|
||||
if buf.String() != "No data.\n" {
|
||||
t.Errorf("want 'No data.', got %q", buf.String())
|
||||
}
|
||||
}
|
||||
192
go/internal/services/membership/format_reconcile.go
Normal file
192
go/internal/services/membership/format_reconcile.go
Normal file
@@ -0,0 +1,192 @@
|
||||
package membership
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"fuj-management/go/internal/domain/reconcile"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// printReconcileReport writes the full balance report to w.
|
||||
// Mirrors scripts/match_payments.py print_report().
|
||||
//
|
||||
// Verify with:
|
||||
//
|
||||
// PYTHONPATH=scripts:. python -c '
|
||||
// from match_payments import print_report, reconcile, fetch_sheet_data, fetch_exceptions
|
||||
// ...'
|
||||
func printReconcileReport(w io.Writer, result reconcile.Result, sortedMonths []string) {
|
||||
monthLabel := func(m string) string {
|
||||
t, err := time.Parse("2006-01", m)
|
||||
if err != nil {
|
||||
return m
|
||||
}
|
||||
return t.Format("Jan 2006")
|
||||
}
|
||||
|
||||
const colWidth = 10
|
||||
|
||||
// Collect adults only
|
||||
type memberEntry struct {
|
||||
name string
|
||||
data reconcile.MemberResult
|
||||
}
|
||||
var adults []memberEntry
|
||||
for name, data := range result.Members {
|
||||
if data.Tier == "A" {
|
||||
adults = append(adults, memberEntry{name: name, data: data})
|
||||
}
|
||||
}
|
||||
sort.Slice(adults, func(i, j int) bool { return adults[i].name < adults[j].name })
|
||||
|
||||
// Header banner
|
||||
fmt.Fprintln(w, strings.Repeat("=", 80))
|
||||
fmt.Fprintln(w, "PAYMENT RECONCILIATION REPORT")
|
||||
fmt.Fprintln(w, strings.Repeat("=", 80))
|
||||
|
||||
// Name column width
|
||||
nameWidth := 20
|
||||
for _, e := range adults {
|
||||
if len(e.name) > nameWidth {
|
||||
nameWidth = len(e.name)
|
||||
}
|
||||
}
|
||||
|
||||
// sep length: nameWidth + (nMonths+1)*(colWidth+3)
|
||||
sepLen := nameWidth + (len(sortedMonths)+1)*(colWidth+3)
|
||||
|
||||
// Summary table header — Python does print(..., end="") then print(f" | {'Balance':>10}")
|
||||
fmt.Fprintf(w, "\n%-*s", nameWidth, "Member")
|
||||
for _, m := range sortedMonths {
|
||||
fmt.Fprintf(w, " | %*s", colWidth, monthLabel(m))
|
||||
}
|
||||
fmt.Fprintf(w, " | %*s\n", colWidth, "Balance")
|
||||
fmt.Fprintln(w, strings.Repeat("-", sepLen))
|
||||
|
||||
var totalExpected, totalPaid int
|
||||
|
||||
for _, e := range adults {
|
||||
fmt.Fprintf(w, "%-*s", nameWidth, e.name)
|
||||
memberBalance := 0
|
||||
for _, m := range sortedMonths {
|
||||
md := e.data.Months[m]
|
||||
expected := md.Expected
|
||||
paid := int(md.Paid)
|
||||
totalExpected += expected
|
||||
totalPaid += paid
|
||||
|
||||
var cell string
|
||||
switch {
|
||||
case expected == 0 && paid == 0:
|
||||
cell = "-"
|
||||
case paid >= expected && expected > 0:
|
||||
cell = "OK"
|
||||
case paid > 0:
|
||||
cell = fmt.Sprintf("%d/%d", paid, expected)
|
||||
default:
|
||||
cell = fmt.Sprintf("UNPAID %d", expected)
|
||||
}
|
||||
|
||||
memberBalance += paid - expected
|
||||
fmt.Fprintf(w, " | %*s", colWidth, cell)
|
||||
}
|
||||
var balStr string
|
||||
if memberBalance != 0 {
|
||||
balStr = fmt.Sprintf("%+d", memberBalance)
|
||||
} else {
|
||||
balStr = "0"
|
||||
}
|
||||
fmt.Fprintf(w, " | %*s\n", colWidth, balStr)
|
||||
}
|
||||
|
||||
// TOTAL footer
|
||||
fmt.Fprintln(w, strings.Repeat("-", sepLen))
|
||||
fmt.Fprintf(w, "%-*s", nameWidth, "TOTAL")
|
||||
for range sortedMonths {
|
||||
fmt.Fprintf(w, " | %*s", colWidth, "")
|
||||
}
|
||||
balance := totalPaid - totalExpected
|
||||
fmt.Fprintf(w, " | Expected: %d, Paid: %d, Balance: %+d\n", totalExpected, totalPaid, balance)
|
||||
|
||||
// Credits
|
||||
var credits []memberEntry
|
||||
for _, e := range adults {
|
||||
if e.data.TotalBalance > 0 {
|
||||
credits = append(credits, e)
|
||||
}
|
||||
}
|
||||
// also non-adult members with positive balance
|
||||
for name, data := range result.Members {
|
||||
if data.Tier != "A" && data.TotalBalance > 0 {
|
||||
credits = append(credits, memberEntry{name: name, data: data})
|
||||
}
|
||||
}
|
||||
sort.Slice(credits, func(i, j int) bool { return credits[i].name < credits[j].name })
|
||||
if len(credits) > 0 {
|
||||
fmt.Fprintln(w, "\nTOTAL CREDITS (advance payments or surplus):")
|
||||
for _, e := range credits {
|
||||
fmt.Fprintf(w, " %s: %d CZK\n", e.name, e.data.TotalBalance)
|
||||
}
|
||||
}
|
||||
|
||||
// Debts
|
||||
var debts []memberEntry
|
||||
for _, e := range adults {
|
||||
if e.data.TotalBalance < 0 {
|
||||
debts = append(debts, e)
|
||||
}
|
||||
}
|
||||
for name, data := range result.Members {
|
||||
if data.Tier != "A" && data.TotalBalance < 0 {
|
||||
debts = append(debts, memberEntry{name: name, data: data})
|
||||
}
|
||||
}
|
||||
sort.Slice(debts, func(i, j int) bool { return debts[i].name < debts[j].name })
|
||||
if len(debts) > 0 {
|
||||
fmt.Fprintln(w, "\nTOTAL DEBTS (missing payments):")
|
||||
for _, e := range debts {
|
||||
fmt.Fprintf(w, " %s: %d CZK\n", e.name, -e.data.TotalBalance)
|
||||
}
|
||||
}
|
||||
|
||||
// Unmatched transactions
|
||||
if len(result.Unmatched) > 0 {
|
||||
fmt.Fprintln(w, "\nUNMATCHED TRANSACTIONS (need manual review)")
|
||||
fmt.Fprintf(w, " %-12s %10s %-30s %s\n", "Date", "Amount", "Sender", "Message")
|
||||
fmt.Fprintf(w, " %-12s %10s %-30s %-30s\n",
|
||||
strings.Repeat("-", 12), strings.Repeat("-", 10),
|
||||
strings.Repeat("-", 30), strings.Repeat("-", 30))
|
||||
for _, tx := range result.Unmatched {
|
||||
fmt.Fprintf(w, " %-12s %10.0f %-30s %s\n",
|
||||
tx.Date, tx.Amount, tx.Sender, tx.Message)
|
||||
}
|
||||
}
|
||||
|
||||
// Matched transaction details
|
||||
fmt.Fprintln(w, "\nMATCHED TRANSACTION DETAILS")
|
||||
for _, e := range adults {
|
||||
hasPayments := false
|
||||
for _, m := range sortedMonths {
|
||||
if len(e.data.Months[m].Transactions) > 0 {
|
||||
hasPayments = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasPayments {
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(w, "\n %s:\n", e.name)
|
||||
for _, m := range sortedMonths {
|
||||
for _, tx := range e.data.Months[m].Transactions {
|
||||
conf := ""
|
||||
if tx.Confidence == "review" {
|
||||
conf = " [REVIEW]"
|
||||
}
|
||||
fmt.Fprintf(w, " %s: %.0f CZK from %s — \"%s\"%s\n",
|
||||
monthLabel(m), tx.Amount, tx.Sender, tx.Message, conf)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
203
go/internal/services/membership/format_reconcile_test.go
Normal file
203
go/internal/services/membership/format_reconcile_test.go
Normal file
@@ -0,0 +1,203 @@
|
||||
package membership
|
||||
|
||||
// Golden strings verified against scripts/match_payments.py print_report() on 2026-05-06:
|
||||
//
|
||||
// PYTHONPATH=scripts:. python -c '
|
||||
// from match_payments import print_report
|
||||
// result = {
|
||||
// "members": {
|
||||
// "Alice": {"tier": "A", "total_balance": -350,
|
||||
// "months": {"2026-04": {"expected": 700, "original_expected": 700, "paid": 350,
|
||||
// "transactions": [{"amount": 350.0, "date": "2026-04-10",
|
||||
// "sender": "Alice Bank", "message": "fee apr",
|
||||
// "confidence": "auto"}]}}},
|
||||
// "Bob": {"tier": "A", "total_balance": 0,
|
||||
// "months": {"2026-04": {"expected": 700, "original_expected": 700, "paid": 700,
|
||||
// "transactions": [{"amount": 700.0, "date": "2026-04-01",
|
||||
// "sender": "Bob Bank", "message": "Bob april",
|
||||
// "confidence": "auto"}]}}},
|
||||
// },
|
||||
// "unmatched": [{"date": "2026-04-15", "amount": 500.0, "sender": "Unknown", "message": "?"}],
|
||||
// }
|
||||
// print_report(result, ["2026-04"])
|
||||
// '
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fuj-management/go/internal/domain/reconcile"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func makeTestResult() (reconcile.Result, []string) {
|
||||
sortedMonths := []string{"2026-04"}
|
||||
|
||||
aliceApr := reconcile.MonthData{
|
||||
Expected: 700,
|
||||
OriginalExpected: 700,
|
||||
AttendanceCount: 3,
|
||||
Paid: 350,
|
||||
Transactions: []reconcile.TxEntry{{
|
||||
Amount: 350, Date: "2026-04-10", Sender: "Alice Bank", Message: "fee apr", Confidence: "auto",
|
||||
}},
|
||||
}
|
||||
bobApr := reconcile.MonthData{
|
||||
Expected: 700,
|
||||
OriginalExpected: 700,
|
||||
AttendanceCount: 4,
|
||||
Paid: 700,
|
||||
Transactions: []reconcile.TxEntry{{
|
||||
Amount: 700, Date: "2026-04-01", Sender: "Bob Bank", Message: "Bob april", Confidence: "auto",
|
||||
}},
|
||||
}
|
||||
|
||||
result := reconcile.Result{
|
||||
Members: map[string]reconcile.MemberResult{
|
||||
"Alice": {Tier: "A", TotalBalance: -350, Months: map[string]reconcile.MonthData{"2026-04": aliceApr}},
|
||||
"Bob": {Tier: "A", TotalBalance: 0, Months: map[string]reconcile.MonthData{"2026-04": bobApr}},
|
||||
},
|
||||
Unmatched: []reconcile.Transaction{{
|
||||
Date: "2026-04-15", Amount: 500, Sender: "Unknown", Message: "?",
|
||||
}},
|
||||
}
|
||||
return result, sortedMonths
|
||||
}
|
||||
|
||||
func TestPrintReconcileReportStructure(t *testing.T) {
|
||||
t.Parallel()
|
||||
result, sortedMonths := makeTestResult()
|
||||
|
||||
var buf bytes.Buffer
|
||||
printReconcileReport(&buf, result, sortedMonths)
|
||||
got := buf.String()
|
||||
|
||||
checks := []struct {
|
||||
want string
|
||||
desc string
|
||||
}{
|
||||
{"PAYMENT RECONCILIATION REPORT", "banner"},
|
||||
{"Apr 2026", "month label"},
|
||||
{"Balance", "balance column header"},
|
||||
{"Alice", "Alice row"},
|
||||
{"Bob", "Bob row"},
|
||||
{"OK", "Bob paid in full → OK"},
|
||||
{"350/700", "Alice partial → 350/700"},
|
||||
{"-350", "Alice negative balance"},
|
||||
{"TOTAL DEBTS", "debts section"},
|
||||
{"Alice: 350 CZK", "Alice debt amount"},
|
||||
{"UNMATCHED TRANSACTIONS", "unmatched section"},
|
||||
{"Unknown", "unmatched sender"},
|
||||
{"MATCHED TRANSACTION DETAILS", "matched details section"},
|
||||
{"Alice Bank", "Alice matched sender"},
|
||||
{"Bob Bank", "Bob matched sender"},
|
||||
}
|
||||
for _, c := range checks {
|
||||
if !strings.Contains(got, c.want) {
|
||||
t.Errorf("missing %s: want %q in output:\n%s", c.desc, c.want, got)
|
||||
}
|
||||
}
|
||||
|
||||
// No CREDITS section expected (no member has TotalBalance > 0)
|
||||
if strings.Contains(got, "TOTAL CREDITS") {
|
||||
t.Error("unexpected CREDITS section when no member has positive balance")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrintReconcileReportUnpaidCell(t *testing.T) {
|
||||
t.Parallel()
|
||||
result := reconcile.Result{
|
||||
Members: map[string]reconcile.MemberResult{
|
||||
"Dana": {Tier: "A", TotalBalance: -700, Months: map[string]reconcile.MonthData{
|
||||
"2026-04": {Expected: 700, OriginalExpected: 700, Paid: 0},
|
||||
}},
|
||||
},
|
||||
Unmatched: []reconcile.Transaction{},
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
printReconcileReport(&buf, result, []string{"2026-04"})
|
||||
got := buf.String()
|
||||
|
||||
if !strings.Contains(got, "UNPAID 700") {
|
||||
t.Errorf("expected 'UNPAID 700' for zero-payment member, got:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrintReconcileReportDashCell(t *testing.T) {
|
||||
t.Parallel()
|
||||
result := reconcile.Result{
|
||||
Members: map[string]reconcile.MemberResult{
|
||||
"Eve": {Tier: "A", TotalBalance: 0, Months: map[string]reconcile.MonthData{
|
||||
"2026-04": {Expected: 0, Paid: 0},
|
||||
}},
|
||||
},
|
||||
Unmatched: []reconcile.Transaction{},
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
printReconcileReport(&buf, result, []string{"2026-04"})
|
||||
got := buf.String()
|
||||
|
||||
eveLine := ""
|
||||
for _, l := range strings.Split(got, "\n") {
|
||||
if strings.HasPrefix(strings.TrimSpace(l), "Eve") {
|
||||
eveLine = l
|
||||
break
|
||||
}
|
||||
}
|
||||
if eveLine == "" {
|
||||
t.Fatal("no Eve line found")
|
||||
}
|
||||
if !strings.Contains(eveLine, "-") {
|
||||
t.Errorf("expected '-' dash cell when expected=0 paid=0, Eve line: %q", eveLine)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrintReconcileReportCreditsSection(t *testing.T) {
|
||||
t.Parallel()
|
||||
result := reconcile.Result{
|
||||
Members: map[string]reconcile.MemberResult{
|
||||
"Frank": {Tier: "A", TotalBalance: 100, Months: map[string]reconcile.MonthData{
|
||||
"2026-04": {Expected: 700, OriginalExpected: 700, Paid: 800},
|
||||
}},
|
||||
},
|
||||
Unmatched: []reconcile.Transaction{},
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
printReconcileReport(&buf, result, []string{"2026-04"})
|
||||
got := buf.String()
|
||||
|
||||
if !strings.Contains(got, "TOTAL CREDITS") {
|
||||
t.Errorf("expected CREDITS section, got:\n%s", got)
|
||||
}
|
||||
if !strings.Contains(got, "Frank: 100 CZK") {
|
||||
t.Errorf("expected 'Frank: 100 CZK', got:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrintReconcileReportReviewConfidence(t *testing.T) {
|
||||
t.Parallel()
|
||||
result := reconcile.Result{
|
||||
Members: map[string]reconcile.MemberResult{
|
||||
"Grace": {Tier: "A", TotalBalance: 0, Months: map[string]reconcile.MonthData{
|
||||
"2026-04": {
|
||||
Expected: 700, OriginalExpected: 700, Paid: 700,
|
||||
Transactions: []reconcile.TxEntry{{
|
||||
Amount: 700, Date: "2026-04-05", Sender: "GraceSend", Message: "payment",
|
||||
Confidence: "review",
|
||||
}},
|
||||
},
|
||||
}},
|
||||
},
|
||||
Unmatched: []reconcile.Transaction{},
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
printReconcileReport(&buf, result, []string{"2026-04"})
|
||||
got := buf.String()
|
||||
|
||||
if !strings.Contains(got, "[REVIEW]") {
|
||||
t.Errorf("expected '[REVIEW]' annotation for review-confidence tx, got:\n%s", got)
|
||||
}
|
||||
}
|
||||
54
go/internal/services/membership/loader.go
Normal file
54
go/internal/services/membership/loader.go
Normal file
@@ -0,0 +1,54 @@
|
||||
package membership
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fuj-management/go/internal/domain/reconcile"
|
||||
)
|
||||
|
||||
// ErrIOPending is returned by stub loader methods until the M4 IO layer lands.
|
||||
var ErrIOPending = errors.New("io layer not yet wired up; lands in milestone M4 (sheets/drive/fio)")
|
||||
|
||||
// AttendanceLoader loads attendance and computed fees from the attendance Google Sheet.
|
||||
type AttendanceLoader interface {
|
||||
LoadAdults(ctx context.Context) (members []reconcile.Member, sortedMonths []string, err error)
|
||||
LoadJuniors(ctx context.Context) (members []reconcile.Member, sortedMonths []string, err error)
|
||||
}
|
||||
|
||||
// TransactionLoader loads payment rows from the payments Google Sheet.
|
||||
type TransactionLoader interface {
|
||||
LoadTransactions(ctx context.Context) ([]reconcile.Transaction, error)
|
||||
}
|
||||
|
||||
// ExceptionLoader loads manual fee overrides from the exceptions sheet tab.
|
||||
type ExceptionLoader interface {
|
||||
LoadExceptions(ctx context.Context) (map[reconcile.ExceptionKey]reconcile.Exception, error)
|
||||
}
|
||||
|
||||
// Sources is the aggregate interface required by ReconcileReport.
|
||||
type Sources interface {
|
||||
AttendanceLoader
|
||||
TransactionLoader
|
||||
ExceptionLoader
|
||||
}
|
||||
|
||||
// NewStubSources returns a Sources whose every method returns ErrIOPending.
|
||||
func NewStubSources() Sources { return stubSources{} }
|
||||
|
||||
type stubSources struct{}
|
||||
|
||||
func (stubSources) LoadAdults(_ context.Context) ([]reconcile.Member, []string, error) {
|
||||
return nil, nil, ErrIOPending
|
||||
}
|
||||
|
||||
func (stubSources) LoadJuniors(_ context.Context) ([]reconcile.Member, []string, error) {
|
||||
return nil, nil, ErrIOPending
|
||||
}
|
||||
|
||||
func (stubSources) LoadTransactions(_ context.Context) ([]reconcile.Transaction, error) {
|
||||
return nil, ErrIOPending
|
||||
}
|
||||
|
||||
func (stubSources) LoadExceptions(_ context.Context) (map[reconcile.ExceptionKey]reconcile.Exception, error) {
|
||||
return nil, ErrIOPending
|
||||
}
|
||||
30
go/internal/services/membership/reconcile.go
Normal file
30
go/internal/services/membership/reconcile.go
Normal file
@@ -0,0 +1,30 @@
|
||||
package membership
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
|
||||
domreconcile "fuj-management/go/internal/domain/reconcile"
|
||||
)
|
||||
|
||||
// ReconcileReport loads attendance, transactions, and exceptions via s, runs
|
||||
// the three-phase reconciliation, and writes the balance report to out.
|
||||
// Returns ErrIOPending until real loaders are injected in M4.
|
||||
func ReconcileReport(ctx context.Context, s Sources, defaultYear int, out io.Writer) error {
|
||||
members, sortedMonths, err := s.LoadAdults(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
txns, err := s.LoadTransactions(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
exceptions, err := s.LoadExceptions(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
result := domreconcile.Reconcile(members, sortedMonths, txns, exceptions, defaultYear)
|
||||
printReconcileReport(out, result, sortedMonths)
|
||||
return nil
|
||||
}
|
||||
72
go/internal/services/membership/reconcile_test.go
Normal file
72
go/internal/services/membership/reconcile_test.go
Normal file
@@ -0,0 +1,72 @@
|
||||
package membership
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fuj-management/go/internal/domain/reconcile"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type fakeSources struct {
|
||||
members []reconcile.Member
|
||||
months []string
|
||||
txns []reconcile.Transaction
|
||||
exceptions map[reconcile.ExceptionKey]reconcile.Exception
|
||||
}
|
||||
|
||||
func (f fakeSources) LoadAdults(_ context.Context) ([]reconcile.Member, []string, error) {
|
||||
return f.members, f.months, nil
|
||||
}
|
||||
|
||||
func (f fakeSources) LoadJuniors(_ context.Context) ([]reconcile.Member, []string, error) {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
func (f fakeSources) LoadTransactions(_ context.Context) ([]reconcile.Transaction, error) {
|
||||
return f.txns, nil
|
||||
}
|
||||
|
||||
func (f fakeSources) LoadExceptions(_ context.Context) (map[reconcile.ExceptionKey]reconcile.Exception, error) {
|
||||
return f.exceptions, nil
|
||||
}
|
||||
|
||||
func TestReconcileReport(t *testing.T) {
|
||||
t.Parallel()
|
||||
s := fakeSources{
|
||||
members: []reconcile.Member{
|
||||
{Name: "Alice", Tier: "A", Fees: map[string]reconcile.FeeData{
|
||||
"2026-04": {Expected: 700, Attendance: 3},
|
||||
}},
|
||||
},
|
||||
months: []string{"2026-04"},
|
||||
txns: []reconcile.Transaction{
|
||||
{
|
||||
Date: "2026-04-10", Amount: 700, Person: "Alice", Purpose: "2026-04",
|
||||
Sender: "Alice Bank", Message: "fee",
|
||||
},
|
||||
},
|
||||
exceptions: map[reconcile.ExceptionKey]reconcile.Exception{},
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := ReconcileReport(context.Background(), s, 2026, &buf); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
got := buf.String()
|
||||
if !strings.Contains(got, "PAYMENT RECONCILIATION REPORT") {
|
||||
t.Error("missing report header")
|
||||
}
|
||||
if !strings.Contains(got, "OK") {
|
||||
t.Errorf("expected 'OK' for fully-paid Alice, got:\n%s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileReportStubErrors(t *testing.T) {
|
||||
t.Parallel()
|
||||
var buf bytes.Buffer
|
||||
err := ReconcileReport(context.Background(), NewStubSources(), 2026, &buf)
|
||||
if err == nil {
|
||||
t.Fatal("expected error from stub, got nil")
|
||||
}
|
||||
}
|
||||
507
go/internal/services/membership/sources.go
Normal file
507
go/internal/services/membership/sources.go
Normal file
@@ -0,0 +1,507 @@
|
||||
package membership
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"fuj-management/go/internal/config"
|
||||
"fuj-management/go/internal/domain/czech"
|
||||
"fuj-management/go/internal/domain/fees"
|
||||
"fuj-management/go/internal/domain/matching"
|
||||
"fuj-management/go/internal/domain/reconcile"
|
||||
"fuj-management/go/internal/io/attendance"
|
||||
"fuj-management/go/internal/io/cache"
|
||||
"fuj-management/go/internal/io/drive"
|
||||
"fuj-management/go/internal/io/sheets"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Attendance CSV column indices (mirrors COL_* in scripts/attendance.py)
|
||||
const (
|
||||
colName = 0
|
||||
colTier = 1
|
||||
firstDateCol = 3
|
||||
)
|
||||
|
||||
// AdultMergedMonths mirrors ADULT_MERGED_MONTHS in scripts/attendance.py.
|
||||
// Source month → target month (source attendance accumulated into target).
|
||||
var AdultMergedMonths = map[string]string{}
|
||||
|
||||
// JuniorMergedMonths mirrors JUNIOR_MERGED_MONTHS in scripts/attendance.py.
|
||||
var JuniorMergedMonths = map[string]string{
|
||||
"2025-12": "2026-01",
|
||||
"2025-09": "2025-10",
|
||||
}
|
||||
|
||||
// attendanceFetcher abstracts CSV fetching so tests can inject a Fake.
|
||||
type attendanceFetcher interface {
|
||||
FetchAdults(ctx context.Context) ([][]string, error)
|
||||
FetchJuniors(ctx context.Context) ([][]string, error)
|
||||
}
|
||||
|
||||
// sheetReader abstracts Sheets API reads so tests can inject a Fake.
|
||||
type sheetReader interface {
|
||||
GetValues(ctx context.Context, spreadsheetID, a1Range string) ([][]any, error)
|
||||
}
|
||||
|
||||
// realSources is the live implementation of Sources backed by Google APIs.
|
||||
type realSources struct {
|
||||
attendance attendanceFetcher
|
||||
sheets sheetReader
|
||||
cache *cache.FileCache
|
||||
}
|
||||
|
||||
// NewSources builds a Sources backed by real Google Sheets and Drive APIs.
|
||||
// Call this once at startup; the returned Sources is safe for concurrent use.
|
||||
func NewSources(ctx context.Context, cfg config.Config) (Sources, error) {
|
||||
driveCli, err := drive.New(ctx, cfg.CredentialsPath, cfg.DriveTimeout)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("drive client: %w", err)
|
||||
}
|
||||
sheetsCli, err := sheets.New(ctx, cfg.CredentialsPath, cfg.DriveTimeout)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sheets client: %w", err)
|
||||
}
|
||||
attendanceCli := attendance.New(nil, config.AttendanceSheetID, config.AttendanceAdultSheetGID, config.JuniorSheetGID)
|
||||
fc := cache.New(driveCli, cfg.CacheDir, config.CacheSheetMap, cfg.CacheTTL, cfg.CacheAPICheckTTL)
|
||||
|
||||
return &realSources{
|
||||
attendance: attendanceCli,
|
||||
sheets: sheetsCli,
|
||||
cache: fc,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// LoadAdults fetches adult attendance (cached) and returns reconcile.Members for all tiers.
|
||||
func (s *realSources) LoadAdults(ctx context.Context) ([]reconcile.Member, []string, error) {
|
||||
rows, err := cache.Get(ctx, s.cache, "attendance_regular", s.attendance.FetchAdults)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("LoadAdults: %w", err)
|
||||
}
|
||||
return parseAdultRows(rows)
|
||||
}
|
||||
|
||||
// LoadJuniors fetches junior attendance (cached) and returns reconcile.Members for juniors.
|
||||
func (s *realSources) LoadJuniors(ctx context.Context) ([]reconcile.Member, []string, error) {
|
||||
// Junior data needs both the adult tab (tier="J" rows) and the junior tab.
|
||||
adultRows, err := cache.Get(ctx, s.cache, "attendance_regular", s.attendance.FetchAdults)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("LoadJuniors (adult tab): %w", err)
|
||||
}
|
||||
juniorRows, err := cache.Get(ctx, s.cache, "attendance_juniors", s.attendance.FetchJuniors)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("LoadJuniors (junior tab): %w", err)
|
||||
}
|
||||
return parseJuniorRows(adultRows, juniorRows)
|
||||
}
|
||||
|
||||
// LoadTransactions fetches payment rows from the payments sheet (cached).
|
||||
func (s *realSources) LoadTransactions(ctx context.Context) ([]reconcile.Transaction, error) {
|
||||
rows, err := cache.Get(ctx, s.cache, "payments_transactions",
|
||||
func(ctx context.Context) ([][]any, error) {
|
||||
return s.sheets.GetValues(ctx, config.PaymentsSheetID, "A1:Z")
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("LoadTransactions: %w", err)
|
||||
}
|
||||
return parseTransactionRows(rows)
|
||||
}
|
||||
|
||||
// LoadExceptions fetches the exceptions tab (cached).
|
||||
func (s *realSources) LoadExceptions(ctx context.Context) (map[reconcile.ExceptionKey]reconcile.Exception, error) {
|
||||
rows, err := cache.Get(ctx, s.cache, "exceptions_dict",
|
||||
func(ctx context.Context) ([][]any, error) {
|
||||
return s.sheets.GetValues(ctx, config.PaymentsSheetID, "'exceptions'!A2:D")
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("LoadExceptions: %w", err)
|
||||
}
|
||||
return parseExceptionRows(rows), nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Attendance CSV parsing (ports scripts/attendance.py)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// parseDates returns (columnIndex, YYYY-MM) pairs for all date columns.
|
||||
// Ports scripts/attendance.py parse_dates + strftime("%Y-%m").
|
||||
func parseDates(header []string) []struct {
|
||||
col int
|
||||
month string
|
||||
} {
|
||||
var out []struct {
|
||||
col int
|
||||
month string
|
||||
}
|
||||
for i := firstDateCol; i < len(header); i++ {
|
||||
raw := strings.TrimSpace(header[i])
|
||||
if raw == "" {
|
||||
continue
|
||||
}
|
||||
var dt time.Time
|
||||
var err error
|
||||
// Use the unpadded reference forms ("2.1" and "1/2"): Go's time.Parse
|
||||
// accepts both single-digit and zero-padded inputs against them, so
|
||||
// "1.6.2026", "01.06.2026", "23.3.2026" all parse. Czech sheet authors
|
||||
// drop the leading zero on dates ≤ 9 — Python's strptime is lenient
|
||||
// the same way; the previous "02.01.2006" form silently dropped those
|
||||
// columns and undercounted attendance.
|
||||
for _, fmt_ := range []string{"2.1.2006", "1/2/2006"} {
|
||||
dt, err = time.Parse(fmt_, raw)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, struct {
|
||||
col int
|
||||
month string
|
||||
}{col: i, month: dt.Format("2006-01")})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// groupByMonth groups column indices by YYYY-MM, applying merged month mapping.
|
||||
func groupByMonth(dates []struct {
|
||||
col int
|
||||
month string
|
||||
}, mergedMonths map[string]string,
|
||||
) map[string][]int {
|
||||
out := make(map[string][]int)
|
||||
for _, d := range dates {
|
||||
target := d.month
|
||||
if v, ok := mergedMonths[d.month]; ok {
|
||||
target = v
|
||||
}
|
||||
out[target] = append(out[target], d.col)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// countTrue counts how many cells in the given columns have the value "TRUE" (case-insensitive).
|
||||
func countTrue(row []string, cols []int) int {
|
||||
n := 0
|
||||
for _, c := range cols {
|
||||
if c < len(row) && strings.EqualFold(strings.TrimSpace(row[c]), "true") {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// parseAdultRows converts raw CSV rows to []reconcile.Member.
|
||||
// Includes all tiers; fee is 0 for non-A tiers (reconcile filters downstream).
|
||||
// Ports scripts/attendance.py get_members_with_fees.
|
||||
func parseAdultRows(rows [][]string) ([]reconcile.Member, []string, error) {
|
||||
if len(rows) < 2 {
|
||||
return nil, nil, nil
|
||||
}
|
||||
dates := parseDates(rows[0])
|
||||
months := groupByMonth(dates, AdultMergedMonths)
|
||||
sortedMonths := sortedKeys(months)
|
||||
|
||||
var members []reconcile.Member
|
||||
for _, row := range rows[1:] {
|
||||
if len(row) == 0 {
|
||||
continue
|
||||
}
|
||||
first := strings.TrimSpace(row[colName])
|
||||
if strings.Contains(strings.ToLower(first), "# last line") {
|
||||
break
|
||||
}
|
||||
if strings.HasPrefix(first, "#") || first == "" {
|
||||
continue
|
||||
}
|
||||
if strings.ToLower(first) == "jméno" || strings.ToLower(first) == "name" || strings.ToLower(first) == "jmeno" {
|
||||
continue
|
||||
}
|
||||
tier := ""
|
||||
if len(row) > colTier {
|
||||
tier = strings.ToUpper(strings.TrimSpace(row[colTier]))
|
||||
}
|
||||
|
||||
feeMap := make(map[string]reconcile.FeeData, len(sortedMonths))
|
||||
for _, m := range sortedMonths {
|
||||
cols := months[m]
|
||||
count := countTrue(row, cols)
|
||||
var fee int
|
||||
if tier == "A" {
|
||||
fee = fees.CalculateFee(count, m)
|
||||
}
|
||||
feeMap[m] = reconcile.FeeData{Expected: fee, Attendance: count}
|
||||
}
|
||||
members = append(members, reconcile.Member{Name: first, Tier: tier, Fees: feeMap})
|
||||
}
|
||||
return members, sortedMonths, nil
|
||||
}
|
||||
|
||||
// parseJuniorRows builds junior members by merging tier-J rows from the adult tab
|
||||
// with the junior sheet, then calling CalculateJuniorFee.
|
||||
// Ports scripts/attendance.py get_junior_members_with_fees.
|
||||
func parseJuniorRows(adultRows, juniorRows [][]string) ([]reconcile.Member, []string, error) {
|
||||
if len(adultRows) < 2 || len(juniorRows) < 2 {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
mainDates := parseDates(adultRows[0])
|
||||
juniorDates := parseDates(juniorRows[0])
|
||||
mainMonths := groupByMonth(mainDates, JuniorMergedMonths)
|
||||
jrMonths := groupByMonth(juniorDates, JuniorMergedMonths)
|
||||
|
||||
allMonths := make(map[string]bool)
|
||||
for m := range mainMonths {
|
||||
allMonths[m] = true
|
||||
}
|
||||
for m := range jrMonths {
|
||||
allMonths[m] = true
|
||||
}
|
||||
sortedMonths := sortedKeys(allMonths)
|
||||
|
||||
type counts struct{ adult, junior int }
|
||||
merged := make(map[string]*struct {
|
||||
tier string
|
||||
months map[string]counts
|
||||
})
|
||||
|
||||
// Tier-J rows from adult tab
|
||||
for _, row := range adultRows[1:] {
|
||||
if len(row) == 0 {
|
||||
continue
|
||||
}
|
||||
first := strings.TrimSpace(row[colName])
|
||||
if strings.Contains(strings.ToLower(first), "# last line") {
|
||||
break
|
||||
}
|
||||
if strings.HasPrefix(first, "#") || first == "" {
|
||||
continue
|
||||
}
|
||||
tier := ""
|
||||
if len(row) > colTier {
|
||||
tier = strings.ToUpper(strings.TrimSpace(row[colTier]))
|
||||
}
|
||||
if tier != "J" {
|
||||
continue
|
||||
}
|
||||
if _, ok := merged[first]; !ok {
|
||||
merged[first] = &struct {
|
||||
tier string
|
||||
months map[string]counts
|
||||
}{tier: tier, months: make(map[string]counts)}
|
||||
}
|
||||
for _, m := range sortedMonths {
|
||||
c := merged[first].months[m]
|
||||
c.adult += countTrue(row, mainMonths[m])
|
||||
merged[first].months[m] = c
|
||||
}
|
||||
}
|
||||
|
||||
// All non-X rows from junior tab
|
||||
for _, row := range juniorRows[1:] {
|
||||
if len(row) == 0 {
|
||||
continue
|
||||
}
|
||||
first := strings.TrimSpace(row[colName])
|
||||
fl := strings.ToLower(first)
|
||||
if strings.Contains(fl, "# treneri") || strings.Contains(fl, "# trenéři") {
|
||||
break
|
||||
}
|
||||
if strings.HasPrefix(first, "#") || first == "" {
|
||||
continue
|
||||
}
|
||||
tier := ""
|
||||
if len(row) > colTier {
|
||||
tier = strings.ToUpper(strings.TrimSpace(row[colTier]))
|
||||
}
|
||||
if tier == "X" {
|
||||
continue
|
||||
}
|
||||
if _, ok := merged[first]; !ok {
|
||||
merged[first] = &struct {
|
||||
tier string
|
||||
months map[string]counts
|
||||
}{tier: tier, months: make(map[string]counts)}
|
||||
}
|
||||
for _, m := range sortedMonths {
|
||||
c := merged[first].months[m]
|
||||
c.junior += countTrue(row, jrMonths[m])
|
||||
merged[first].months[m] = c
|
||||
}
|
||||
}
|
||||
|
||||
var members []reconcile.Member
|
||||
for name, data := range merged {
|
||||
feeMap := make(map[string]reconcile.FeeData, len(sortedMonths))
|
||||
for _, m := range sortedMonths {
|
||||
c := data.months[m]
|
||||
total := c.adult + c.junior
|
||||
exp := fees.CalculateJuniorFee(total, m)
|
||||
fee := 0
|
||||
if !exp.Unknown {
|
||||
fee = exp.Value
|
||||
}
|
||||
feeMap[m] = reconcile.FeeData{
|
||||
Expected: fee,
|
||||
IsUnknown: exp.Unknown,
|
||||
Attendance: total,
|
||||
JuniorAttendance: c.junior,
|
||||
AdultAttendance: c.adult,
|
||||
}
|
||||
}
|
||||
members = append(members, reconcile.Member{Name: name, Tier: data.tier, Fees: feeMap})
|
||||
}
|
||||
return members, sortedMonths, nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Payments sheet row parsing (ports scripts/match_payments.py fetch_sheet_data)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func parseTransactionRows(rows [][]any) ([]reconcile.Transaction, error) {
|
||||
if len(rows) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
header := rows[0]
|
||||
|
||||
idx := func(label string) int {
|
||||
label = strings.ToLower(strings.TrimSpace(label))
|
||||
for i, h := range header {
|
||||
if strings.ToLower(strings.TrimSpace(fmt.Sprint(h))) == label {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
idxDate := idx("date")
|
||||
idxAmount := idx("amount")
|
||||
idxManualFix := idx("manual fix")
|
||||
idxPerson := idx("person")
|
||||
idxPurpose := idx("purpose")
|
||||
idxInferred := idx("inferred amount")
|
||||
idxSender := idx("sender")
|
||||
idxVS := idx("vs")
|
||||
idxMessage := idx("message")
|
||||
idxBankID := idx("bank id")
|
||||
idxSyncID := idx("sync id")
|
||||
|
||||
for _, label := range []string{"date", "amount", "person", "purpose"} {
|
||||
if idx(label) == -1 {
|
||||
return nil, fmt.Errorf("payments sheet missing required column %q", label)
|
||||
}
|
||||
}
|
||||
|
||||
getVal := func(row []any, i int) string {
|
||||
if i < 0 || i >= len(row) {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprint(row[i])
|
||||
}
|
||||
|
||||
// getRaw returns row[i] without stringifying — needed for FormatDate to
|
||||
// dispatch on the underlying numeric type (Sheets returns serial-day
|
||||
// numbers as float64). Stringifying first defeats that dispatch.
|
||||
getRaw := func(row []any, i int) any {
|
||||
if i < 0 || i >= len(row) {
|
||||
return nil
|
||||
}
|
||||
return row[i]
|
||||
}
|
||||
|
||||
var txns []reconcile.Transaction
|
||||
for _, row := range rows[1:] {
|
||||
dateStr := matching.FormatDate(getRaw(row, idxDate))
|
||||
amountRaw := row[idxAmount]
|
||||
if idxAmount < 0 || idxAmount >= len(row) {
|
||||
amountRaw = ""
|
||||
}
|
||||
amount := parseFloat(amountRaw)
|
||||
|
||||
var inferredAmount *float64
|
||||
if iv := getVal(row, idxInferred); iv != "" && iv != "<nil>" {
|
||||
if f := parseFloat(iv); f != 0 {
|
||||
inferredAmount = &f
|
||||
}
|
||||
}
|
||||
|
||||
txns = append(txns, reconcile.Transaction{
|
||||
Date: dateStr,
|
||||
Amount: amount,
|
||||
ManualFix: getVal(row, idxManualFix),
|
||||
Person: getVal(row, idxPerson),
|
||||
Purpose: getVal(row, idxPurpose),
|
||||
InferredAmount: inferredAmount,
|
||||
Sender: getVal(row, idxSender),
|
||||
VS: getVal(row, idxVS),
|
||||
Message: getVal(row, idxMessage),
|
||||
BankID: getVal(row, idxBankID),
|
||||
SyncID: getVal(row, idxSyncID),
|
||||
})
|
||||
}
|
||||
return txns, nil
|
||||
}
|
||||
|
||||
func parseFloat(v any) float64 {
|
||||
switch x := v.(type) {
|
||||
case float64:
|
||||
return x
|
||||
case float32:
|
||||
return float64(x)
|
||||
case int:
|
||||
return float64(x)
|
||||
case int64:
|
||||
return float64(x)
|
||||
case string:
|
||||
f, _ := strconv.ParseFloat(strings.TrimSpace(x), 64)
|
||||
return f
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Exceptions tab parsing (ports scripts/match_payments.py fetch_exceptions)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func parseExceptionRows(rows [][]any) map[reconcile.ExceptionKey]reconcile.Exception {
|
||||
out := make(map[reconcile.ExceptionKey]reconcile.Exception)
|
||||
for _, row := range rows {
|
||||
if len(row) < 3 {
|
||||
continue
|
||||
}
|
||||
name := strings.TrimSpace(fmt.Sprint(row[0]))
|
||||
if strings.ToLower(name) == "name" || strings.HasPrefix(strings.ToLower(name), "name") {
|
||||
continue
|
||||
}
|
||||
period := strings.TrimSpace(fmt.Sprint(row[1]))
|
||||
amountStr := fmt.Sprint(row[2])
|
||||
amount, err := strconv.Atoi(strings.TrimSpace(amountStr))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
note := ""
|
||||
if len(row) > 3 {
|
||||
note = strings.TrimSpace(fmt.Sprint(row[3]))
|
||||
}
|
||||
key := reconcile.ExceptionKey{
|
||||
Name: czech.Normalize(name),
|
||||
Period: czech.Normalize(period),
|
||||
}
|
||||
out[key] = reconcile.Exception{Amount: amount, Note: note}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func sortedKeys[V any](m map[string]V) []string {
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
229
go/internal/services/membership/sources_test.go
Normal file
229
go/internal/services/membership/sources_test.go
Normal file
@@ -0,0 +1,229 @@
|
||||
package membership
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fuj-management/go/internal/config"
|
||||
"fuj-management/go/internal/io/attendance"
|
||||
"fuj-management/go/internal/io/cache"
|
||||
"fuj-management/go/internal/io/drive"
|
||||
"fuj-management/go/internal/io/sheets"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// buildSources wires a realSources with in-memory fakes and a no-TTL cache.
|
||||
func buildSources(t *testing.T, att *attendance.Fake, sh *sheets.Fake) *realSources {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
d := &drive.Fake{Times: map[string]string{
|
||||
config.AttendanceSheetID: "t1",
|
||||
config.PaymentsSheetID: "t1",
|
||||
}}
|
||||
fc := cache.New(d, dir, config.CacheSheetMap, 0, 0)
|
||||
return &realSources{attendance: att, sheets: sh, cache: fc}
|
||||
}
|
||||
|
||||
var minimalAdultCSV = [][]string{
|
||||
{"Jméno", "Tier", "", "", "01.09.2025", "08.09.2025"},
|
||||
{"Alice", "A", "", "", "TRUE", "TRUE"},
|
||||
{"Bob", "A", "", "", "TRUE", "FALSE"},
|
||||
{"# last line"},
|
||||
}
|
||||
|
||||
// minimalJuniorCSV has dates in October because the junior merged-month map sends
|
||||
// 2025-09 → 2025-10, so two columns for 01.10.2025 and 08.10.2025 land in "2025-10".
|
||||
var minimalJuniorCSV = [][]string{
|
||||
{"Jméno", "Tier", "", "", "01.10.2025", "08.10.2025"},
|
||||
{"Charlie", "J", "", "", "TRUE", "TRUE"},
|
||||
{"# Trenéři"},
|
||||
{"Coach", "X", "", "", "FALSE", "FALSE"},
|
||||
}
|
||||
|
||||
func TestLoadAdults(t *testing.T) {
|
||||
s := buildSources(t, &attendance.Fake{Adults: minimalAdultCSV}, &sheets.Fake{})
|
||||
|
||||
members, months, err := s.LoadAdults(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// adultMergedMonths is empty so 2025-09 stays as-is
|
||||
if len(months) != 1 || months[0] != "2025-09" {
|
||||
t.Errorf("unexpected months: %v", months)
|
||||
}
|
||||
if len(members) != 2 {
|
||||
t.Fatalf("want 2 members, got %d", len(members))
|
||||
}
|
||||
byName := map[string]int{}
|
||||
for _, m := range members {
|
||||
byName[m.Name] = m.Fees["2025-09"].Attendance
|
||||
}
|
||||
if byName["Alice"] != 2 {
|
||||
t.Errorf("Alice: want 2 sessions, got %d", byName["Alice"])
|
||||
}
|
||||
if byName["Bob"] != 1 {
|
||||
t.Errorf("Bob: want 1 session, got %d", byName["Bob"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadAdults_Fee(t *testing.T) {
|
||||
s := buildSources(t, &attendance.Fake{Adults: minimalAdultCSV}, &sheets.Fake{})
|
||||
members, _, err := s.LoadAdults(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
byName := map[string]int{}
|
||||
for _, m := range members {
|
||||
byName[m.Name] = m.Fees["2025-09"].Expected
|
||||
}
|
||||
// 2 sessions in 2025-09 → AdultFeeMonthlyRate["2025-09"] = 750
|
||||
if byName["Alice"] != 750 {
|
||||
t.Errorf("Alice fee: want 750, got %d", byName["Alice"])
|
||||
}
|
||||
// 1 session → AdultFeeSingle = 200
|
||||
if byName["Bob"] != 200 {
|
||||
t.Errorf("Bob fee: want 200, got %d", byName["Bob"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadJuniors(t *testing.T) {
|
||||
s := buildSources(t,
|
||||
&attendance.Fake{Adults: minimalAdultCSV, Juniors: minimalJuniorCSV},
|
||||
&sheets.Fake{})
|
||||
|
||||
members, months, err := s.LoadJuniors(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(months) == 0 {
|
||||
t.Fatal("want months, got none")
|
||||
}
|
||||
found := false
|
||||
for _, m := range members {
|
||||
if m.Name == "Charlie" {
|
||||
found = true
|
||||
// Charlie has 2 sessions in 2025-10 (October dates in junior CSV)
|
||||
if m.Fees["2025-10"].Attendance != 2 {
|
||||
t.Errorf("Charlie 2025-10 attendance: want 2, got %d", m.Fees["2025-10"].Attendance)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Error("Charlie not found in juniors")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadTransactions(t *testing.T) {
|
||||
// Sheets fake keyed by "<spreadsheetID>/<range>" — use the real constant.
|
||||
// Row 1 uses a pre-formatted date string; row 2 uses the numeric Sheets
|
||||
// serial-day form (float64) — the API returns either depending on cell
|
||||
// formatting, and FormatDate must handle both.
|
||||
paymentsKey := config.PaymentsSheetID + "/A1:Z"
|
||||
sh := &sheets.Fake{Values: map[string][][]any{
|
||||
paymentsKey: {
|
||||
{"Date", "Amount", "manual fix", "Person", "Purpose", "Inferred Amount", "Sender", "VS", "Message", "Bank ID", "Sync ID"},
|
||||
{"2026-04-01", 700.0, "", "Alice", "2026-04", "", "Alice Bank", "", "fee", "", "abc"},
|
||||
{46147.0, 500.0, "", "", "", "", "Bob Bank", "", "platba", "", "def"}, // 46147 serial-day = 2026-05-05
|
||||
},
|
||||
}}
|
||||
s := buildSources(t, &attendance.Fake{}, sh)
|
||||
|
||||
txns, err := s.LoadTransactions(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(txns) != 2 {
|
||||
t.Fatalf("want 2 transactions, got %d", len(txns))
|
||||
}
|
||||
if txns[0].Person != "Alice" {
|
||||
t.Errorf("txn[0].Person: want Alice, got %q", txns[0].Person)
|
||||
}
|
||||
if txns[0].Amount != 700 {
|
||||
t.Errorf("txn[0].Amount: want 700, got %v", txns[0].Amount)
|
||||
}
|
||||
if txns[0].Date != "2026-04-01" {
|
||||
t.Errorf("txn[0].Date: want 2026-04-01, got %q", txns[0].Date)
|
||||
}
|
||||
if txns[1].Date != "2026-05-05" {
|
||||
t.Errorf("txn[1].Date (numeric serial-day): want 2026-05-05, got %q", txns[1].Date)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadExceptions(t *testing.T) {
|
||||
excKey := config.PaymentsSheetID + "/'exceptions'!A2:D"
|
||||
sh := &sheets.Fake{Values: map[string][][]any{
|
||||
excKey: {
|
||||
{"Alice", "2026-04", 350, "reduced"},
|
||||
},
|
||||
}}
|
||||
s := buildSources(t, &attendance.Fake{}, sh)
|
||||
|
||||
exc, err := s.LoadExceptions(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(exc) != 1 {
|
||||
t.Fatalf("want 1 exception, got %d", len(exc))
|
||||
}
|
||||
for k, v := range exc {
|
||||
if v.Amount != 350 {
|
||||
t.Errorf("exception amount: want 350, got %d (key=%v)", v.Amount, k)
|
||||
}
|
||||
if v.Note != "reduced" {
|
||||
t.Errorf("exception note: want 'reduced', got %q", v.Note)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseDates_SingleDigitDayMonth covers the regression where Go's strict
|
||||
// "02.01.2006" format dropped header cells written without leading zeros
|
||||
// (e.g. "1.6.2026", "23.3.2026"), causing attendance undercounts and missing
|
||||
// months on the /api/juniors response. Czech sheet authors drop the zero
|
||||
// pad freely; Python's strptime tolerates it, so the parsers must match.
|
||||
func TestParseDates_SingleDigitDayMonth(t *testing.T) {
|
||||
// Czech form ("DD.MM.YYYY", with leading zeros optional) is the primary
|
||||
// path. The "M/D/YYYY" fallback mirrors Python's %m/%d/%Y secondary
|
||||
// strptime branch — month-first, day-second.
|
||||
header := []string{"Jméno", "Tier", "", "01.06.2026", "1.6.2026", "23.3.2026", "6.4.2026", "01/02/2026", "1/2/2026"}
|
||||
got := parseDates(header)
|
||||
want := []string{"2026-06", "2026-06", "2026-03", "2026-04", "2026-01", "2026-01"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("parseDates: got %d entries, want %d (%v)", len(got), len(want), got)
|
||||
}
|
||||
for i, e := range got {
|
||||
if e.month != want[i] {
|
||||
t.Errorf("parseDates[%d].month = %q, want %q (raw=%q)", i, e.month, want[i], header[e.col])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TTL smoke test: second call within TTL must not call fetch again.
|
||||
func TestLoadAdults_CacheHit(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
d := &drive.Fake{Times: map[string]string{config.AttendanceSheetID: "t1"}}
|
||||
fc := cache.New(d, dir, config.CacheSheetMap, time.Minute, time.Minute)
|
||||
|
||||
calls := 0
|
||||
att := &countingFetcher{rows: minimalAdultCSV, calls: &calls}
|
||||
s := &realSources{attendance: att, sheets: &sheets.Fake{}, cache: fc}
|
||||
|
||||
if _, _, err := s.LoadAdults(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err := s.LoadAdults(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Errorf("want 1 fetch (cache hit on 2nd call), got %d", calls)
|
||||
}
|
||||
}
|
||||
|
||||
type countingFetcher struct {
|
||||
rows [][]string
|
||||
calls *int
|
||||
}
|
||||
|
||||
func (f *countingFetcher) FetchAdults(_ context.Context) ([][]string, error) {
|
||||
*f.calls++
|
||||
return f.rows, nil
|
||||
}
|
||||
func (f *countingFetcher) FetchJuniors(_ context.Context) ([][]string, error) { return nil, nil }
|
||||
27
go/internal/services/membership/stub_test.go
Normal file
27
go/internal/services/membership/stub_test.go
Normal file
@@ -0,0 +1,27 @@
|
||||
package membership
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestStubLoaderReturnsErrIOPending(t *testing.T) {
|
||||
t.Parallel()
|
||||
s := NewStubSources()
|
||||
|
||||
_, _, err := s.LoadAdults(context.Background())
|
||||
if !errors.Is(err, ErrIOPending) {
|
||||
t.Errorf("LoadAdults: want ErrIOPending, got %v", err)
|
||||
}
|
||||
|
||||
_, err = s.LoadTransactions(context.Background())
|
||||
if !errors.Is(err, ErrIOPending) {
|
||||
t.Errorf("LoadTransactions: want ErrIOPending, got %v", err)
|
||||
}
|
||||
|
||||
_, err = s.LoadExceptions(context.Background())
|
||||
if !errors.Is(err, ErrIOPending) {
|
||||
t.Errorf("LoadExceptions: want ErrIOPending, got %v", err)
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user