From 6465e2a221bb9070e28a621de703377d12bd580d Mon Sep 17 00:00:00 2001 From: Jan Novak Date: Thu, 7 May 2026 01:05:59 +0200 Subject: [PATCH 1/4] feat(go): IO layer behind interfaces (M4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - io/attendance: CSV-over-public-URL client + Fake for adult/junior tabs - io/drive: Drive v3 modifiedTime client + Fake - io/sheets: Sheets v4 client (GetValues/AppendValues/BatchUpdateValues/ WriteHeader/SortByDateColumn) + Fake with call-capture - io/cache: Drive-modifiedTime-gated FileCache; two TTL knobs; atomic writes; generic Get[T]; Python-compatible JSON format; Flush() - io/fio: Client interface backed by Fio REST API (apiClient) and HTML scraper (transparentClient); Fake; testdata fixtures - membership/sources: NewSources wires attendance CSV + Sheets + cache into LoadAdults/LoadJuniors/LoadTransactions/LoadExceptions; Czech month parsing + merged-month maps - banksync: SyncToSheets (SHA-256 dedup, optional sort) and InferPayments ([?] review prefix, dry-run) — tested with fakes - cmd/fuj: sync and infer subcommands wired; fees and reconcile use real NewSources; go.mod gains google.golang.org/api + x/net - gofumpt extra-rules applied across all packages; lint clean Co-Authored-By: Claude Opus 4.7 --- CHANGELOG.md | 12 + ...-05-03-2349-go-backend-rewrite-progress.md | 23 +- docs/plans/2026-05-06-2341-go-m4-io-layer.md | 313 ++++++++++++ go/cmd/fuj/main.go | 119 ++++- go/go.mod | 31 +- go/go.sum | 73 +++ go/internal/config/config.go | 39 +- go/internal/io/attendance/client.go | 64 +++ go/internal/io/attendance/client_test.go | 93 ++++ go/internal/io/attendance/fake.go | 12 + .../io/attendance/testdata/adults_minimal.csv | 4 + .../attendance/testdata/juniors_minimal.csv | 4 + go/internal/io/cache/filecache.go | 209 ++++++++ go/internal/io/cache/filecache_test.go | 125 +++++ go/internal/io/drive/client.go | 46 ++ go/internal/io/drive/fake.go | 18 + go/internal/io/fio/api.go | 128 +++++ go/internal/io/fio/client.go | 37 ++ go/internal/io/fio/fake.go | 19 + go/internal/io/fio/fio_test.go | 157 ++++++ go/internal/io/fio/testdata/api_response.json | 29 ++ go/internal/io/fio/testdata/transparent.html | 37 ++ go/internal/io/fio/transparent.go | 217 ++++++++ go/internal/io/sheets/client.go | 124 +++++ go/internal/io/sheets/fake.go | 53 ++ go/internal/services/banksync/infer.go | 170 +++++++ go/internal/services/banksync/infer_test.go | 157 ++++++ go/internal/services/banksync/sync.go | 139 +++++ go/internal/services/banksync/sync_test.go | 140 +++++ go/internal/services/membership/fees_test.go | 4 + go/internal/services/membership/loader.go | 8 +- .../services/membership/reconcile_test.go | 4 + go/internal/services/membership/sources.go | 477 ++++++++++++++++++ .../services/membership/sources_test.go | 198 ++++++++ .../build_name_variants_parity_test.go | 5 +- .../calculate_fee_parity_test.go | 5 +- .../calculate_junior_fee_parity_test.go | 5 +- .../format_date/format_date_parity_test.go | 5 +- .../generate_sync_id_parity_test.go | 5 +- .../infer_transaction_details_parity_test.go | 5 +- .../match_members_parity_test.go | 5 +- .../pure/normalize/normalize_parity_test.go | 5 +- .../parse_czk_amount_parity_test.go | 5 +- .../parse_month_references_parity_test.go | 5 +- .../parity/reconcile/reconcile_parity_test.go | 5 +- 45 files changed, 3292 insertions(+), 46 deletions(-) create mode 100644 docs/plans/2026-05-06-2341-go-m4-io-layer.md create mode 100644 go/internal/io/attendance/client.go create mode 100644 go/internal/io/attendance/client_test.go create mode 100644 go/internal/io/attendance/fake.go create mode 100644 go/internal/io/attendance/testdata/adults_minimal.csv create mode 100644 go/internal/io/attendance/testdata/juniors_minimal.csv create mode 100644 go/internal/io/cache/filecache.go create mode 100644 go/internal/io/cache/filecache_test.go create mode 100644 go/internal/io/drive/client.go create mode 100644 go/internal/io/drive/fake.go create mode 100644 go/internal/io/fio/api.go create mode 100644 go/internal/io/fio/client.go create mode 100644 go/internal/io/fio/fake.go create mode 100644 go/internal/io/fio/fio_test.go create mode 100644 go/internal/io/fio/testdata/api_response.json create mode 100644 go/internal/io/fio/testdata/transparent.html create mode 100644 go/internal/io/fio/transparent.go create mode 100644 go/internal/io/sheets/client.go create mode 100644 go/internal/io/sheets/fake.go create mode 100644 go/internal/services/banksync/infer.go create mode 100644 go/internal/services/banksync/infer_test.go create mode 100644 go/internal/services/banksync/sync.go create mode 100644 go/internal/services/banksync/sync_test.go create mode 100644 go/internal/services/membership/sources.go create mode 100644 go/internal/services/membership/sources_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index fff61fb..c315de1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## 2026-05-07 01:03 CEST — feat(go/M4): IO layer behind interfaces + +- `go/internal/io/attendance`: CSV-over-public-URL client + `Fake` for both adult and junior tabs. +- `go/internal/io/drive`: thin Drive v3 wrapper for `modifiedTime` reads + `Fake`. +- `go/internal/io/sheets`: Sheets v4 client (`GetValues`, `AppendValues`, `BatchUpdateValues`, `WriteHeader`, `SortByDateColumn`) + `Fake` with call-capture for assertions. +- `go/internal/io/cache`: Drive-modifiedTime-gated `FileCache` with two TTL knobs, atomic writes, and generic `Get[T]`; Python-compatible JSON format; `Flush()` support. +- `go/internal/io/fio`: `Client` interface backed by Fio REST API (`apiClient`) and HTML-scraper (`transparentClient`); `Fake` for tests. Fixtures in `testdata/`. +- `go/internal/services/membership/sources.go`: `NewSources` wires attendance CSV + Sheets + cache into `LoadAdults`, `LoadJuniors`, `LoadTransactions`, `LoadExceptions`. Includes Czech month/merged-month parsing logic. +- `go/internal/services/banksync`: `SyncToSheets` (dedup via SHA-256 Sync ID, optional sort) and `InferPayments` (name-match + `[?]` review prefix, dry-run) — fully tested with fakes. +- `go/cmd/fuj/main.go`: `sync` and `infer` subcommands wired to real clients; `fees` and `reconcile` now use real `NewSources`. +- All packages lint-clean (golangci-lint v1.64.8, gofumpt extra-rules). + ## 2026-05-06 23:25 CEST — feat(go/M3): fixture capture + parity test framework - `scripts/capture_fixtures.py`: dispatcher CLI that calls each ported function with seeded inputs and emits captured output as JSON fixtures. diff --git a/docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md b/docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md index 9e44eb2..4ed759d 100644 --- a/docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md +++ b/docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md @@ -2,9 +2,9 @@ Companion to [2026-05-03-2349-go-backend-rewrite.md](2026-05-03-2349-go-backend-rewrite.md). -**Current milestone:** M3 — Fixture capture + characterization framework ✅ +**Current milestone:** M4 — IO layer behind interfaces ✅ **Started:** 2026-05-04 -**Last updated:** 2026-05-06 +**Last updated:** 2026-05-07 ## How to use @@ -80,16 +80,16 @@ Goal: deterministic, PII-free fixture corpus that drives parity tests. Runs in p Goal: every external IO (Sheets, Drive, Fio, file cache) accessed through a narrow Go interface with both a real and a fake implementation. -- [ ] **M4.1** Design IO interfaces (`SheetsClient`, `DriveClient`, `FioClient`, `FileCache`) + in-memory fakes seeded from M3 fixtures -- [ ] **M4.2** `internal/io/sheets` — Google client (read + append + batchUpdate); integration test against a separate test sheet (NOT prod) -- [ ] **M4.3** `internal/io/drive` — Drive `modifiedTime` client + integration test -- [ ] **M4.4** `internal/io/fio` — API JSON impl (token-based); parses by hardcoded `column0..column22` indices matching [fio_utils.py](scripts/fio_utils.py) -- [ ] **M4.5** `internal/io/fio` — transparent-page HTML scraper using `golang.org/x/net/html` token visitor; targets the **second** `` -- [ ] **M4.6** `internal/io/cache` — FileCache with `modifiedTime` gating + two TTL knobs + atomic writes (`os.Rename`) -- [ ] **M4.7** `services/banksync.SyncToSheets` + `fuj sync` subcommand -- [ ] **M4.8** `services/banksync.InferPayments` + `fuj infer [--dry-run]` subcommand +- [x] **M4.1** Design IO interfaces (`SheetsClient`, `DriveClient`, `FioClient`, `FileCache`) + in-memory fakes seeded from M3 fixtures +- [x] **M4.2** `internal/io/sheets` — Google client (read + append + batchUpdate); fake with call-capture +- [x] **M4.3** `internal/io/drive` — Drive `modifiedTime` client + fake +- [x] **M4.4** `internal/io/fio` — API JSON impl (token-based); parses by hardcoded `column0..column22` indices matching [fio_utils.py](scripts/fio_utils.py) +- [x] **M4.5** `internal/io/fio` — transparent-page HTML scraper using `golang.org/x/net/html` token visitor; targets the **second** `
` +- [x] **M4.6** `internal/io/cache` — FileCache with `modifiedTime` gating + two TTL knobs + atomic writes (`os.Rename`) +- [x] **M4.7** `services/banksync.SyncToSheets` + `fuj sync` subcommand +- [x] **M4.8** `services/banksync.InferPayments` + `fuj infer [--dry-run]` subcommand; `NewSources` wires all IO into fees+reconcile -**Gate:** `go test -tags=integration ./internal/io/...` round-trips against test sheet; default-tag tests run on fakes. +**Gate:** ✅ Fakes-only unit tests; `make go-test` + `make go-lint` both green. Live smoke test deferred to first real sync run. --- @@ -155,4 +155,5 @@ Goal: Go is the one true backend. (Add entries as you go. Format: `YYYY-MM-DD — short note`.) - 2026-05-04 — Plan approved. Versioning policy: latest stable for Go and all libs at the time M1 starts. Frontends explicitly allowed to diverge between Python and Go; only the JSON API contract is parity-locked. No reverse proxy — both backends run on different ports via `make web-py` / `make web-go`. +- 2026-05-07 — M4 complete. Chose fakes-only unit tests (no live integration tests) and CSV-via-public-URL for attendance (no Sheets API auth required for read-only). golangci-lint gofumpt extra-rules differ slightly from standalone gofumpt; used `golangci-lint run --fix --enable-only gofumpt` to auto-resolve formatting. - 2026-05-04 — M1 complete. Dockerfile base changed from `distroless/static:nonroot` → `alpine:3` for debuggability (can tighten later). CLI dispatcher uses stdlib `flag`; module path `fuj-management/go`. golangci-lint v1 embedded gofumpt merges all imports into one group (no stdlib/local split) — accepted as the project style. diff --git a/docs/plans/2026-05-06-2341-go-m4-io-layer.md b/docs/plans/2026-05-06-2341-go-m4-io-layer.md new file mode 100644 index 0000000..78ac8be --- /dev/null +++ b/docs/plans/2026-05-06-2341-go-m4-io-layer.md @@ -0,0 +1,313 @@ +# Plan: Go rewrite — M4 IO layer behind interfaces + +Companion to [2026-05-03-2349-go-backend-rewrite.md](2026-05-03-2349-go-backend-rewrite.md) +and [2026-05-03-2349-go-backend-rewrite-progress.md](2026-05-03-2349-go-backend-rewrite-progress.md). + +## Context + +M1–M3 are merged: skeleton + tooling, every pure-domain function ported and +parity-tested against PII-scrubbed fixtures, and the `fuj fees` / `fuj +reconcile` subcommands wired but stubbed (`membership.NewStubSources()` +returns `ErrIOPending` for every loader). M4's job is to replace that stub +with real IO: read attendance CSVs, read the payments sheet + exceptions +tab, fetch Drive `modifiedTime` for cache gating, fetch Fio bank +transactions, and append/update rows on the payments sheet — all behind +narrow Go interfaces that have in-memory fakes for tests. + +Once M4 lands, `fuj fees`, `fuj reconcile`, `fuj sync`, and `fuj infer` all +work end-to-end against the real Google Sheets and the real Fio account, and +M5 can start porting the JSON API on top of that IO. + +User-confirmed scope choices for this milestone: +- **No live integration tests.** Fakes-only at unit level; live + verification deferred to manual smoke during M7. +- **Three PRs** (sheets/drive/cache → fio/sync → infer), one per major + area, each independently reviewable. +- **Attendance stays on CSV-via-public-URL** — matches Python, no extra + service-account grant needed. + +## Approach + +### Layering + +``` +internal/io/ ← raw, narrow clients (one per external system) + sheets/ ← typed wrapper around google.golang.org/api/sheets/v4 + drive/ ← Drive v3, only ModifiedTime + attendance/ ← CSV-via-public-URL fetcher (no auth, no Sheets API) + fio/ ← FioClient interface + apiClient + transparentClient + cache/ ← FileCache: modifiedTime gate + two-TTL fallback + atomic write + +internal/services/membership/ ← already exists; M4 adds adapters that satisfy + AttendanceLoader / TransactionLoader / ExceptionLoader + by composing io/sheets + io/drive + io/cache + io/attendance. + +internal/services/banksync/ ← new: SyncToSheets (M4.7) + InferPayments (M4.8) + composing fio + sheets + attendance loaders. +``` + +The existing interfaces in [go/internal/services/membership/loader.go](../../go/internal/services/membership/loader.go) +(`AttendanceLoader`, `TransactionLoader`, `ExceptionLoader`, `Sources`) are +the seam — M4 adds a `NewSources(cfg config.Config) (Sources, error)` +constructor next to `NewStubSources()`, and `cmd/fuj/main.go` swaps the +stub for it. + +### Auth — service-account only + +Drop the OAuth+`token.pickle` path entirely (the production already uses a +service account; the fallback only existed because the original Python +script ran from a developer laptop). Sheets and Drive both authenticate via +`option.WithCredentialsFile(cfg.CredentialsPath)` plus +`option.WithScopes(...)`. Single shared `*http.Client` per backend with a +10s timeout (matches `DRIVE_TIMEOUT`). + +### Cache shape + +Match Python's wire format so the `tmp/*_cache.json` directory is shared +safely while both backends run side-by-side: + +```json +{ "modifiedTime": "", "data": , "cachedAt": "" } +``` + +Improvements over Python: +- Atomic write: marshal → `os.WriteFile(path+".tmp", ..., 0o600)` → + `os.Rename`. Python's plain truncate-write stays as-is until M8. +- The two TTLs (`CacheTTL` and `CacheAPICheckTTL`) live in `config.Config` + already; only the `CacheDir` field is new. + +The four cache keys mirror Python's `CACHE_SHEET_MAP`: +`attendance_regular`, `attendance_juniors`, `exceptions_dict`, +`payments_transactions` → maps to either `AttendanceSheetID` or +`PaymentsSheetID`. + +When Drive fails, fall back to a synthetic key +`fmt.Sprintf("ttl-5m-%d", time.Now().Unix()/300)` so cache still keys +deterministically per 5-min bucket (same as Python). + +### Fio: two impls behind one interface + +```go +type Client interface { + FetchTransactions(ctx context.Context, from, to time.Time) ([]Transaction, error) +} +``` + +`apiClient` (when `cfg.FioAPIToken != ""`) hits +`https://fioapi.fio.cz/v1/rest/periods/{token}/{from}/{to}/transactions.json`, +unmarshals via a typed struct, and maps `column0..column22` to fields per +[scripts/fio_utils.py](../../scripts/fio_utils.py:90). Negative-amount rows +dropped (matches Python). + +`transparentClient` (fallback) GETs +`https://ib.fio.cz/ib/transparent?a={accountNum}&f={DD.MM.YYYY}&t={DD.MM.YYYY}` +and walks the response with `golang.org/x/net/html` token visitor, counting +`
` tags and grabbing rows from the **second** one +(skipping ``). `bank_id`, `currency`, `user_id`, `sender_account` +are empty (matches Python — known limitation). + +`accountNum` is derived from `cfg.BankAccount` by stripping the IBAN prefix +(`CZ85 2010 0000 0028 0035 9168` → `2800359168`); add a small helper in +`config` for this since both the API URL and the transparent URL need it. + +### Fakes + +In-memory fakes live next to each real impl: `sheets/fake.go`, +`drive/fake.go`, `fio/fake.go`, `attendance/fake.go`, +`cache/fake.go` (a passthrough). All exported as `Fake` so tests do +`sheets.NewFake(rows)` and inject. The membership-adapter tests use these +fakes plus a couple of new raw-bytes fixtures under +`go/internal/io//testdata/`: + +- `sheets/testdata/payments_minimal.json` — 2D-string array shaped like + `values.get` would return. +- `sheets/testdata/exceptions_minimal.json` — same, for the exceptions tab. +- `attendance/testdata/adults_minimal.csv` — small adult attendance CSV. +- `attendance/testdata/juniors_minimal.csv` — small junior CSV. +- `fio/testdata/api_response.json` — captured Fio API JSON shape. +- `fio/testdata/transparent.html` — captured transparent-page HTML. + +Existing M3 domain fixtures under `go/tests/fixtures/` stay where they are +and continue to drive parity tests; they aren't reused for IO-layer tests +because they're at the wrong layer (post-parse domain types). + +## Tasks (mapped to tracker) + +Same 8 sub-milestones as the tracker, grouped into 3 PRs. + +### PR 1 — sheets / drive / cache + membership wiring (M4.1, M4.2, M4.3, M4.6) + +1. **Add deps** in [go/go.mod](../../go/go.mod): + `google.golang.org/api/{sheets/v4,drive/v3,option}`, + `golang.org/x/oauth2/google` (transitively pulled), `golang.org/x/net/html`. +2. **`internal/io/sheets/`**: + - `client.go` — `Client` struct holding `*sheets.Service`; methods + `GetValues(ctx, spreadsheetID, a1Range string) ([][]any, error)`, + `AppendValues(ctx, spreadsheetID, a1Range string, rows [][]any) error`, + `BatchUpdateValues(ctx, spreadsheetID, updates []ValueRange) error`, + `SortByColumn(ctx, spreadsheetID, sheetGID int64, columnIndex int) error`. + - `fake.go` — exported `Fake` with seedable `Values map[string][][]any`. +3. **`internal/io/drive/`**: + - `client.go` — `Client.ModifiedTime(ctx, fileID string) (string, error)` + using `drive.New(...).Files.Get(fileID).Fields("modifiedTime").SupportsAllDrives(true)`. + - `fake.go` with seedable `Times map[string]string`. +4. **`internal/io/attendance/`** (new — public-URL CSV): + - `client.go` — `Client.FetchAdults(ctx) ([][]string, error)` and + `FetchJuniors(ctx) ([][]string, error)` using `http.Get` on + `https://docs.google.com/spreadsheets/d/{ID}/export?format=csv&gid={GID}`, + decoded via `encoding/csv`. + - Add `AttendanceAdultSheetGID = "0"` constant in `internal/config`. +5. **`internal/io/cache/`**: + - `filecache.go` — `FileCache` with `Get(ctx, key string, fetch func(ctx) (any, error)) (any, error)` + wired through `Drive.ModifiedTime` and the two TTL knobs. Atomic write + via tmp-file + rename. + - Cache key → sheet ID map mirrors Python's `CACHE_SHEET_MAP`. +6. **`internal/services/membership/sources.go`** (new file in existing + package): + - `realSources struct { sheets *sheets.Client; drive *drive.Client; attendance *attendance.Client; cache *cache.FileCache }`. + - Constructor `NewSources(ctx, cfg) (Sources, error)` builds all clients. + - `LoadAdults` reads cached attendance CSV, runs through + `domain/fees.CalculateFee` + merged-month logic (port of + [scripts/attendance.py](../../scripts/attendance.py:170) + `get_members_with_fees`), returns `[]reconcile.Member`. + - `LoadTransactions` reads payments sheet rows via cache, parses to + `[]reconcile.Transaction` (port of + [match_payments.py:208](../../scripts/match_payments.py:208) + `fetch_sheet_data`). + - `LoadExceptions` reads `'exceptions'!A2:D` via cache, builds + `map[ExceptionKey]Exception` (port of `match_payments.py:266`). +7. **Add `LoadJuniors`** to the `AttendanceLoader` interface (Python infer + pulls both adult + junior member lists; needed for M4.8). +8. **Wire into [cmd/fuj/main.go](../../go/cmd/fuj/main.go)**: replace + `membership.NewStubSources()` in `feesCmd` and `reconcileCmd` with + `membership.NewSources(ctx, cfg)`. +9. **Tests** (default tag, no live IO): + - `sheets/client_test.go`, `drive/client_test.go`, + `cache/filecache_test.go` — exercise fakes + parsing logic with + testdata fixtures. + - `membership/sources_test.go` — adapter tests with sheets/drive/cache + fakes verify CSV→Member, rows→Transaction, exceptions tab → map. +10. **Config additions**: `CacheDir` (default `tmp` relative to `$PWD`, + overridable via `CACHE_DIR` env), `DriveTimeout` (default 10s). +11. **Manual verification**: `make go-build && go run ./cmd/fuj fees` and + `... reconcile` print real reports against the live sheet (with valid + `.secret/...credentials.json`). +12. CHANGELOG entry; tick M4.1, M4.2, M4.3, M4.6 in the progress tracker. + +### PR 2 — fio + bank sync (M4.4, M4.5, M4.7) + +1. **`internal/io/fio/`**: + - `client.go` — `Client` interface, `Transaction` struct. + - `api.go` — `apiClient` impl + URL builder + JSON struct definitions + for `accountStatement.transactionList.transaction[].column{N}.value`. + - `transparent.go` — `transparentClient` impl using + `golang.org/x/net/html` token visitor; helper functions + `parseCzechAmount` (NBSP/space strip + comma→dot) and + `parseCzechDate` (DD.MM.YYYY / DD/MM/YYYY). + - `fake.go`. + - `New(cfg) Client` chooses impl based on `cfg.FioAPIToken`. + - `accountNum(iban)` helper in `internal/config` strips IBAN prefix. +2. **`internal/services/banksync/sync.go`** (new package): + - `SyncToSheets(ctx, cfg, fio Client, sheets *sheets.Client, opts SyncOpts) (added int, err error)`. + - Reads existing rows via `sheets.GetValues(... "A1:K")`, validates + header against `COLUMN_LABELS`, writes header if missing, builds + `existingIDs` from column K (`Sync ID`). + - Computes date window: explicit `from`/`to` or `now - days*24h` (default 30d). + - For each fetched tx, computes `domain/synch.GenerateSyncID`, skips if + present, otherwise builds row in COLUMN_LABELS order with empty + manual/person/purpose/inferred slots. + - `sheets.AppendValues(... "A2", rows)`. + - Optional sort: `sheets.SortByColumn(... gid, 0)` — sheet GID resolved + once via `spreadsheets.Get`. +3. **Wire `fuj sync` subcommand** in `cmd/fuj/main.go`: + - Flags: `--days N` (default 30), `--from YYYY-MM-DD`, `--to YYYY-MM-DD`, + `--sort` (default true matching `make sync-2026`). + - Replace the M4-stub error path. +4. **Tests** (default tag): `banksync/sync_test.go` with fakes — verify + header insertion, dedup against existing sync IDs, multi-row append, + sort call. +5. **Manual verification**: dry-run sync against the real Fio account in a + throwaway test sheet; or visually verify `--from --to` window in stdout + with a no-write flag (only if cheap to add — otherwise skip per the + "no live integration tests" decision). +6. CHANGELOG entry; tick M4.4, M4.5, M4.7. + +### PR 3 — infer (M4.8) + +1. **`internal/services/banksync/infer.go`**: + - `InferPayments(ctx, cfg, sheets *sheets.Client, attendanceLoader, juniorLoader, opts InferOpts) (updated int, err error)`. + - Reads payments sheet `A1:Z` with case-insensitive header lookup. + - Required columns: `Person, Purpose, Inferred Amount`. Optional input: + `Date, Amount, Sender, Message, VS, manual fix`. + - Skip rule (matches [scripts/infer_payments.py:127](../../scripts/infer_payments.py:127)): + non-empty `manual fix` OR `Person` OR `Purpose` → leave row alone. + - Member list = union of `LoadAdults` + `LoadJuniors` deduped via + `domain/matching.CanonicalKey` (already exists from M2). + - For each empty row: build tx dict, call + `domain/matching.InferTransactionDetails`, prefix `[?] ` if + confidence == "review", emit a `ValueRange` update with R1C1 range + `R{i}C{personCol+1}:R{i}C{amountCol+1}`. + - Single `sheets.BatchUpdateValues` call for all updates. +2. **Wire `fuj infer` subcommand**: flags `--dry-run` (prints planned + updates, no API write). +3. **Tests** (default tag): `banksync/infer_test.go` — fixture rows, + verify skip rule, verify `[?]` prefix on review matches, verify + batchUpdate payload shape, verify `--dry-run` is no-op. +4. CHANGELOG entry; tick M4.8 → milestone gate ✅. + +## Critical files + +To modify: +- [go/internal/services/membership/loader.go](../../go/internal/services/membership/loader.go) — add `LoadJuniors` to `AttendanceLoader`, add `NewSources`. +- [go/cmd/fuj/main.go](../../go/cmd/fuj/main.go) — swap stub for real sources, add `sync`/`infer` subcommands. +- [go/internal/config/config.go](../../go/internal/config/config.go) — add `CacheDir`, `DriveTimeout`, `AttendanceAdultSheetGID` constant, IBAN→account-num helper. +- [go/go.mod](../../go/go.mod) / `go.sum` — google APIs + `x/net/html`. +- [docs/plans/2026-05-03-2349-go-backend-rewrite-progress.md](2026-05-03-2349-go-backend-rewrite-progress.md) — tick M4.x boxes after each PR. +- [CHANGELOG.md](../../CHANGELOG.md) — entry per PR. + +To create: +- `go/internal/io/{sheets,drive,attendance,fio,cache}/{client,fake,*_test}.go` +- `go/internal/io/{sheets,attendance,fio}/testdata/*` +- `go/internal/services/membership/sources.go` (+ `sources_test.go`) +- `go/internal/services/banksync/{sync,infer}.go` (+ tests) + +## Reused existing helpers + +- `domain/fees.CalculateFee` / `CalculateJuniorFee` — fee math (M2.3, M2.4). +- `domain/matching.{BuildNameVariants,MatchMembers,InferTransactionDetails,FormatDate,CanonicalKey}` — match logic (M2.7–M2.9). +- `domain/synch.GenerateSyncID` — dedup hash (M2.6). +- `domain/reconcile.{Member,Transaction,Exception,ExceptionKey}` — domain types. +- `domain/czech.{Normalize,ParseMonthReferences}` — used inside the + attendance/exceptions parsers. +- `domain/money.ParseCZK` — for parsing transparent-scrape amounts. + +## Verification + +End-to-end checks once all three PRs land: + +1. `make go-build && make go-lint && make go-test` — clean. +2. `make go-parity` — M3 fixtures still pass (no domain regressions). +3. `./bin/fuj fees` — prints adult fee report matching Python `make fees` + (visual diff acceptable for now; byte-equality enforced in M5). +4. `./bin/fuj reconcile` — prints balance report comparable to + [scripts/match_payments.py](../../scripts/match_payments.py) `print_balance_report`. +5. `./bin/fuj sync --days 7` — appends new Fio rows to the payments sheet + (run with a real but recent date window; verify by counting added rows + and confirming no duplicates on a second run). +6. `./bin/fuj infer --dry-run` — prints planned Person/Purpose/Inferred + Amount updates without modifying the sheet. Then `./bin/fuj infer` + applies them; second run is a no-op (skip rule). +7. **Cache check**: delete `tmp/*_cache.json`, run `fuj fees`, verify file + appears with `modifiedTime` matching Drive. Re-run within 5 min; + verify no Drive call (debug log). +8. **Cross-process cache safety**: while `make web-py` is running, run + `fuj reconcile`; verify Python's cache file isn't corrupted and Go + reads the same data. + +Gate (per tracker): +> `go test -tags=integration ./internal/io/...` round-trips against test sheet; default-tag tests run on fakes. + +Per the user's scope decision, **the integration-test gate is downgraded +to "default-tag tests on fakes" only**. Live verification is deferred to +manual smoke during M7's parallel-run watch period. The progress tracker's +M4 gate line will be amended in PR 1. diff --git a/go/cmd/fuj/main.go b/go/cmd/fuj/main.go index 3123351..8c7a439 100644 --- a/go/cmd/fuj/main.go +++ b/go/cmd/fuj/main.go @@ -5,7 +5,10 @@ import ( "flag" "fmt" "fuj-management/go/internal/config" + "fuj-management/go/internal/io/fio" + "fuj-management/go/internal/io/sheets" "fuj-management/go/internal/logging" + "fuj-management/go/internal/services/banksync" "fuj-management/go/internal/services/membership" "fuj-management/go/internal/web" "os" @@ -36,9 +39,10 @@ func main() { feesCmd(args) case "reconcile": reconcileCmd(args) - case "sync", "infer": - fmt.Fprintf(os.Stderr, "fuj %s: not implemented yet (lands in M4)\n", cmd) - os.Exit(2) + case "sync": + syncCmd(args) + case "infer": + inferCmd(args) case "-h", "--help", "help": usage() default: @@ -84,8 +88,14 @@ func feesCmd(args []string) { os.Exit(2) } - sources := membership.NewStubSources() - if err := membership.FeesReport(context.Background(), sources, os.Stdout); err != nil { + ctx := context.Background() + cfg := config.Load() + sources, err := membership.NewSources(ctx, cfg) + if err != nil { + fmt.Fprintf(os.Stderr, "fuj fees: %v\n", err) + os.Exit(1) + } + if err := membership.FeesReport(ctx, sources, os.Stdout); err != nil { fmt.Fprintf(os.Stderr, "fuj fees: %v\n", err) os.Exit(1) } @@ -101,8 +111,14 @@ func reconcileCmd(args []string) { os.Exit(2) } - sources := membership.NewStubSources() - if err := membership.ReconcileReport(context.Background(), sources, time.Now().Year(), os.Stdout); err != nil { + ctx := context.Background() + cfg := config.Load() + sources, err := membership.NewSources(ctx, cfg) + if err != nil { + fmt.Fprintf(os.Stderr, "fuj reconcile: %v\n", err) + os.Exit(1) + } + if err := membership.ReconcileReport(ctx, sources, time.Now().Year(), os.Stdout); err != nil { fmt.Fprintf(os.Stderr, "fuj reconcile: %v\n", err) os.Exit(1) } @@ -112,6 +128,91 @@ func versionCmd() { fmt.Printf("fuj %s (%s) built %s\n", version, commit, buildDate) } +func syncCmd(args []string) { + fs := flag.NewFlagSet("sync", flag.ExitOnError) + days := fs.Int("days", 30, "look-back window in days (ignored when --from/--to are set)") + fromStr := fs.String("from", "", "start date YYYY-MM-DD") + toStr := fs.String("to", "", "end date YYYY-MM-DD") + sort := fs.Bool("sort", true, "sort sheet by date after appending") + fs.Usage = func() { + fmt.Fprintln(os.Stderr, "usage: fuj sync [--days N] [--from YYYY-MM-DD --to YYYY-MM-DD] [--sort]") + fs.PrintDefaults() + } + if err := fs.Parse(args); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } + + ctx := context.Background() + cfg := config.Load() + + sheetsCli, err := sheets.New(ctx, cfg.CredentialsPath, cfg.DriveTimeout) + if err != nil { + fmt.Fprintf(os.Stderr, "fuj sync: sheets client: %v\n", err) + os.Exit(1) + } + fioCli := fio.New(cfg.FioAPIToken, config.IBANAccountNum(cfg.BankAccount), nil) + + opts := banksync.SyncOpts{Days: *days, Sort: *sort} + if *fromStr != "" && *toStr != "" { + opts.From, err = time.Parse("2006-01-02", *fromStr) + if err != nil { + fmt.Fprintf(os.Stderr, "fuj sync: invalid --from: %v\n", err) + os.Exit(2) + } + opts.To, err = time.Parse("2006-01-02", *toStr) + if err != nil { + fmt.Fprintf(os.Stderr, "fuj sync: invalid --to: %v\n", err) + os.Exit(2) + } + } + + n, err := banksync.SyncToSheets(ctx, config.PaymentsSheetID, fioCli, sheetsCli, opts) + if err != nil { + fmt.Fprintf(os.Stderr, "fuj sync: %v\n", err) + os.Exit(1) + } + fmt.Printf("Synced %d new transaction(s).\n", n) +} + +func inferCmd(args []string) { + fs := flag.NewFlagSet("infer", flag.ExitOnError) + dryRun := fs.Bool("dry-run", false, "print planned updates without writing to the sheet") + fs.Usage = func() { + fmt.Fprintln(os.Stderr, "usage: fuj infer [--dry-run]") + fs.PrintDefaults() + } + if err := fs.Parse(args); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } + + ctx := context.Background() + cfg := config.Load() + + sheetsCli, err := sheets.New(ctx, cfg.CredentialsPath, cfg.DriveTimeout) + if err != nil { + fmt.Fprintf(os.Stderr, "fuj infer: sheets client: %v\n", err) + os.Exit(1) + } + sources, err := membership.NewSources(ctx, cfg) + if err != nil { + fmt.Fprintf(os.Stderr, "fuj infer: sources: %v\n", err) + os.Exit(1) + } + + n, err := banksync.InferPayments(ctx, config.PaymentsSheetID, sheetsCli, sources, banksync.InferOpts{DryRun: *dryRun}) + if err != nil { + fmt.Fprintf(os.Stderr, "fuj infer: %v\n", err) + os.Exit(1) + } + if *dryRun { + fmt.Printf("Dry run: would update %d row(s).\n", n) + } else { + fmt.Printf("Updated %d row(s).\n", n) + } +} + func usage() { fmt.Fprintln(os.Stderr, `usage: fuj [flags] @@ -120,6 +221,6 @@ Commands: version Print version information fees Calculate monthly fees reconcile Show balance report - sync Sync Fio transactions [M4] - infer Infer payment details [M4]`) + sync Sync Fio transactions to payments sheet + infer Infer payment details in payments sheet`) } diff --git a/go/go.mod b/go/go.mod index 718127a..1cb254e 100644 --- a/go/go.mod +++ b/go/go.mod @@ -2,4 +2,33 @@ module fuj-management/go go 1.26.1 -require golang.org/x/text v0.36.0 +require ( + golang.org/x/net v0.53.0 + golang.org/x/text v0.36.0 + google.golang.org/api v0.278.0 +) + +require ( + cloud.google.com/go/auth v0.20.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.15 // indirect + github.com/googleapis/gax-go/v2 v2.22.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect + golang.org/x/crypto v0.50.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sys v0.43.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect + google.golang.org/grpc v1.80.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect +) diff --git a/go/go.sum b/go/go.sum index 65d3299..11193b0 100644 --- a/go/go.sum +++ b/go/go.sum @@ -1,2 +1,75 @@ +cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= +cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.15 h1:xolVQTEXusUcAA5UgtyRLjelpFFHWlPQ4XfWGc7MBas= +github.com/googleapis/enterprise-certificate-proxy v0.3.15/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg= +github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU5vlZD4= +github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= +golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/api v0.278.0 h1:W7jiRvRi53VYFfZ/HoZjQBtJk7gOFbHD8ot1RzVZU6E= +google.golang.org/api v0.278.0/go.mod h1:B9TqLBwJqVjp1mtt7WeoQwWRwvu/400y5lETOql+giQ= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= +google.golang.org/genproto/googleapis/api v0.0.0-20260319201613-d00831a3d3e7 h1:41r6JMbpzBMen0R/4TZeeAmGXSJC7DftGINUodzTkPI= +google.golang.org/genproto/googleapis/api v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:EIQZ5bFCfRQDV4MhRle7+OgjNtZ6P1PiZBgAKuxXu/Y= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 h1:tEkOQcXgF6dH1G+MVKZrfpYvozGrzb91k6ha7jireSM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= +google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go/internal/config/config.go b/go/internal/config/config.go index cceb53e..40815ca 100644 --- a/go/internal/config/config.go +++ b/go/internal/config/config.go @@ -3,6 +3,7 @@ package config import ( "os" "strconv" + "strings" "time" ) @@ -10,16 +11,34 @@ import ( const ( AttendanceSheetID = "1E2e_gT_K5AwSRCDLDTa2UetZTkHmBOcz0kFbBUNUNBA" PaymentsSheetID = "1Om0YPoDVCH5cV8BrNz5LG5eR5MMU05ypQC7UMN1xn_Y" - JuniorSheetGID = "1213318614" + + // Both attendance tabs live in the same Google Spreadsheet (AttendanceSheetID). + // The original adult and junior attendance data lives in separate source spreadsheets, + // but is collected into this one sheet via IMPORTRANGE — one tab per group. + // Tabs are identified by the gid= query param in the CSV export URL. + AttendanceAdultSheetGID = "0" // gid=0 — adult practices tab (IMPORTRANGE'd) + JuniorSheetGID = "1213318614" // gid=1213318614 — junior practices tab (IMPORTRANGE'd) ) +// CacheSheetMap mirrors scripts/config.py CACHE_SHEET_MAP. +// Maps a cache key to the Google Sheet ID whose Drive modifiedTime gates it. +// Both attendance keys map to the same spreadsheet — different tabs, one Drive file. +var CacheSheetMap = map[string]string{ + "attendance_regular": AttendanceSheetID, + "attendance_juniors": AttendanceSheetID, + "exceptions_dict": PaymentsSheetID, + "payments_transactions": PaymentsSheetID, +} + // Config holds all runtime configuration loaded from environment variables. // Mirrors scripts/config.py. type Config struct { CredentialsPath string BankAccount string + CacheDir string CacheTTL time.Duration CacheAPICheckTTL time.Duration + DriveTimeout time.Duration LogLevel string FioAPIToken string ServerAddr string @@ -31,14 +50,32 @@ func Load() Config { return Config{ CredentialsPath: env("CREDENTIALS_PATH", ".secret/fuj-management-bot-credentials.json"), BankAccount: env("BANK_ACCOUNT", "CZ8520100000002800359168"), + CacheDir: env("CACHE_DIR", "tmp"), CacheTTL: envDuration("CACHE_TTL_SECONDS", 300), CacheAPICheckTTL: envDuration("CACHE_API_CHECK_TTL_SECONDS", 300), + DriveTimeout: envDuration("DRIVE_TIMEOUT_SECONDS", 10), LogLevel: env("LOG_LEVEL", "INFO"), FioAPIToken: env("FIO_API_TOKEN", ""), ServerAddr: env("SERVER_ADDR", ":8080"), } } +// IBANAccountNum extracts the bare account number from a Czech IBAN. +// "CZ8520100000002800359168" → "2800359168" +// Structure: CZ(2 check)(4 bank code)(16 zero-padded account). +func IBANAccountNum(iban string) string { + s := strings.ReplaceAll(iban, " ", "") + if len(s) < 8 { + return iban + } + raw := s[8:] // 16-digit zero-padded account portion + n := strings.TrimLeft(raw, "0") + if n == "" { + return "0" + } + return n +} + func env(key, fallback string) string { if v := os.Getenv(key); v != "" { return v diff --git a/go/internal/io/attendance/client.go b/go/internal/io/attendance/client.go new file mode 100644 index 0000000..4db3bca --- /dev/null +++ b/go/internal/io/attendance/client.go @@ -0,0 +1,64 @@ +// Package attendance fetches attendance CSV exports from Google Sheets. +// No auth required — the sheet must be publicly readable. +package attendance + +import ( + "context" + "encoding/csv" + "fmt" + "io" + "net/http" + "strings" +) + +const exportBase = "https://docs.google.com/spreadsheets/d" + +// Client fetches attendance CSV exports from a public Google Spreadsheet. +type Client struct { + http *http.Client + sheetID string + adultGID string + juniorGID string +} + +// New returns a Client for the given spreadsheet. +// adultGID is typically "0"; juniorGID is the GID of the junior tab. +func New(httpClient *http.Client, sheetID, adultGID, juniorGID string) *Client { + if httpClient == nil { + httpClient = http.DefaultClient + } + return &Client{http: httpClient, sheetID: sheetID, adultGID: adultGID, juniorGID: juniorGID} +} + +// FetchAdults returns the adult attendance tab as raw CSV rows. +func (c *Client) FetchAdults(ctx context.Context) ([][]string, error) { + return c.fetch(ctx, c.adultGID) +} + +// FetchJuniors returns the junior attendance tab as raw CSV rows. +func (c *Client) FetchJuniors(ctx context.Context) ([][]string, error) { + return c.fetch(ctx, c.juniorGID) +} + +func (c *Client) fetch(ctx context.Context, gid string) ([][]string, error) { + url := fmt.Sprintf("%s/%s/export?format=csv&gid=%s", exportBase, c.sheetID, gid) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := c.http.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("attendance fetch: HTTP %d for gid=%s", resp.StatusCode, gid) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + r := csv.NewReader(strings.NewReader(string(body))) + r.FieldsPerRecord = -1 // rows may have different lengths + return r.ReadAll() +} diff --git a/go/internal/io/attendance/client_test.go b/go/internal/io/attendance/client_test.go new file mode 100644 index 0000000..5e39366 --- /dev/null +++ b/go/internal/io/attendance/client_test.go @@ -0,0 +1,93 @@ +package attendance + +import ( + "context" + "encoding/csv" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" +) + +func TestClientFetchAdults(t *testing.T) { + data, err := os.ReadFile("testdata/adults_minimal.csv") + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(data) + })) + defer srv.Close() + + // Point the client at our test server by re-implementing fetch against its URL. + rows, err := fetchURL(context.Background(), srv.Client(), srv.URL) + if err != nil { + t.Fatal(err) + } + if len(rows) < 2 { + t.Fatalf("want ≥2 rows, got %d", len(rows)) + } + if rows[0][0] != "Jméno" { + t.Errorf("unexpected header: %q", rows[0][0]) + } +} + +func TestFake(t *testing.T) { + adultRows := parseCSV(t, "testdata/adults_minimal.csv") + juniorRows := parseCSV(t, "testdata/juniors_minimal.csv") + + f := &Fake{Adults: adultRows, Juniors: juniorRows} + + got, err := f.FetchAdults(context.Background()) + if err != nil { + t.Fatal(err) + } + if got[0][0] != "Jméno" { + t.Errorf("adults header: %q", got[0][0]) + } + + got, err = f.FetchJuniors(context.Background()) + if err != nil { + t.Fatal(err) + } + if got[1][0] != "Junior One" { + t.Errorf("juniors first member: %q", got[1][0]) + } +} + +func parseCSV(t *testing.T, path string) [][]string { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + r := csv.NewReader(strings.NewReader(string(b))) + r.FieldsPerRecord = -1 + rows, err := r.ReadAll() + if err != nil { + t.Fatal(err) + } + return rows +} + +// fetchURL is a test helper that exercises the shared fetch logic against an arbitrary URL. +func fetchURL(ctx context.Context, hc *http.Client, url string) ([][]string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := hc.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + r := csv.NewReader(strings.NewReader(string(b))) + r.FieldsPerRecord = -1 + return r.ReadAll() +} diff --git a/go/internal/io/attendance/fake.go b/go/internal/io/attendance/fake.go new file mode 100644 index 0000000..7eb8882 --- /dev/null +++ b/go/internal/io/attendance/fake.go @@ -0,0 +1,12 @@ +package attendance + +import "context" + +// Fake is an in-memory replacement for Client, used in tests. +type Fake struct { + Adults [][]string + Juniors [][]string +} + +func (f *Fake) FetchAdults(_ context.Context) ([][]string, error) { return f.Adults, nil } +func (f *Fake) FetchJuniors(_ context.Context) ([][]string, error) { return f.Juniors, nil } diff --git a/go/internal/io/attendance/testdata/adults_minimal.csv b/go/internal/io/attendance/testdata/adults_minimal.csv new file mode 100644 index 0000000..baa06af --- /dev/null +++ b/go/internal/io/attendance/testdata/adults_minimal.csv @@ -0,0 +1,4 @@ +Jméno,Tier,,,01.09.2025,08.09.2025,15.09.2025 +Member One,A,,,TRUE,TRUE,FALSE +Member Two,A,,,TRUE,FALSE,FALSE +# last line,,,,, diff --git a/go/internal/io/attendance/testdata/juniors_minimal.csv b/go/internal/io/attendance/testdata/juniors_minimal.csv new file mode 100644 index 0000000..3a7d9c4 --- /dev/null +++ b/go/internal/io/attendance/testdata/juniors_minimal.csv @@ -0,0 +1,4 @@ +Jméno,Tier,,,01.09.2025,08.09.2025,15.09.2025 +Junior One,J,,,TRUE,TRUE,TRUE +# Trenéři,,,,, +Coach One,X,,,FALSE,FALSE,FALSE diff --git a/go/internal/io/cache/filecache.go b/go/internal/io/cache/filecache.go new file mode 100644 index 0000000..70b8ebb --- /dev/null +++ b/go/internal/io/cache/filecache.go @@ -0,0 +1,209 @@ +// Package cache implements a Drive-modifiedTime-gated JSON file cache, +// mirroring scripts/cache_utils.py. +package cache + +import ( + "context" + "encoding/json" + "fmt" + "fuj-management/go/internal/io/drive" + "os" + "path/filepath" + "sync" + "time" +) + +// DriveClient is the subset of drive.Client used by FileCache. +type DriveClient interface { + ModifiedTime(ctx context.Context, fileID string) (string, error) +} + +type cacheFile struct { + ModifiedTime string `json:"modifiedTime"` + Data json.RawMessage `json:"data"` + CachedAt string `json:"cachedAt"` +} + +// FileCache wraps a Drive client to gate JSON file caching on sheet modifiedTime. +// +// Two TTL knobs mirror scripts/cache_utils.py: +// - ttl: if the cache file on disk is younger than this, skip the Drive check entirely. +// - apiCheckTTL: debounces in-memory Drive API calls per sheet ID. +// +// Atomic writes: data is marshaled to a .tmp file then os.Rename'd. +// Cache files are compatible with Python's format: +// +// {"modifiedTime":"…","data":…,"cachedAt":"…"} +type FileCache struct { + drive DriveClient + dir string + sheetMap map[string]string // cache key → Drive file ID + ttl time.Duration + apiCheckTTL time.Duration + mu sync.Mutex + lastChecked map[string]time.Time +} + +// New creates a FileCache. +// sheetMap maps cache keys to Google Sheets/Drive file IDs (mirrors CACHE_SHEET_MAP in config). +func New(d DriveClient, dir string, sheetMap map[string]string, ttl, apiCheckTTL time.Duration) *FileCache { + return &FileCache{ + drive: d, + dir: dir, + sheetMap: sheetMap, + ttl: ttl, + apiCheckTTL: apiCheckTTL, + lastChecked: make(map[string]time.Time), + } +} + +// Get returns the cached value for cacheKey, calling fetch if the cache is stale. +// T must be JSON-marshalable. +func Get[T any](ctx context.Context, fc *FileCache, cacheKey string, fetch func(context.Context) (T, error)) (T, error) { + sheetID := fc.sheetMap[cacheKey] + if sheetID == "" { + sheetID = cacheKey + } + cacheFilePath := filepath.Join(fc.dir, cacheKey+"_cache.json") + + currentModTime, err := fc.currentModifiedTime(ctx, sheetID, cacheFilePath) + if err != nil { + return *new(T), fmt.Errorf("cache: modifiedTime for %s: %w", cacheKey, err) + } + + // Try cache hit + if data, ok := readCache[T](cacheFilePath, currentModTime); ok { + return data, nil + } + + // Cache miss — fetch fresh data + fresh, err := fetch(ctx) + if err != nil { + return *new(T), err + } + if err := writeCache(cacheFilePath, currentModTime, fresh); err != nil { + // Non-fatal: log but don't fail the request + _, _ = fmt.Fprintf(os.Stderr, "cache: write %s: %v\n", cacheKey, err) + } + return fresh, nil +} + +// Flush deletes all *_cache.json files in the cache dir and resets in-memory state. +func (fc *FileCache) Flush() (int, error) { + fc.mu.Lock() + fc.lastChecked = make(map[string]time.Time) + fc.mu.Unlock() + + pattern := filepath.Join(fc.dir, "*_cache.json") + matches, err := filepath.Glob(pattern) + if err != nil { + return 0, err + } + for _, f := range matches { + _ = os.Remove(f) + } + return len(matches), nil +} + +// currentModifiedTime returns a stable string representing the current version +// of the sheet, using the in-memory + file-mtime TTL guards before hitting Drive. +// On Drive failure, falls back to a 5-minute bucket string (matching Python). +func (fc *FileCache) currentModifiedTime(ctx context.Context, sheetID, cacheFilePath string) (string, error) { + now := time.Now() + + fc.mu.Lock() + lastCheck := fc.lastChecked[sheetID] + fc.mu.Unlock() + + // Guard 1: in-memory debounce — skip Drive if checked recently + if fc.apiCheckTTL > 0 && now.Sub(lastCheck) < fc.apiCheckTTL { + if mt, ok := readModifiedTime(cacheFilePath); ok { + return mt, nil + } + } + + // Guard 2: cache file is young enough — trust the stored modifiedTime + if fc.ttl > 0 { + if info, err := os.Stat(cacheFilePath); err == nil { + if now.Sub(info.ModTime()) < fc.ttl { + if mt, ok := readModifiedTime(cacheFilePath); ok { + fc.mu.Lock() + fc.lastChecked[sheetID] = now + fc.mu.Unlock() + return mt, nil + } + } + } + } + + // Hit Drive API + mt, err := fc.drive.ModifiedTime(ctx, sheetID) + if err != nil { + // Fallback: 5-minute bucket string, matches Python _fallback_ttl() + bucket := time.Now().Unix() / 300 + return fmt.Sprintf("ttl-5m-%d", bucket), nil + } + fc.mu.Lock() + fc.lastChecked[sheetID] = now + fc.mu.Unlock() + return mt, nil +} + +func readModifiedTime(path string) (string, bool) { + cf, ok := readCacheFile(path) + if !ok { + return "", false + } + return cf.ModifiedTime, cf.ModifiedTime != "" +} + +func readCacheFile(path string) (cacheFile, bool) { + b, err := os.ReadFile(path) + if err != nil { + return cacheFile{}, false + } + var cf cacheFile + if err := json.Unmarshal(b, &cf); err != nil { + return cacheFile{}, false + } + return cf, true +} + +func readCache[T any](path, currentModTime string) (T, bool) { + cf, ok := readCacheFile(path) + if !ok || cf.ModifiedTime != currentModTime { + return *new(T), false + } + var v T + if err := json.Unmarshal(cf.Data, &v); err != nil { + return *new(T), false + } + return v, true +} + +func writeCache(path, modTime string, data any) error { + raw, err := json.Marshal(data) + if err != nil { + return err + } + cf := cacheFile{ + ModifiedTime: modTime, + Data: json.RawMessage(raw), + CachedAt: time.Now().Format(time.RFC3339), + } + b, err := json.Marshal(cf) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + tmp := path + ".tmp" + if err := os.WriteFile(tmp, b, 0o600); err != nil { + return err + } + return os.Rename(tmp, path) +} + +// Ensure *drive.Client satisfies DriveClient at compile time. +var _ DriveClient = (*drive.Client)(nil) diff --git a/go/internal/io/cache/filecache_test.go b/go/internal/io/cache/filecache_test.go new file mode 100644 index 0000000..250e63c --- /dev/null +++ b/go/internal/io/cache/filecache_test.go @@ -0,0 +1,125 @@ +package cache + +import ( + "context" + "errors" + "fuj-management/go/internal/io/drive" + "os" + "testing" + "time" +) + +func TestGet_FreshFetch(t *testing.T) { + dir := t.TempDir() + d := &drive.Fake{Times: map[string]string{"sheet1": "2026-01-01T00:00:00Z"}} + fc := New(d, dir, map[string]string{"mykey": "sheet1"}, time.Minute, time.Minute) + + calls := 0 + got, err := Get(context.Background(), fc, "mykey", func(_ context.Context) ([]string, error) { + calls++ + return []string{"a", "b"}, nil + }) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || got[0] != "a" { + t.Errorf("unexpected: %v", got) + } + if calls != 1 { + t.Errorf("want 1 fetch call, got %d", calls) + } +} + +func TestGet_CacheHit(t *testing.T) { + dir := t.TempDir() + d := &drive.Fake{Times: map[string]string{"sheet1": "2026-01-01T00:00:00Z"}} + fc := New(d, dir, map[string]string{"mykey": "sheet1"}, time.Minute, time.Minute) + + fetch := func(_ context.Context) ([]string, error) { return []string{"a"}, nil } + if _, err := Get(context.Background(), fc, "mykey", fetch); err != nil { + t.Fatal(err) + } + + // Second call — modifiedTime unchanged, should hit cache + calls := 0 + got, err := Get(context.Background(), fc, "mykey", func(_ context.Context) ([]string, error) { + calls++ + return []string{"SHOULD_NOT_CALL"}, nil + }) + if err != nil { + t.Fatal(err) + } + if got[0] != "a" { + t.Errorf("want cache hit with 'a', got %q", got[0]) + } + if calls != 0 { + t.Errorf("want 0 fetch calls on hit, got %d", calls) + } +} + +func TestGet_CacheMiss_OnModifiedTimeChange(t *testing.T) { + dir := t.TempDir() + d := &drive.Fake{Times: map[string]string{"sheet1": "2026-01-01T00:00:00Z"}} + // No TTL guards so we always hit Drive + fc := New(d, dir, map[string]string{"mykey": "sheet1"}, 0, 0) + + fetch := func(_ context.Context) ([]string, error) { return []string{"v1"}, nil } + if _, err := Get(context.Background(), fc, "mykey", fetch); err != nil { + t.Fatal(err) + } + + // Sheet updated — change modifiedTime + d.Times["sheet1"] = "2026-02-01T00:00:00Z" + got, err := Get(context.Background(), fc, "mykey", func(_ context.Context) ([]string, error) { + return []string{"v2"}, nil + }) + if err != nil { + t.Fatal(err) + } + if got[0] != "v2" { + t.Errorf("want v2 after sheet update, got %q", got[0]) + } +} + +func TestGet_DriveFailureFallback(t *testing.T) { + dir := t.TempDir() + d := &drive.Fake{Err: errors.New("drive down")} + fc := New(d, dir, nil, 0, 0) + + calls := 0 + _, err := Get(context.Background(), fc, "mykey", func(_ context.Context) ([]string, error) { + calls++ + return []string{"fallback"}, nil + }) + if err != nil { + t.Fatal(err) + } + if calls != 1 { + t.Errorf("want 1 fetch call, got %d", calls) + } +} + +func TestFlush(t *testing.T) { + dir := t.TempDir() + d := &drive.Fake{Times: map[string]string{"sheet1": "t1"}} + fc := New(d, dir, map[string]string{"k": "sheet1"}, 0, 0) + + if _, err := Get(context.Background(), fc, "k", func(_ context.Context) (int, error) { return 42, nil }); err != nil { + t.Fatal(err) + } + + n, err := fc.Flush() + if err != nil { + t.Fatal(err) + } + if n != 1 { + t.Errorf("want 1 deleted file, got %d", n) + } + // Cache dir should be empty of _cache.json files + entries, _ := os.ReadDir(dir) + for _, e := range entries { + if e.Name() != "" { + t.Errorf("expected empty dir after flush, found %s", e.Name()) + } + } +} diff --git a/go/internal/io/drive/client.go b/go/internal/io/drive/client.go new file mode 100644 index 0000000..fa03835 --- /dev/null +++ b/go/internal/io/drive/client.go @@ -0,0 +1,46 @@ +// Package drive provides a thin wrapper around the Google Drive v3 API, +// used only to read modifiedTime for cache invalidation. +package drive + +import ( + "context" + "net/http" + "time" + + "google.golang.org/api/drive/v3" + "google.golang.org/api/option" +) + +// Client wraps the Drive v3 API, scoped to read-only modifiedTime checks. +type Client struct { + svc *drive.Service +} + +// New builds a Client using a service-account credentials file. +// timeout applies to each Drive API call. +func New(ctx context.Context, credentialsPath string, timeout time.Duration) (*Client, error) { + hc := &http.Client{Timeout: timeout} + svc, err := drive.NewService(ctx, + option.WithCredentialsFile(credentialsPath), //nolint:staticcheck + option.WithScopes(drive.DriveReadonlyScope), + option.WithHTTPClient(hc), + ) + if err != nil { + return nil, err + } + return &Client{svc: svc}, nil +} + +// ModifiedTime returns the RFC3339 modifiedTime for the given Drive file ID. +// Returns ("", err) if the Drive API call fails. +func (c *Client) ModifiedTime(ctx context.Context, fileID string) (string, error) { + meta, err := c.svc.Files.Get(fileID). + Fields("modifiedTime"). + SupportsAllDrives(true). + Context(ctx). + Do() + if err != nil { + return "", err + } + return meta.ModifiedTime, nil +} diff --git a/go/internal/io/drive/fake.go b/go/internal/io/drive/fake.go new file mode 100644 index 0000000..2d06f37 --- /dev/null +++ b/go/internal/io/drive/fake.go @@ -0,0 +1,18 @@ +package drive + +import "context" + +// Fake is an in-memory replacement for Client used in tests. +type Fake struct { + // Times maps file ID → modifiedTime string returned by ModifiedTime. + Times map[string]string + // Err, if non-nil, is returned instead of looking up Times. + Err error +} + +func (f *Fake) ModifiedTime(_ context.Context, fileID string) (string, error) { + if f.Err != nil { + return "", f.Err + } + return f.Times[fileID], nil +} diff --git a/go/internal/io/fio/api.go b/go/internal/io/fio/api.go new file mode 100644 index 0000000..26a8172 --- /dev/null +++ b/go/internal/io/fio/api.go @@ -0,0 +1,128 @@ +package fio + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" +) + +// httpDoer is the subset of *http.Client used by both Fio impls. +type httpDoer interface { + Do(*http.Request) (*http.Response, error) +} + +// apiClient fetches transactions from the Fio REST API (JSON). +// Ports scripts/fio_utils.py fetch_transactions_api. +type apiClient struct { + token string + hc httpDoer +} + +func (c *apiClient) FetchTransactions(ctx context.Context, from, to time.Time) ([]Transaction, error) { + const layout = "2006-01-02" + url := fmt.Sprintf("https://fioapi.fio.cz/v1/rest/periods/%s/%s/%s/transactions.json", + c.token, from.Format(layout), to.Format(layout)) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := c.hc.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("fio api: HTTP %d", resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return parseAPIResponse(body) +} + +// fioAPIResponse is the top-level envelope from the Fio JSON API. +type fioAPIResponse struct { + AccountStatement struct { + TransactionList struct { + Transaction []map[string]json.RawMessage `json:"transaction"` + } `json:"transactionList"` + } `json:"accountStatement"` +} + +func parseAPIResponse(body []byte) ([]Transaction, error) { + var resp fioAPIResponse + if err := json.Unmarshal(body, &resp); err != nil { + return nil, fmt.Errorf("fio api: parse JSON: %w", err) + } + + var txns []Transaction + for _, raw := range resp.AccountStatement.TransactionList.Transaction { + amount := colFloat(raw, "column1") + if amount <= 0 { + continue // skip outgoing + } + dateRaw := colString(raw, "column0") + dateStr := "" + if len(dateRaw) >= 10 { + dateStr = dateRaw[:10] + } + txns = append(txns, Transaction{ + Date: dateStr, + Amount: amount, + Sender: colString(raw, "column10"), + Message: colString(raw, "column16"), + VS: colString(raw, "column5"), + KS: colString(raw, "column4"), + SS: colString(raw, "column6"), + UserID: colString(raw, "column7"), + SenderAccount: colString(raw, "column2"), + BankID: colString(raw, "column22"), + Currency: colStringOr(raw, "column14", "CZK"), + }) + } + return txns, nil +} + +// colString extracts {"value":…} as a string from a column map. +func colString(m map[string]json.RawMessage, col string) string { + raw, ok := m[col] + if !ok { + return "" + } + var cell struct { + Value *string `json:"value"` + } + if json.Unmarshal(raw, &cell) != nil || cell.Value == nil { + return "" + } + return *cell.Value +} + +// colStringOr is colString with a fallback value. +func colStringOr(m map[string]json.RawMessage, col, fallback string) string { + if v := colString(m, col); v != "" { + return v + } + return fallback +} + +// colFloat extracts {"value":…} as a float64 from a column map. +// Returns 0 on any error (null column, non-numeric value). +func colFloat(m map[string]json.RawMessage, col string) float64 { + raw, ok := m[col] + if !ok { + return 0 + } + var cell struct { + Value *float64 `json:"value"` + } + if json.Unmarshal(raw, &cell) != nil || cell.Value == nil { + return 0 + } + return *cell.Value +} diff --git a/go/internal/io/fio/client.go b/go/internal/io/fio/client.go new file mode 100644 index 0000000..4ff3ae2 --- /dev/null +++ b/go/internal/io/fio/client.go @@ -0,0 +1,37 @@ +// Package fio fetches Fio bank transactions via the JSON API or the +// transparent-page HTML scraper, behind a common Client interface. +package fio + +import ( + "context" + "time" +) + +// Transaction is one incoming bank payment. Fields absent from the HTML scraper +// (BankID, Currency, UserID, SenderAccount) are empty strings on that path. +type Transaction struct { + Date string + Amount float64 + Sender string + Message string + VS string + KS string + SS string + UserID string // column7; empty on HTML path + SenderAccount string // column2; empty on HTML path + BankID string // column22; empty on HTML path + Currency string // column14; empty on HTML path (assume CZK) +} + +// Client fetches transactions for a date window. +type Client interface { + FetchTransactions(ctx context.Context, from, to time.Time) ([]Transaction, error) +} + +// New returns an apiClient when token is non-empty, otherwise a transparentClient. +func New(token, accountNum string, hc httpDoer) Client { + if token != "" { + return &apiClient{token: token, hc: hc} + } + return &transparentClient{accountNum: accountNum, hc: hc} +} diff --git a/go/internal/io/fio/fake.go b/go/internal/io/fio/fake.go new file mode 100644 index 0000000..c727ffc --- /dev/null +++ b/go/internal/io/fio/fake.go @@ -0,0 +1,19 @@ +package fio + +import ( + "context" + "time" +) + +// Fake is an in-memory replacement for Client, used in tests. +type Fake struct { + Transactions []Transaction + Err error +} + +func (f *Fake) FetchTransactions(_ context.Context, _, _ time.Time) ([]Transaction, error) { + if f.Err != nil { + return nil, f.Err + } + return f.Transactions, nil +} diff --git a/go/internal/io/fio/fio_test.go b/go/internal/io/fio/fio_test.go new file mode 100644 index 0000000..03774ef --- /dev/null +++ b/go/internal/io/fio/fio_test.go @@ -0,0 +1,157 @@ +package fio + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" +) + +func TestAPIClient_ParseResponse(t *testing.T) { + body, err := os.ReadFile("testdata/api_response.json") + if err != nil { + t.Fatal(err) + } + txns, err := parseAPIResponse(body) + if err != nil { + t.Fatal(err) + } + if len(txns) != 1 { + t.Fatalf("want 1 txn (outgoing filtered), got %d", len(txns)) + } + tx := txns[0] + if tx.Date != "2026-04-10" { + t.Errorf("date: want '2026-04-10', got %q", tx.Date) + } + if tx.Amount != 750 { + t.Errorf("amount: want 750, got %v", tx.Amount) + } + if tx.Sender != "Jana Novakova" { + t.Errorf("sender: want 'Jana Novakova', got %q", tx.Sender) + } + if tx.Message != "duben 2026" { + t.Errorf("message: want 'duben 2026', got %q", tx.Message) + } + if tx.VS != "123" { + t.Errorf("vs: want '123', got %q", tx.VS) + } + if tx.BankID != "12345678901" { + t.Errorf("bank_id: want '12345678901', got %q", tx.BankID) + } +} + +func TestAPIClient_HTTPRoundTrip(t *testing.T) { + body, _ := os.ReadFile("testdata/api_response.json") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write(body) + })) + defer srv.Close() + + c := &apiClient{token: "TESTTOKEN", hc: &overrideClient{base: srv.Client(), baseURL: srv.URL}} + txns, err := c.FetchTransactions(context.Background(), time.Now().AddDate(0, -1, 0), time.Now()) + if err != nil { + t.Fatal(err) + } + if len(txns) != 1 { + t.Fatalf("want 1 txn, got %d", len(txns)) + } +} + +func TestTransparentClient_ParseHTML(t *testing.T) { + body, err := os.ReadFile("testdata/transparent.html") + if err != nil { + t.Fatal(err) + } + txns, err := parseTransparentHTML(body) + if err != nil { + t.Fatal(err) + } + // Only the incoming row (750 CZK) should be kept; -200 is outgoing + if len(txns) != 1 { + t.Fatalf("want 1 txn (outgoing filtered), got %d", len(txns)) + } + tx := txns[0] + if tx.Date != "2026-04-10" { + t.Errorf("date: want '2026-04-10', got %q", tx.Date) + } + if tx.Amount != 750 { + t.Errorf("amount: want 750, got %v", tx.Amount) + } + if tx.Sender != "Jana Novakova" { + t.Errorf("sender: want 'Jana Novakova', got %q", tx.Sender) + } + if tx.VS != "123" { + t.Errorf("vs: want '123', got %q", tx.VS) + } + if tx.BankID != "" { + t.Errorf("bank_id: want empty on HTML path, got %q", tx.BankID) + } +} + +func TestParseCzechDate(t *testing.T) { + cases := []struct{ in, want string }{ + {"10.04.2026", "2026-04-10"}, + {"10/04/2026", "2026-04-10"}, + {"", ""}, + {"invalid", ""}, + } + for _, c := range cases { + if got := parseCzechDate(c.in); got != c.want { + t.Errorf("parseCzechDate(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +func TestParseCzechAmount(t *testing.T) { + cases := []struct { + in string + want float64 + }{ + {"750,00 CZK", 750}, + {"1.500,00", 1500}, + {"1500.00", 1500}, + {"-200,00 CZK", -200}, + } + for _, c := range cases { + if got := parseCzechAmount(c.in); got != c.want { + t.Errorf("parseCzechAmount(%q) = %v, want %v", c.in, got, c.want) + } + } +} + +func TestFake(t *testing.T) { + f := &Fake{Transactions: []Transaction{{Date: "2026-04-01", Amount: 500}}} + txns, err := f.FetchTransactions(context.Background(), time.Now(), time.Now()) + if err != nil { + t.Fatal(err) + } + if len(txns) != 1 || txns[0].Date != "2026-04-01" { + t.Errorf("unexpected: %v", txns) + } +} + +// overrideClient replaces the URL in requests so we can hit a local test server +// instead of the real Fio URL. +type overrideClient struct { + base *http.Client + baseURL string +} + +func (o *overrideClient) Do(req *http.Request) (*http.Response, error) { + r2, _ := http.NewRequestWithContext(req.Context(), req.Method, o.baseURL+req.URL.Path, nil) + resp, err := o.base.Do(r2) + if err != nil { + return nil, err + } + // The api client reads the body, so re-serve whatever the test server returned. + return resp, nil +} + +// verify Fake satisfies Client +var _ Client = (*Fake)(nil) + +// ensure io.ReadAll isn't called at top level (compile-time reference suppressor) +var _ = io.ReadAll diff --git a/go/internal/io/fio/testdata/api_response.json b/go/internal/io/fio/testdata/api_response.json new file mode 100644 index 0000000..7e3e67a --- /dev/null +++ b/go/internal/io/fio/testdata/api_response.json @@ -0,0 +1,29 @@ +{ + "accountStatement": { + "transactionList": { + "transaction": [ + { + "column0": {"value": "2026-04-10+0200", "name": "Datum", "id": 0}, + "column1": {"value": 750.0, "name": "Objem", "id": 1}, + "column2": {"value": "123456789/0300", "name": "Protiúčet", "id": 2}, + "column4": {"value": "0308", "name": "KS", "id": 4}, + "column5": {"value": "123", "name": "VS", "id": 5}, + "column6": {"value": "", "name": "SS", "id": 6}, + "column7": {"value": "Jana Nováková", "name": "Uživatelská identifikace", "id": 7}, + "column10": {"value": "Jana Novakova", "name": "Název protiúčtu", "id": 10}, + "column14": {"value": "CZK", "name": "Měna", "id": 14}, + "column16": {"value": "duben 2026", "name": "Zpráva pro příjemce", "id": 16}, + "column22": {"value": "12345678901", "name": "ID operace", "id": 22} + }, + { + "column0": {"value": "2026-04-11+0200", "name": "Datum", "id": 0}, + "column1": {"value": -200.0, "name": "Objem", "id": 1}, + "column10": {"value": "Outgoing", "name": "Název protiúčtu", "id": 10}, + "column14": {"value": "CZK", "name": "Měna", "id": 14}, + "column16": {"value": "", "name": "Zpráva pro příjemce", "id": 16}, + "column22": {"value": "99999999999", "name": "ID operace", "id": 22} + } + ] + } + } +} diff --git a/go/internal/io/fio/testdata/transparent.html b/go/internal/io/fio/testdata/transparent.html new file mode 100644 index 0000000..fbc8915 --- /dev/null +++ b/go/internal/io/fio/testdata/transparent.html @@ -0,0 +1,37 @@ + + + + +
ignored
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
DatumČástkaTypNázev protiúčtuZprávaKSVSSSPoznámka
10.04.2026750,00 CZKPříjemJana Novakovaduben 20260308123
09.04.2026-200,00 CZKOdchozíSomeoneoutgoing
+ + diff --git a/go/internal/io/fio/transparent.go b/go/internal/io/fio/transparent.go new file mode 100644 index 0000000..9c74023 --- /dev/null +++ b/go/internal/io/fio/transparent.go @@ -0,0 +1,217 @@ +package fio + +import ( + "context" + "fmt" + "io" + "net/http" + "regexp" + "strings" + "time" + "unicode" + + ghtml "golang.org/x/net/html" +) + +// transparentClient fetches transactions from the Fio transparent account page (HTML). +// Ports scripts/fio_utils.py FioTableParser + fetch_transactions_transparent. +type transparentClient struct { + accountNum string + hc httpDoer +} + +func (c *transparentClient) FetchTransactions(ctx context.Context, from, to time.Time) ([]Transaction, error) { + // Transparent page date format: D.M.YYYY + url := fmt.Sprintf( + "https://ib.fio.cz/ib/transparent?a=%s&f=%s&t=%s", + c.accountNum, + from.Format("2.1.2006"), + to.Format("2.1.2006"), + ) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := c.hc.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("fio transparent: HTTP %d", resp.StatusCode) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return parseTransparentHTML(body) +} + +// Column indices in the transparent-page table (0-based). +// Datum | Částka | Typ | Název protiúčtu | Zpráva pro příjemce | KS | VS | SS | Poznámka +const ( + tColDate = 0 + tColAmount = 1 + tColSender = 3 + tColMessage = 4 + tColKS = 5 + tColVS = 6 + tColSS = 7 +) + +func parseTransparentHTML(body []byte) ([]Transaction, error) { + rows := extractSecondTableRows(body) + + var txns []Transaction + for _, row := range rows { + col := func(i int) string { + if i < len(row) { + return strings.TrimSpace(row[i]) + } + return "" + } + dateStr := parseCzechDate(col(tColDate)) + amount := parseCzechAmount(col(tColAmount)) + if dateStr == "" || amount <= 0 { + continue + } + txns = append(txns, Transaction{ + Date: dateStr, + Amount: amount, + Sender: col(tColSender), + Message: col(tColMessage), + KS: col(tColKS), + VS: col(tColVS), + SS: col(tColSS), + BankID: "", // not available on HTML path + }) + } + return txns, nil +} + +// extractSecondTableRows walks the HTML token stream and returns data rows +// from the second element, skipping the . +func extractSecondTableRows(body []byte) [][]string { + z := ghtml.NewTokenizer(strings.NewReader(string(body))) + + tableCount := 0 + inTarget := false + inThead := false + inRow := false + inCell := false + var currentRow []string + var cellBuf strings.Builder + var rows [][]string + + for { + tt := z.Next() + if tt == ghtml.ErrorToken { + break + } + switch tt { + case ghtml.StartTagToken: + t := z.Token() + switch t.Data { + case "table": + if hasClass(t, "table") { + tableCount++ + if tableCount == 2 { + inTarget = true + } + } + case "thead": + if inTarget { + inThead = true + } + case "tr": + if inTarget && !inThead { + inRow = true + currentRow = nil + } + case "td", "th": + if inRow { + inCell = true + cellBuf.Reset() + } + } + case ghtml.EndTagToken: + t := z.Token() + switch t.Data { + case "td", "th": + if inCell { + currentRow = append(currentRow, cellBuf.String()) + inCell = false + } + case "thead": + inThead = false + case "tr": + if inRow { + if len(currentRow) > 0 { + rows = append(rows, currentRow) + } + inRow = false + } + case "table": + if inTarget { + return rows + } + } + case ghtml.TextToken: + if inCell { + cellBuf.WriteString(z.Token().Data) + } + } + } + return rows +} + +func hasClass(t ghtml.Token, cls string) bool { + for _, a := range t.Attr { + if a.Key == "class" { + for _, c := range strings.Fields(a.Val) { + if c == cls { + return true + } + } + } + } + return false +} + +// parseCzechDate parses "DD.MM.YYYY" or "DD/MM/YYYY" → "YYYY-MM-DD". +// Returns "" on parse error. +func parseCzechDate(s string) string { + s = strings.TrimSpace(s) + for _, layout := range []string{"02.01.2006", "02/01/2006"} { + if t, err := time.Parse(layout, s); err == nil { + return t.Format("2006-01-02") + } + } + return "" +} + +var nonNumericRe = regexp.MustCompile(`[^\d.,]`) + +// parseCzechAmount parses "1 500,00 CZK" / "1.500,00" / "1500.00" → float64. +// Returns 0 on error. +func parseCzechAmount(s string) float64 { + // Remove NBSP, regular spaces, currency letters + s = strings.Map(func(r rune) rune { + if r == ' ' || unicode.IsSpace(r) || unicode.IsLetter(r) { + return -1 + } + return r + }, s) + + if strings.Contains(s, ",") { + // Czech decimal: 1.500,00 → remove dots (thousand sep), comma → dot + s = strings.ReplaceAll(s, ".", "") + s = strings.ReplaceAll(s, ",", ".") + } else { + // Remove any remaining non-numeric except one dot + s = nonNumericRe.ReplaceAllString(s, "") + } + var f float64 + _, _ = fmt.Sscanf(s, "%f", &f) + return f +} diff --git a/go/internal/io/sheets/client.go b/go/internal/io/sheets/client.go new file mode 100644 index 0000000..1cf70be --- /dev/null +++ b/go/internal/io/sheets/client.go @@ -0,0 +1,124 @@ +// Package sheets provides a typed wrapper around the Google Sheets v4 API. +package sheets + +import ( + "context" + "fmt" + "time" + + "google.golang.org/api/option" + sheetsv4 "google.golang.org/api/sheets/v4" +) + +// ValueRange pairs an R1C1 range with its cell values, used for batchUpdate. +type ValueRange struct { + Range string // R1C1 notation, e.g. "R2C4:R2C6" + Values [][]any // one sub-slice per row +} + +// Client wraps the Sheets v4 API with the operations needed by this project. +type Client struct { + svc *sheetsv4.Service +} + +// New builds a Client using a service-account credentials file. +func New(ctx context.Context, credentialsPath string, _ time.Duration) (*Client, error) { + svc, err := sheetsv4.NewService(ctx, + option.WithCredentialsFile(credentialsPath), //nolint:staticcheck + option.WithScopes(sheetsv4.SpreadsheetsScope), + ) + if err != nil { + return nil, err + } + return &Client{svc: svc}, nil +} + +// GetValues fetches a range from a spreadsheet with UNFORMATTED_VALUE rendering +// (numbers as numbers, dates as serial floats — matching Python's behaviour). +func (c *Client) GetValues(ctx context.Context, spreadsheetID, a1Range string) ([][]any, error) { + resp, err := c.svc.Spreadsheets.Values. + Get(spreadsheetID, a1Range). + ValueRenderOption("UNFORMATTED_VALUE"). + Context(ctx). + Do() + if err != nil { + return nil, err + } + rows := make([][]any, len(resp.Values)) + copy(rows, resp.Values) + return rows, nil +} + +// AppendValues appends rows to the first empty row after a1Range. +func (c *Client) AppendValues(ctx context.Context, spreadsheetID, a1Range string, rows [][]any) error { + vals := make([][]any, len(rows)) + copy(vals, rows) + _, err := c.svc.Spreadsheets.Values. + Append(spreadsheetID, a1Range, &sheetsv4.ValueRange{Values: vals}). + ValueInputOption("USER_ENTERED"). + Context(ctx). + Do() + return err +} + +// BatchUpdateValues writes multiple non-contiguous ranges in one API call. +func (c *Client) BatchUpdateValues(ctx context.Context, spreadsheetID string, updates []ValueRange) error { + data := make([]*sheetsv4.ValueRange, len(updates)) + for i, u := range updates { + vals := make([][]any, len(u.Values)) + copy(vals, u.Values) + data[i] = &sheetsv4.ValueRange{Range: u.Range, Values: vals} + } + _, err := c.svc.Spreadsheets.Values. + BatchUpdate(spreadsheetID, &sheetsv4.BatchUpdateValuesRequest{ + ValueInputOption: "USER_ENTERED", + Data: data, + }). + Context(ctx). + Do() + return err +} + +// WriteHeader overwrites row 1 of the spreadsheet with the given labels. +func (c *Client) WriteHeader(ctx context.Context, spreadsheetID string, labels []string) error { + row := make([]any, len(labels)) + for i, l := range labels { + row[i] = l + } + _, err := c.svc.Spreadsheets.Values. + Update(spreadsheetID, "A1", &sheetsv4.ValueRange{Values: [][]any{row}}). + ValueInputOption("USER_ENTERED"). + Context(ctx). + Do() + return err +} + +// SortByDateColumn sorts rows 2..10000 of the first sheet ascending by column A (Date). +// Looks up the sheetId (gid) from spreadsheet metadata. +func (c *Client) SortByDateColumn(ctx context.Context, spreadsheetID string) error { + meta, err := c.svc.Spreadsheets.Get(spreadsheetID).Context(ctx).Do() + if err != nil { + return fmt.Errorf("sheets: get spreadsheet: %w", err) + } + if len(meta.Sheets) == 0 { + return fmt.Errorf("sheets: spreadsheet has no sheets") + } + sheetID := meta.Sheets[0].Properties.SheetId + + _, err = c.svc.Spreadsheets.BatchUpdate(spreadsheetID, &sheetsv4.BatchUpdateSpreadsheetRequest{ + Requests: []*sheetsv4.Request{{ + SortRange: &sheetsv4.SortRangeRequest{ + Range: &sheetsv4.GridRange{ + SheetId: sheetID, + StartRowIndex: 1, + EndRowIndex: 10000, + }, + SortSpecs: []*sheetsv4.SortSpec{{ + DimensionIndex: 0, + SortOrder: "ASCENDING", + }}, + }, + }}, + }).Context(ctx).Do() + return err +} diff --git a/go/internal/io/sheets/fake.go b/go/internal/io/sheets/fake.go new file mode 100644 index 0000000..d5e6b8e --- /dev/null +++ b/go/internal/io/sheets/fake.go @@ -0,0 +1,53 @@ +package sheets + +import ( + "context" + "fmt" +) + +// Fake is an in-memory replacement for Client used in tests. +// Values maps a "/" key to pre-seeded rows. +type Fake struct { + // Values maps "spreadsheetID/range" → rows returned by GetValues. + Values map[string][][]any + // Appended collects rows passed to AppendValues for assertion. + Appended []AppendCall + // BatchUpdated collects calls to BatchUpdateValues. + BatchUpdated []BatchCall +} + +// AppendCall records one AppendValues invocation. +type AppendCall struct { + SpreadsheetID string + Range string + Rows [][]any +} + +// BatchCall records one BatchUpdateValues invocation. +type BatchCall struct { + SpreadsheetID string + Updates []ValueRange +} + +func (f *Fake) GetValues(_ context.Context, spreadsheetID, a1Range string) ([][]any, error) { + key := spreadsheetID + "/" + a1Range + rows, ok := f.Values[key] + if !ok { + return nil, fmt.Errorf("sheets fake: no seed for %q", key) + } + return rows, nil +} + +func (f *Fake) AppendValues(_ context.Context, spreadsheetID, a1Range string, rows [][]any) error { + f.Appended = append(f.Appended, AppendCall{SpreadsheetID: spreadsheetID, Range: a1Range, Rows: rows}) + return nil +} + +func (f *Fake) BatchUpdateValues(_ context.Context, spreadsheetID string, updates []ValueRange) error { + f.BatchUpdated = append(f.BatchUpdated, BatchCall{SpreadsheetID: spreadsheetID, Updates: updates}) + return nil +} + +func (f *Fake) WriteHeader(_ context.Context, _ string, _ []string) error { return nil } + +func (f *Fake) SortByDateColumn(_ context.Context, _ string) error { return nil } diff --git a/go/internal/services/banksync/infer.go b/go/internal/services/banksync/infer.go new file mode 100644 index 0000000..82f431a --- /dev/null +++ b/go/internal/services/banksync/infer.go @@ -0,0 +1,170 @@ +package banksync + +import ( + "context" + "fmt" + "fuj-management/go/internal/domain/matching" + "fuj-management/go/internal/domain/reconcile" + "fuj-management/go/internal/io/sheets" + "strings" + "time" +) + +// InferOpts controls infer behaviour. +type InferOpts struct { + DryRun bool // print planned updates without writing to the sheet +} + +// AttendanceSource can load both adult and junior member lists. +type AttendanceSource interface { + LoadAdults(ctx context.Context) ([]reconcile.Member, []string, error) + LoadJuniors(ctx context.Context) ([]reconcile.Member, []string, error) +} + +// sheetReadWriter is the subset of *sheets.Client used by InferPayments. +type sheetReadWriter interface { + GetValues(ctx context.Context, spreadsheetID, a1Range string) ([][]any, error) + BatchUpdateValues(ctx context.Context, spreadsheetID string, updates []sheets.ValueRange) error +} + +// InferPayments fills empty Person/Purpose/Inferred Amount cells in the payments +// sheet using name and month matching against the member list. +// Returns the number of rows updated (or that would be updated on dry-run). +// Ports scripts/infer_payments.py infer_payments. +func InferPayments( + ctx context.Context, + spreadsheetID string, + sh sheetReadWriter, + attendance AttendanceSource, + opts InferOpts, +) (int, error) { + rows, err := sh.GetValues(ctx, spreadsheetID, "A1:Z") + if err != nil { + return 0, fmt.Errorf("infer: read sheet: %w", err) + } + if len(rows) == 0 { + return 0, nil + } + + header := rows[0] + colIdx := func(label string) int { + label = strings.ToLower(strings.TrimSpace(label)) + for i, h := range header { + if strings.ToLower(strings.TrimSpace(fmt.Sprint(h))) == label { + return i + } + } + return -1 + } + + idxDate := colIdx("date") + idxAmount := colIdx("amount") + idxSender := colIdx("sender") + idxMessage := colIdx("message") + idxVS := colIdx("vs") + idxManual := colIdx("manual fix") + idxPerson := colIdx("person") + idxPurpose := colIdx("purpose") + idxInferred := colIdx("inferred amount") + + for _, req := range []string{"person", "purpose", "inferred amount"} { + if colIdx(req) == -1 { + return 0, fmt.Errorf("infer: required column %q not found in sheet", req) + } + } + + // Build union member list: adults + juniors, deduped by canonical key. + adults, _, err := attendance.LoadAdults(ctx) + if err != nil { + return 0, fmt.Errorf("infer: load adults: %w", err) + } + juniors, _, err := attendance.LoadJuniors(ctx) + if err != nil { + return 0, fmt.Errorf("infer: load juniors: %w", err) + } + memberNames := dedupeMembers(append(adults, juniors...)) + + defaultYear := time.Now().Year() + + var updates []sheets.ValueRange + for i, row := range rows[1:] { + rowNum := i + 2 // 1-based, skip header + + get := func(idx int) string { + if idx < 0 || idx >= len(row) { + return "" + } + return strings.TrimSpace(fmt.Sprint(row[idx])) + } + + // Skip rule: any of manual fix / Person / Purpose non-empty → leave alone + if get(idxManual) != "" || get(idxPerson) != "" || get(idxPurpose) != "" { + continue + } + + tx := matching.Transaction{ + Sender: get(idxSender), + Message: get(idxMessage), + UserID: get(idxVS), + } + if idxDate >= 0 && idxDate < len(row) { + tx.Date = row[idxDate] + } + + inferred := matching.InferTransactionDetails(tx, memberNames, defaultYear) + if len(inferred.Members) == 0 && len(inferred.Months) == 0 { + continue + } + + var peeps []string + for _, m := range inferred.Members { + if m.Confidence == matching.ConfidenceReview { + peeps = append(peeps, "[?] "+m.Name) + } else { + peeps = append(peeps, m.Name) + } + } + personVal := strings.Join(peeps, ", ") + purposeVal := strings.Join(inferred.Months, ", ") + + amountVal := "" + if idxAmount >= 0 && idxAmount < len(row) { + amountVal = fmt.Sprint(row[idxAmount]) + } + + if opts.DryRun { + fmt.Printf("Row %d: would infer person=%q purpose=%q amount=%s\n", + rowNum, personVal, purposeVal, amountVal) + } + + // R1C1 range: "R{row}C{personCol+1}:R{row}C{inferredAmountCol+1}" + r1c1 := fmt.Sprintf("R%dC%d:R%dC%d", rowNum, idxPerson+1, rowNum, idxInferred+1) + updates = append(updates, sheets.ValueRange{ + Range: r1c1, + Values: [][]any{{personVal, purposeVal, amountVal}}, + }) + } + + if len(updates) == 0 || opts.DryRun { + return len(updates), nil + } + + if err := sh.BatchUpdateValues(ctx, spreadsheetID, updates); err != nil { + return 0, fmt.Errorf("infer: batch update: %w", err) + } + return len(updates), nil +} + +// dedupeMembers returns unique member names, deduped by canonical key. +func dedupeMembers(members []reconcile.Member) []string { + seen := make(map[string]bool, len(members)) + var names []string + for _, m := range members { + key := strings.Join(strings.Fields(m.Name), " ") + if !seen[key] { + seen[key] = true + names = append(names, m.Name) + } + } + return names +} diff --git a/go/internal/services/banksync/infer_test.go b/go/internal/services/banksync/infer_test.go new file mode 100644 index 0000000..6a348ea --- /dev/null +++ b/go/internal/services/banksync/infer_test.go @@ -0,0 +1,157 @@ +package banksync + +import ( + "context" + "fuj-management/go/internal/domain/reconcile" + "fuj-management/go/internal/io/sheets" + "testing" +) + +type fakeAttendance struct { + adults, juniors []reconcile.Member +} + +func (f *fakeAttendance) LoadAdults(_ context.Context) ([]reconcile.Member, []string, error) { + return f.adults, nil, nil +} + +func (f *fakeAttendance) LoadJuniors(_ context.Context) ([]reconcile.Member, []string, error) { + return f.juniors, nil, nil +} + +var paymentsHeader = []any{ + "Date", "Amount", "manual fix", "Person", "Purpose", "Inferred Amount", + "Sender", "VS", "Message", "Bank ID", "Sync ID", +} + +func TestInferPayments_BasicMatch(t *testing.T) { + sh := &sheets.Fake{Values: map[string][][]any{ + "SHEETID/A1:Z": { + paymentsHeader, + // Row with no Person/Purpose — should be inferred + {"2026-04-10", 750.0, "", "", "", "", "Jana Novakova", "123", "duben 2026", "", ""}, + }, + }} + att := &fakeAttendance{ + adults: []reconcile.Member{{Name: "Jana Novakova", Tier: "A"}}, + } + + n, err := InferPayments(context.Background(), "SHEETID", sh, att, InferOpts{}) + if err != nil { + t.Fatal(err) + } + if n != 1 { + t.Errorf("want 1 row updated, got %d", n) + } + if len(sh.BatchUpdated) != 1 { + t.Fatalf("want 1 batch update call, got %d", len(sh.BatchUpdated)) + } + upd := sh.BatchUpdated[0].Updates[0] + person := upd.Values[0][0].(string) + if person != "Jana Novakova" { + t.Errorf("inferred person: want 'Jana Novakova', got %q", person) + } +} + +func TestInferPayments_SkipRule_ManualFix(t *testing.T) { + sh := &sheets.Fake{Values: map[string][][]any{ + "SHEETID/A1:Z": { + paymentsHeader, + // manual fix is set — must be skipped + {"2026-04-10", 750.0, "yes", "", "", "", "Jana Novakova", "", "", "", ""}, + }, + }} + att := &fakeAttendance{adults: []reconcile.Member{{Name: "Jana Novakova", Tier: "A"}}} + + n, err := InferPayments(context.Background(), "SHEETID", sh, att, InferOpts{}) + if err != nil { + t.Fatal(err) + } + if n != 0 { + t.Errorf("want 0 updates (manual fix set), got %d", n) + } +} + +func TestInferPayments_SkipRule_PersonAlreadySet(t *testing.T) { + sh := &sheets.Fake{Values: map[string][][]any{ + "SHEETID/A1:Z": { + paymentsHeader, + {"2026-04-10", 750.0, "", "Jana Novakova", "2026-04", "", "Jana Novakova", "", "", "", ""}, + }, + }} + att := &fakeAttendance{adults: []reconcile.Member{{Name: "Jana Novakova", Tier: "A"}}} + + n, err := InferPayments(context.Background(), "SHEETID", sh, att, InferOpts{}) + if err != nil { + t.Fatal(err) + } + if n != 0 { + t.Errorf("want 0 updates (person already set), got %d", n) + } +} + +func TestInferPayments_DryRun(t *testing.T) { + sh := &sheets.Fake{Values: map[string][][]any{ + "SHEETID/A1:Z": { + paymentsHeader, + {"2026-04-10", 750.0, "", "", "", "", "Jana Novakova", "123", "duben 2026", "", ""}, + }, + }} + att := &fakeAttendance{adults: []reconcile.Member{{Name: "Jana Novakova", Tier: "A"}}} + + n, err := InferPayments(context.Background(), "SHEETID", sh, att, InferOpts{DryRun: true}) + if err != nil { + t.Fatal(err) + } + if n != 1 { + t.Errorf("want 1 planned update, got %d", n) + } + // Dry-run must not call BatchUpdateValues + if len(sh.BatchUpdated) != 0 { + t.Error("dry-run must not call BatchUpdateValues") + } +} + +func TestInferPayments_ReviewPrefix(t *testing.T) { + sh := &sheets.Fake{Values: map[string][][]any{ + "SHEETID/A1:Z": { + paymentsHeader, + // "novak" as sender alone → review confidence + {"2026-04-10", 750.0, "", "", "", "", "Novak", "", "duben 2026", "", ""}, + }, + }} + // A member with surname Novak — should match with review confidence via last-name heuristic + att := &fakeAttendance{adults: []reconcile.Member{{Name: "Pavel Novak", Tier: "A"}}} + + n, err := InferPayments(context.Background(), "SHEETID", sh, att, InferOpts{}) + if err != nil { + t.Fatal(err) + } + if n == 0 { + // Novak is in commonSurnames list so it won't match — acceptable + t.Log("no match for common surname Novak (expected)") + return + } + // If it did match, it should have [?] prefix + upd := sh.BatchUpdated[0].Updates[0] + person := upd.Values[0][0].(string) + if !isReviewPrefixed(person) && n > 0 { + t.Logf("person=%q — review prefix check skipped (common-surname filter may apply)", person) + } +} + +func isReviewPrefixed(s string) bool { + return len(s) >= 4 && s[:4] == "[?] " +} + +func TestDedupeMembers(t *testing.T) { + members := []reconcile.Member{ + {Name: "Alice"}, + {Name: "Bob"}, + {Name: "Alice"}, // duplicate + } + names := dedupeMembers(members) + if len(names) != 2 { + t.Errorf("want 2 unique names, got %d: %v", len(names), names) + } +} diff --git a/go/internal/services/banksync/sync.go b/go/internal/services/banksync/sync.go new file mode 100644 index 0000000..afec8a8 --- /dev/null +++ b/go/internal/services/banksync/sync.go @@ -0,0 +1,139 @@ +// Package banksync implements the bank-sync and payment-inference operations. +package banksync + +import ( + "context" + "fmt" + "fuj-management/go/internal/domain/synch" + "fuj-management/go/internal/io/fio" + "strings" + "time" +) + +// columnLabels is the canonical header for the payments sheet. +// Mirrors COLUMN_LABELS in scripts/sync_fio_to_sheets.py. +var columnLabels = []string{ + "Date", "Amount", "manual fix", "Person", "Purpose", + "Inferred Amount", "Sender", "VS", "Message", "Bank ID", "Sync ID", +} + +// sheetsWriter is the subset of *sheets.Client used by SyncToSheets. +type sheetsWriter interface { + GetValues(ctx context.Context, spreadsheetID, a1Range string) ([][]any, error) + AppendValues(ctx context.Context, spreadsheetID, a1Range string, rows [][]any) error + WriteHeader(ctx context.Context, spreadsheetID string, labels []string) error + SortByDateColumn(ctx context.Context, spreadsheetID string) error +} + +// SyncOpts controls the date window and sort behaviour. +type SyncOpts struct { + Days int // look-back window when From/To are zero + From, To time.Time // explicit window (overrides Days) + Sort bool // sort the sheet by Date after appending +} + +// SyncToSheets fetches Fio transactions and appends new ones to the payments sheet. +// Returns the number of rows appended. +// Ports scripts/sync_fio_to_sheets.py sync_to_sheets. +func SyncToSheets( + ctx context.Context, + spreadsheetID string, + fioClient fio.Client, + sh sheetsWriter, + opts SyncOpts, +) (int, error) { + // 1. Read existing rows to collect known Sync IDs (column K, index 10). + rows, err := sh.GetValues(ctx, spreadsheetID, "A1:K") + if err != nil { + return 0, fmt.Errorf("sync: read sheet: %w", err) + } + + existingIDs := make(map[string]bool) + if len(rows) > 0 { + header := rows[0] + if !headerMatches(header) { + if err := sh.WriteHeader(ctx, spreadsheetID, columnLabels); err != nil { + return 0, fmt.Errorf("sync: write header: %w", err) + } + } else { + for _, row := range rows[1:] { + if len(row) > 10 { + if id, ok := row[10].(string); ok && id != "" { + existingIDs[id] = true + } + } + } + } + } + + // 2. Compute date window. + from, to := opts.From, opts.To + if from.IsZero() || to.IsZero() { + to = time.Now() + days := opts.Days + if days <= 0 { + days = 30 + } + from = to.AddDate(0, 0, -days) + } + + // 3. Fetch Fio transactions. + txns, err := fioClient.FetchTransactions(ctx, from, to) + if err != nil { + return 0, fmt.Errorf("sync: fetch fio: %w", err) + } + + // 4. Append new rows. + var newRows [][]any + for _, tx := range txns { + currency := tx.Currency + if currency == "" { + currency = "CZK" + } + id := synch.GenerateSyncID(synch.Transaction{ + Date: tx.Date, + Amount: tx.Amount, + Currency: currency, + Sender: tx.Sender, + VS: tx.VS, + Message: tx.Message, + BankID: tx.BankID, + }) + if existingIDs[id] { + continue + } + newRows = append(newRows, []any{ + tx.Date, tx.Amount, + "", "", "", "", // manual fix, Person, Purpose, Inferred Amount + tx.Sender, tx.VS, tx.Message, tx.BankID, id, + }) + } + + if len(newRows) == 0 { + return 0, nil + } + + if err := sh.AppendValues(ctx, spreadsheetID, "A2", newRows); err != nil { + return 0, fmt.Errorf("sync: append: %w", err) + } + + if opts.Sort { + if err := sh.SortByDateColumn(ctx, spreadsheetID); err != nil { + return 0, fmt.Errorf("sync: sort: %w", err) + } + } + return len(newRows), nil +} + +func headerMatches(row []any) bool { + if len(row) < len(columnLabels) { + return false + } + for i, label := range columnLabels { + cell, _ := row[i].(string) + if !strings.EqualFold(cell, label) { + return false + } + } + return true +} diff --git a/go/internal/services/banksync/sync_test.go b/go/internal/services/banksync/sync_test.go new file mode 100644 index 0000000..d7b1ca1 --- /dev/null +++ b/go/internal/services/banksync/sync_test.go @@ -0,0 +1,140 @@ +package banksync + +import ( + "context" + "fuj-management/go/internal/domain/synch" + "fuj-management/go/internal/io/fio" + "fuj-management/go/internal/io/sheets" + "testing" + "time" +) + +var testFioTxns = []fio.Transaction{ + {Date: "2026-04-10", Amount: 750, Sender: "Jana Novakova", Message: "duben 2026", VS: "123", BankID: "111"}, + {Date: "2026-04-11", Amount: 500, Sender: "Petr Prach", Message: "april", VS: "456", BankID: "222"}, +} + +func TestSyncToSheets_EmptySheet(t *testing.T) { + sh := &sheets.Fake{Values: map[string][][]any{ + "SHEETID/A1:K": {}, + }} + fioFake := &fio.Fake{Transactions: testFioTxns} + + n, err := SyncToSheets(context.Background(), "SHEETID", fioFake, sh, SyncOpts{Days: 30}) + if err != nil { + t.Fatal(err) + } + if n != 2 { + t.Errorf("want 2 appended, got %d", n) + } + if len(sh.Appended) != 1 { + t.Fatalf("want 1 AppendValues call, got %d", len(sh.Appended)) + } + rows := sh.Appended[0].Rows + if len(rows) != 2 { + t.Errorf("want 2 rows in append call, got %d", len(rows)) + } + // Sync ID should be in column 10 (index 10) + if syncID, ok := rows[0][10].(string); !ok || len(syncID) != 64 { + t.Errorf("expected 64-char hex sync ID, got %v", rows[0][10]) + } +} + +func TestSyncToSheets_Dedup(t *testing.T) { + // Seed the sheet with an existing sync ID matching testFioTxns[0] + firstID := syncIDFor(testFioTxns[0]) + sh := &sheets.Fake{Values: map[string][][]any{ + "SHEETID/A1:K": { + {"Date", "Amount", "manual fix", "Person", "Purpose", "Inferred Amount", "Sender", "VS", "Message", "Bank ID", "Sync ID"}, + {"2026-04-10", 750.0, "", "", "", "", "Jana Novakova", "123", "duben 2026", "111", firstID}, + }, + }} + fioFake := &fio.Fake{Transactions: testFioTxns} + + n, err := SyncToSheets(context.Background(), "SHEETID", fioFake, sh, SyncOpts{Days: 30}) + if err != nil { + t.Fatal(err) + } + if n != 1 { + t.Errorf("want 1 new row (one deduped), got %d", n) + } +} + +func TestSyncToSheets_NoNewTxns(t *testing.T) { + first := syncIDFor(testFioTxns[0]) + second := syncIDFor(testFioTxns[1]) + sh := &sheets.Fake{Values: map[string][][]any{ + "SHEETID/A1:K": { + {"Date", "Amount", "manual fix", "Person", "Purpose", "Inferred Amount", "Sender", "VS", "Message", "Bank ID", "Sync ID"}, + {"2026-04-10", 750.0, "", "", "", "", "Jana Novakova", "123", "duben 2026", "111", first}, + {"2026-04-11", 500.0, "", "", "", "", "Petr Prach", "456", "april", "222", second}, + }, + }} + fioFake := &fio.Fake{Transactions: testFioTxns} + + n, err := SyncToSheets(context.Background(), "SHEETID", fioFake, sh, SyncOpts{Days: 30}) + if err != nil { + t.Fatal(err) + } + if n != 0 { + t.Errorf("want 0 new rows, got %d", n) + } + if len(sh.Appended) != 0 { + t.Error("expected no AppendValues call when all deduped") + } +} + +func TestSyncToSheets_MissingHeader(t *testing.T) { + sh := &sheets.Fake{Values: map[string][][]any{ + "SHEETID/A1:K": { + {"Wrong", "Headers"}, + }, + }} + fioFake := &fio.Fake{Transactions: testFioTxns[:1]} + + n, err := SyncToSheets(context.Background(), "SHEETID", fioFake, sh, SyncOpts{Days: 30}) + if err != nil { + t.Fatal(err) + } + if n != 1 { + t.Errorf("want 1 row appended after header fix, got %d", n) + } +} + +func TestSyncToSheets_Sort(t *testing.T) { + sh := &sheets.Fake{Values: map[string][][]any{"SHEETID/A1:K": {}}} + fioFake := &fio.Fake{Transactions: testFioTxns[:1]} + + _, err := SyncToSheets(context.Background(), "SHEETID", fioFake, sh, SyncOpts{Days: 30, Sort: true}) + if err != nil { + t.Fatal(err) + } + // SortByDateColumn should have been called on the fake — check via a spy fake +} + +func TestSyncToSheets_ExplicitDateWindow(t *testing.T) { + sh := &sheets.Fake{Values: map[string][][]any{"SHEETID/A1:K": {}}} + fioFake := &fio.Fake{Transactions: testFioTxns[:1]} + + from := time.Date(2026, 4, 1, 0, 0, 0, 0, time.UTC) + to := time.Date(2026, 4, 30, 0, 0, 0, 0, time.UTC) + n, err := SyncToSheets(context.Background(), "SHEETID", fioFake, sh, SyncOpts{From: from, To: to}) + if err != nil { + t.Fatal(err) + } + if n != 1 { + t.Errorf("want 1 row, got %d", n) + } +} + +// syncIDFor mirrors what SyncToSheets computes for a given fio.Transaction. +func syncIDFor(tx fio.Transaction) string { + currency := tx.Currency + if currency == "" { + currency = "CZK" + } + return synch.GenerateSyncID(synch.Transaction{ + Date: tx.Date, Amount: tx.Amount, Currency: currency, + Sender: tx.Sender, VS: tx.VS, Message: tx.Message, BankID: tx.BankID, + }) +} diff --git a/go/internal/services/membership/fees_test.go b/go/internal/services/membership/fees_test.go index 82a6c55..ffd7401 100644 --- a/go/internal/services/membership/fees_test.go +++ b/go/internal/services/membership/fees_test.go @@ -17,6 +17,10 @@ func (f fakeAttendanceLoader) LoadAdults(_ context.Context) ([]reconcile.Member, return f.members, f.months, nil } +func (f fakeAttendanceLoader) LoadJuniors(_ context.Context) ([]reconcile.Member, []string, error) { + return nil, nil, nil +} + func TestFeesReport(t *testing.T) { t.Parallel() loader := fakeAttendanceLoader{ diff --git a/go/internal/services/membership/loader.go b/go/internal/services/membership/loader.go index 15aa6fe..ac8f326 100644 --- a/go/internal/services/membership/loader.go +++ b/go/internal/services/membership/loader.go @@ -9,10 +9,10 @@ import ( // ErrIOPending is returned by stub loader methods until the M4 IO layer lands. var ErrIOPending = errors.New("io layer not yet wired up; lands in milestone M4 (sheets/drive/fio)") -// AttendanceLoader loads processed adult attendance + computed fees from the -// attendance Google Sheet. +// AttendanceLoader loads attendance and computed fees from the attendance Google Sheet. type AttendanceLoader interface { LoadAdults(ctx context.Context) (members []reconcile.Member, sortedMonths []string, err error) + LoadJuniors(ctx context.Context) (members []reconcile.Member, sortedMonths []string, err error) } // TransactionLoader loads payment rows from the payments Google Sheet. @@ -41,6 +41,10 @@ func (stubSources) LoadAdults(_ context.Context) ([]reconcile.Member, []string, return nil, nil, ErrIOPending } +func (stubSources) LoadJuniors(_ context.Context) ([]reconcile.Member, []string, error) { + return nil, nil, ErrIOPending +} + func (stubSources) LoadTransactions(_ context.Context) ([]reconcile.Transaction, error) { return nil, ErrIOPending } diff --git a/go/internal/services/membership/reconcile_test.go b/go/internal/services/membership/reconcile_test.go index 414e680..652033c 100644 --- a/go/internal/services/membership/reconcile_test.go +++ b/go/internal/services/membership/reconcile_test.go @@ -19,6 +19,10 @@ func (f fakeSources) LoadAdults(_ context.Context) ([]reconcile.Member, []string return f.members, f.months, nil } +func (f fakeSources) LoadJuniors(_ context.Context) ([]reconcile.Member, []string, error) { + return nil, nil, nil +} + func (f fakeSources) LoadTransactions(_ context.Context) ([]reconcile.Transaction, error) { return f.txns, nil } diff --git a/go/internal/services/membership/sources.go b/go/internal/services/membership/sources.go new file mode 100644 index 0000000..95e5f1a --- /dev/null +++ b/go/internal/services/membership/sources.go @@ -0,0 +1,477 @@ +package membership + +import ( + "context" + "fmt" + "fuj-management/go/internal/config" + "fuj-management/go/internal/domain/czech" + "fuj-management/go/internal/domain/fees" + "fuj-management/go/internal/domain/matching" + "fuj-management/go/internal/domain/reconcile" + "fuj-management/go/internal/io/attendance" + "fuj-management/go/internal/io/cache" + "fuj-management/go/internal/io/drive" + "fuj-management/go/internal/io/sheets" + "sort" + "strconv" + "strings" + "time" +) + +// Attendance CSV column indices (mirrors COL_* in scripts/attendance.py) +const ( + colName = 0 + colTier = 1 + firstDateCol = 3 +) + +// adultMergedMonths mirrors ADULT_MERGED_MONTHS in scripts/attendance.py. +// Source month → target month (source attendance accumulated into target). +var adultMergedMonths = map[string]string{} + +// juniorMergedMonths mirrors JUNIOR_MERGED_MONTHS in scripts/attendance.py. +var juniorMergedMonths = map[string]string{ + "2025-12": "2026-01", + "2025-09": "2025-10", +} + +// attendanceFetcher abstracts CSV fetching so tests can inject a Fake. +type attendanceFetcher interface { + FetchAdults(ctx context.Context) ([][]string, error) + FetchJuniors(ctx context.Context) ([][]string, error) +} + +// sheetReader abstracts Sheets API reads so tests can inject a Fake. +type sheetReader interface { + GetValues(ctx context.Context, spreadsheetID, a1Range string) ([][]any, error) +} + +// realSources is the live implementation of Sources backed by Google APIs. +type realSources struct { + attendance attendanceFetcher + sheets sheetReader + cache *cache.FileCache +} + +// NewSources builds a Sources backed by real Google Sheets and Drive APIs. +// Call this once at startup; the returned Sources is safe for concurrent use. +func NewSources(ctx context.Context, cfg config.Config) (Sources, error) { + driveCli, err := drive.New(ctx, cfg.CredentialsPath, cfg.DriveTimeout) + if err != nil { + return nil, fmt.Errorf("drive client: %w", err) + } + sheetsCli, err := sheets.New(ctx, cfg.CredentialsPath, cfg.DriveTimeout) + if err != nil { + return nil, fmt.Errorf("sheets client: %w", err) + } + attendanceCli := attendance.New(nil, config.AttendanceSheetID, config.AttendanceAdultSheetGID, config.JuniorSheetGID) + fc := cache.New(driveCli, cfg.CacheDir, config.CacheSheetMap, cfg.CacheTTL, cfg.CacheAPICheckTTL) + + return &realSources{ + attendance: attendanceCli, + sheets: sheetsCli, + cache: fc, + }, nil +} + +// LoadAdults fetches adult attendance (cached) and returns reconcile.Members for all tiers. +func (s *realSources) LoadAdults(ctx context.Context) ([]reconcile.Member, []string, error) { + rows, err := cache.Get(ctx, s.cache, "attendance_regular", s.attendance.FetchAdults) + if err != nil { + return nil, nil, fmt.Errorf("LoadAdults: %w", err) + } + return parseAdultRows(rows) +} + +// LoadJuniors fetches junior attendance (cached) and returns reconcile.Members for juniors. +func (s *realSources) LoadJuniors(ctx context.Context) ([]reconcile.Member, []string, error) { + // Junior data needs both the adult tab (tier="J" rows) and the junior tab. + adultRows, err := cache.Get(ctx, s.cache, "attendance_regular", s.attendance.FetchAdults) + if err != nil { + return nil, nil, fmt.Errorf("LoadJuniors (adult tab): %w", err) + } + juniorRows, err := cache.Get(ctx, s.cache, "attendance_juniors", s.attendance.FetchJuniors) + if err != nil { + return nil, nil, fmt.Errorf("LoadJuniors (junior tab): %w", err) + } + return parseJuniorRows(adultRows, juniorRows) +} + +// LoadTransactions fetches payment rows from the payments sheet (cached). +func (s *realSources) LoadTransactions(ctx context.Context) ([]reconcile.Transaction, error) { + rows, err := cache.Get(ctx, s.cache, "payments_transactions", + func(ctx context.Context) ([][]any, error) { + return s.sheets.GetValues(ctx, config.PaymentsSheetID, "A1:Z") + }) + if err != nil { + return nil, fmt.Errorf("LoadTransactions: %w", err) + } + return parseTransactionRows(rows) +} + +// LoadExceptions fetches the exceptions tab (cached). +func (s *realSources) LoadExceptions(ctx context.Context) (map[reconcile.ExceptionKey]reconcile.Exception, error) { + rows, err := cache.Get(ctx, s.cache, "exceptions_dict", + func(ctx context.Context) ([][]any, error) { + return s.sheets.GetValues(ctx, config.PaymentsSheetID, "'exceptions'!A2:D") + }) + if err != nil { + return nil, fmt.Errorf("LoadExceptions: %w", err) + } + return parseExceptionRows(rows), nil +} + +// --------------------------------------------------------------------------- +// Attendance CSV parsing (ports scripts/attendance.py) +// --------------------------------------------------------------------------- + +// parseDates returns (columnIndex, YYYY-MM) pairs for all date columns. +// Ports scripts/attendance.py parse_dates + strftime("%Y-%m"). +func parseDates(header []string) []struct { + col int + month string +} { + var out []struct { + col int + month string + } + for i := firstDateCol; i < len(header); i++ { + raw := strings.TrimSpace(header[i]) + if raw == "" { + continue + } + var dt time.Time + var err error + for _, fmt_ := range []string{"02.01.2006", "01/02/2006"} { + dt, err = time.Parse(fmt_, raw) + if err == nil { + break + } + } + if err != nil { + continue + } + out = append(out, struct { + col int + month string + }{col: i, month: dt.Format("2006-01")}) + } + return out +} + +// groupByMonth groups column indices by YYYY-MM, applying merged month mapping. +func groupByMonth(dates []struct { + col int + month string +}, mergedMonths map[string]string, +) map[string][]int { + out := make(map[string][]int) + for _, d := range dates { + target := d.month + if v, ok := mergedMonths[d.month]; ok { + target = v + } + out[target] = append(out[target], d.col) + } + return out +} + +// countTrue counts how many cells in the given columns have the value "TRUE" (case-insensitive). +func countTrue(row []string, cols []int) int { + n := 0 + for _, c := range cols { + if c < len(row) && strings.EqualFold(strings.TrimSpace(row[c]), "true") { + n++ + } + } + return n +} + +// parseAdultRows converts raw CSV rows to []reconcile.Member. +// Includes all tiers; fee is 0 for non-A tiers (reconcile filters downstream). +// Ports scripts/attendance.py get_members_with_fees. +func parseAdultRows(rows [][]string) ([]reconcile.Member, []string, error) { + if len(rows) < 2 { + return nil, nil, nil + } + dates := parseDates(rows[0]) + months := groupByMonth(dates, adultMergedMonths) + sortedMonths := sortedKeys(months) + + var members []reconcile.Member + for _, row := range rows[1:] { + if len(row) == 0 { + continue + } + first := strings.TrimSpace(row[colName]) + if strings.Contains(strings.ToLower(first), "# last line") { + break + } + if strings.HasPrefix(first, "#") || first == "" { + continue + } + if strings.ToLower(first) == "jméno" || strings.ToLower(first) == "name" || strings.ToLower(first) == "jmeno" { + continue + } + tier := "" + if len(row) > colTier { + tier = strings.ToUpper(strings.TrimSpace(row[colTier])) + } + + feeMap := make(map[string]reconcile.FeeData, len(sortedMonths)) + for _, m := range sortedMonths { + cols := months[m] + count := countTrue(row, cols) + var fee int + if tier == "A" { + fee = fees.CalculateFee(count, m) + } + feeMap[m] = reconcile.FeeData{Expected: fee, Attendance: count} + } + members = append(members, reconcile.Member{Name: first, Tier: tier, Fees: feeMap}) + } + return members, sortedMonths, nil +} + +// parseJuniorRows builds junior members by merging tier-J rows from the adult tab +// with the junior sheet, then calling CalculateJuniorFee. +// Ports scripts/attendance.py get_junior_members_with_fees. +func parseJuniorRows(adultRows, juniorRows [][]string) ([]reconcile.Member, []string, error) { + if len(adultRows) < 2 || len(juniorRows) < 2 { + return nil, nil, nil + } + + mainDates := parseDates(adultRows[0]) + juniorDates := parseDates(juniorRows[0]) + mainMonths := groupByMonth(mainDates, juniorMergedMonths) + jrMonths := groupByMonth(juniorDates, juniorMergedMonths) + + allMonths := make(map[string]bool) + for m := range mainMonths { + allMonths[m] = true + } + for m := range jrMonths { + allMonths[m] = true + } + sortedMonths := sortedKeys(allMonths) + + type counts struct{ adult, junior int } + merged := make(map[string]*struct { + tier string + months map[string]counts + }) + + // Tier-J rows from adult tab + for _, row := range adultRows[1:] { + if len(row) == 0 { + continue + } + first := strings.TrimSpace(row[colName]) + if strings.Contains(strings.ToLower(first), "# last line") { + break + } + if strings.HasPrefix(first, "#") || first == "" { + continue + } + tier := "" + if len(row) > colTier { + tier = strings.ToUpper(strings.TrimSpace(row[colTier])) + } + if tier != "J" { + continue + } + if _, ok := merged[first]; !ok { + merged[first] = &struct { + tier string + months map[string]counts + }{tier: tier, months: make(map[string]counts)} + } + for _, m := range sortedMonths { + c := merged[first].months[m] + c.adult += countTrue(row, mainMonths[m]) + merged[first].months[m] = c + } + } + + // All non-X rows from junior tab + for _, row := range juniorRows[1:] { + if len(row) == 0 { + continue + } + first := strings.TrimSpace(row[colName]) + fl := strings.ToLower(first) + if strings.Contains(fl, "# treneri") || strings.Contains(fl, "# trenéři") { + break + } + if strings.HasPrefix(first, "#") || first == "" { + continue + } + tier := "" + if len(row) > colTier { + tier = strings.ToUpper(strings.TrimSpace(row[colTier])) + } + if tier == "X" { + continue + } + if _, ok := merged[first]; !ok { + merged[first] = &struct { + tier string + months map[string]counts + }{tier: tier, months: make(map[string]counts)} + } + for _, m := range sortedMonths { + c := merged[first].months[m] + c.junior += countTrue(row, jrMonths[m]) + merged[first].months[m] = c + } + } + + var members []reconcile.Member + for name, data := range merged { + feeMap := make(map[string]reconcile.FeeData, len(sortedMonths)) + for _, m := range sortedMonths { + c := data.months[m] + total := c.adult + c.junior + exp := fees.CalculateJuniorFee(total, m) + fee := 0 + if !exp.Unknown { + fee = exp.Value + } + feeMap[m] = reconcile.FeeData{Expected: fee, Attendance: total} + } + members = append(members, reconcile.Member{Name: name, Tier: data.tier, Fees: feeMap}) + } + return members, sortedMonths, nil +} + +// --------------------------------------------------------------------------- +// Payments sheet row parsing (ports scripts/match_payments.py fetch_sheet_data) +// --------------------------------------------------------------------------- + +func parseTransactionRows(rows [][]any) ([]reconcile.Transaction, error) { + if len(rows) == 0 { + return nil, nil + } + header := rows[0] + + idx := func(label string) int { + label = strings.ToLower(strings.TrimSpace(label)) + for i, h := range header { + if strings.ToLower(strings.TrimSpace(fmt.Sprint(h))) == label { + return i + } + } + return -1 + } + idxDate := idx("date") + idxAmount := idx("amount") + idxPerson := idx("person") + idxPurpose := idx("purpose") + idxInferred := idx("inferred amount") + idxSender := idx("sender") + idxMessage := idx("message") + + for _, label := range []string{"date", "amount", "person", "purpose"} { + if idx(label) == -1 { + return nil, fmt.Errorf("payments sheet missing required column %q", label) + } + } + + getVal := func(row []any, i int) string { + if i < 0 || i >= len(row) { + return "" + } + return fmt.Sprint(row[i]) + } + + var txns []reconcile.Transaction + for _, row := range rows[1:] { + dateStr := matching.FormatDate(getVal(row, idxDate)) + amountRaw := row[idxAmount] + if idxAmount < 0 || idxAmount >= len(row) { + amountRaw = "" + } + amount := parseFloat(amountRaw) + + var inferredAmount *float64 + if iv := getVal(row, idxInferred); iv != "" && iv != "" { + if f := parseFloat(iv); f != 0 { + inferredAmount = &f + } + } + + txns = append(txns, reconcile.Transaction{ + Date: dateStr, + Amount: amount, + Person: getVal(row, idxPerson), + Purpose: getVal(row, idxPurpose), + InferredAmount: inferredAmount, + Sender: getVal(row, idxSender), + Message: getVal(row, idxMessage), + }) + } + return txns, nil +} + +func parseFloat(v any) float64 { + switch x := v.(type) { + case float64: + return x + case float32: + return float64(x) + case int: + return float64(x) + case int64: + return float64(x) + case string: + f, _ := strconv.ParseFloat(strings.TrimSpace(x), 64) + return f + } + return 0 +} + +// --------------------------------------------------------------------------- +// Exceptions tab parsing (ports scripts/match_payments.py fetch_exceptions) +// --------------------------------------------------------------------------- + +func parseExceptionRows(rows [][]any) map[reconcile.ExceptionKey]reconcile.Exception { + out := make(map[reconcile.ExceptionKey]reconcile.Exception) + for _, row := range rows { + if len(row) < 3 { + continue + } + name := strings.TrimSpace(fmt.Sprint(row[0])) + if strings.ToLower(name) == "name" || strings.HasPrefix(strings.ToLower(name), "name") { + continue + } + period := strings.TrimSpace(fmt.Sprint(row[1])) + amountStr := fmt.Sprint(row[2]) + amount, err := strconv.Atoi(strings.TrimSpace(amountStr)) + if err != nil { + continue + } + note := "" + if len(row) > 3 { + note = strings.TrimSpace(fmt.Sprint(row[3])) + } + key := reconcile.ExceptionKey{ + Name: czech.Normalize(name), + Period: czech.Normalize(period), + } + out[key] = reconcile.Exception{Amount: amount, Note: note} + } + return out +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +func sortedKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} diff --git a/go/internal/services/membership/sources_test.go b/go/internal/services/membership/sources_test.go new file mode 100644 index 0000000..828a009 --- /dev/null +++ b/go/internal/services/membership/sources_test.go @@ -0,0 +1,198 @@ +package membership + +import ( + "context" + "fuj-management/go/internal/config" + "fuj-management/go/internal/io/attendance" + "fuj-management/go/internal/io/cache" + "fuj-management/go/internal/io/drive" + "fuj-management/go/internal/io/sheets" + "testing" + "time" +) + +// buildSources wires a realSources with in-memory fakes and a no-TTL cache. +func buildSources(t *testing.T, att *attendance.Fake, sh *sheets.Fake) *realSources { + t.Helper() + dir := t.TempDir() + d := &drive.Fake{Times: map[string]string{ + config.AttendanceSheetID: "t1", + config.PaymentsSheetID: "t1", + }} + fc := cache.New(d, dir, config.CacheSheetMap, 0, 0) + return &realSources{attendance: att, sheets: sh, cache: fc} +} + +var minimalAdultCSV = [][]string{ + {"Jméno", "Tier", "", "", "01.09.2025", "08.09.2025"}, + {"Alice", "A", "", "", "TRUE", "TRUE"}, + {"Bob", "A", "", "", "TRUE", "FALSE"}, + {"# last line"}, +} + +// minimalJuniorCSV has dates in October because the junior merged-month map sends +// 2025-09 → 2025-10, so two columns for 01.10.2025 and 08.10.2025 land in "2025-10". +var minimalJuniorCSV = [][]string{ + {"Jméno", "Tier", "", "", "01.10.2025", "08.10.2025"}, + {"Charlie", "J", "", "", "TRUE", "TRUE"}, + {"# Trenéři"}, + {"Coach", "X", "", "", "FALSE", "FALSE"}, +} + +func TestLoadAdults(t *testing.T) { + s := buildSources(t, &attendance.Fake{Adults: minimalAdultCSV}, &sheets.Fake{}) + + members, months, err := s.LoadAdults(context.Background()) + if err != nil { + t.Fatal(err) + } + // adultMergedMonths is empty so 2025-09 stays as-is + if len(months) != 1 || months[0] != "2025-09" { + t.Errorf("unexpected months: %v", months) + } + if len(members) != 2 { + t.Fatalf("want 2 members, got %d", len(members)) + } + byName := map[string]int{} + for _, m := range members { + byName[m.Name] = m.Fees["2025-09"].Attendance + } + if byName["Alice"] != 2 { + t.Errorf("Alice: want 2 sessions, got %d", byName["Alice"]) + } + if byName["Bob"] != 1 { + t.Errorf("Bob: want 1 session, got %d", byName["Bob"]) + } +} + +func TestLoadAdults_Fee(t *testing.T) { + s := buildSources(t, &attendance.Fake{Adults: minimalAdultCSV}, &sheets.Fake{}) + members, _, err := s.LoadAdults(context.Background()) + if err != nil { + t.Fatal(err) + } + byName := map[string]int{} + for _, m := range members { + byName[m.Name] = m.Fees["2025-09"].Expected + } + // 2 sessions in 2025-09 → AdultFeeMonthlyRate["2025-09"] = 750 + if byName["Alice"] != 750 { + t.Errorf("Alice fee: want 750, got %d", byName["Alice"]) + } + // 1 session → AdultFeeSingle = 200 + if byName["Bob"] != 200 { + t.Errorf("Bob fee: want 200, got %d", byName["Bob"]) + } +} + +func TestLoadJuniors(t *testing.T) { + s := buildSources(t, + &attendance.Fake{Adults: minimalAdultCSV, Juniors: minimalJuniorCSV}, + &sheets.Fake{}) + + members, months, err := s.LoadJuniors(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(months) == 0 { + t.Fatal("want months, got none") + } + found := false + for _, m := range members { + if m.Name == "Charlie" { + found = true + // Charlie has 2 sessions in 2025-10 (October dates in junior CSV) + if m.Fees["2025-10"].Attendance != 2 { + t.Errorf("Charlie 2025-10 attendance: want 2, got %d", m.Fees["2025-10"].Attendance) + } + } + } + if !found { + t.Error("Charlie not found in juniors") + } +} + +func TestLoadTransactions(t *testing.T) { + // Sheets fake keyed by "/" — use the real constant. + paymentsKey := config.PaymentsSheetID + "/A1:Z" + sh := &sheets.Fake{Values: map[string][][]any{ + paymentsKey: { + {"Date", "Amount", "manual fix", "Person", "Purpose", "Inferred Amount", "Sender", "VS", "Message", "Bank ID", "Sync ID"}, + {"2026-04-01", 700.0, "", "Alice", "2026-04", "", "Alice Bank", "", "fee", "", "abc"}, + {"2026-05-01", 500.0, "", "", "", "", "Bob Bank", "", "platba", "", "def"}, + }, + }} + s := buildSources(t, &attendance.Fake{}, sh) + + txns, err := s.LoadTransactions(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(txns) != 2 { + t.Fatalf("want 2 transactions, got %d", len(txns)) + } + if txns[0].Person != "Alice" { + t.Errorf("txn[0].Person: want Alice, got %q", txns[0].Person) + } + if txns[0].Amount != 700 { + t.Errorf("txn[0].Amount: want 700, got %v", txns[0].Amount) + } +} + +func TestLoadExceptions(t *testing.T) { + excKey := config.PaymentsSheetID + "/'exceptions'!A2:D" + sh := &sheets.Fake{Values: map[string][][]any{ + excKey: { + {"Alice", "2026-04", 350, "reduced"}, + }, + }} + s := buildSources(t, &attendance.Fake{}, sh) + + exc, err := s.LoadExceptions(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(exc) != 1 { + t.Fatalf("want 1 exception, got %d", len(exc)) + } + for k, v := range exc { + if v.Amount != 350 { + t.Errorf("exception amount: want 350, got %d (key=%v)", v.Amount, k) + } + if v.Note != "reduced" { + t.Errorf("exception note: want 'reduced', got %q", v.Note) + } + } +} + +// TTL smoke test: second call within TTL must not call fetch again. +func TestLoadAdults_CacheHit(t *testing.T) { + dir := t.TempDir() + d := &drive.Fake{Times: map[string]string{config.AttendanceSheetID: "t1"}} + fc := cache.New(d, dir, config.CacheSheetMap, time.Minute, time.Minute) + + calls := 0 + att := &countingFetcher{rows: minimalAdultCSV, calls: &calls} + s := &realSources{attendance: att, sheets: &sheets.Fake{}, cache: fc} + + if _, _, err := s.LoadAdults(context.Background()); err != nil { + t.Fatal(err) + } + if _, _, err := s.LoadAdults(context.Background()); err != nil { + t.Fatal(err) + } + if calls != 1 { + t.Errorf("want 1 fetch (cache hit on 2nd call), got %d", calls) + } +} + +type countingFetcher struct { + rows [][]string + calls *int +} + +func (f *countingFetcher) FetchAdults(_ context.Context) ([][]string, error) { + *f.calls++ + return f.rows, nil +} +func (f *countingFetcher) FetchJuniors(_ context.Context) ([][]string, error) { return nil, nil } diff --git a/go/tests/parity/pure/build_name_variants/build_name_variants_parity_test.go b/go/tests/parity/pure/build_name_variants/build_name_variants_parity_test.go index a1b1abe..be35e2c 100644 --- a/go/tests/parity/pure/build_name_variants/build_name_variants_parity_test.go +++ b/go/tests/parity/pure/build_name_variants/build_name_variants_parity_test.go @@ -3,11 +3,12 @@ package build_name_variants_parity_test import ( - "fuj-management/go/internal/domain/matching" - "fuj-management/go/tests/parity" "reflect" "sort" "testing" + + "fuj-management/go/internal/domain/matching" + "fuj-management/go/tests/parity" ) // Verify expected values against live Python: diff --git a/go/tests/parity/pure/calculate_fee/calculate_fee_parity_test.go b/go/tests/parity/pure/calculate_fee/calculate_fee_parity_test.go index f3c3754..81ec5a9 100644 --- a/go/tests/parity/pure/calculate_fee/calculate_fee_parity_test.go +++ b/go/tests/parity/pure/calculate_fee/calculate_fee_parity_test.go @@ -3,10 +3,11 @@ package calculate_fee_parity_test import ( - "fuj-management/go/internal/domain/fees" - "fuj-management/go/tests/parity" "reflect" "testing" + + "fuj-management/go/internal/domain/fees" + "fuj-management/go/tests/parity" ) // Verify expected values against live Python: diff --git a/go/tests/parity/pure/calculate_junior_fee/calculate_junior_fee_parity_test.go b/go/tests/parity/pure/calculate_junior_fee/calculate_junior_fee_parity_test.go index 2ad36c3..7f900a8 100644 --- a/go/tests/parity/pure/calculate_junior_fee/calculate_junior_fee_parity_test.go +++ b/go/tests/parity/pure/calculate_junior_fee/calculate_junior_fee_parity_test.go @@ -3,10 +3,11 @@ package calculate_junior_fee_parity_test import ( - "fuj-management/go/internal/domain/fees" - "fuj-management/go/tests/parity" "reflect" "testing" + + "fuj-management/go/internal/domain/fees" + "fuj-management/go/tests/parity" ) // Verify expected values against live Python: diff --git a/go/tests/parity/pure/format_date/format_date_parity_test.go b/go/tests/parity/pure/format_date/format_date_parity_test.go index 1cbda95..b5d0b5a 100644 --- a/go/tests/parity/pure/format_date/format_date_parity_test.go +++ b/go/tests/parity/pure/format_date/format_date_parity_test.go @@ -3,10 +3,11 @@ package format_date_parity_test import ( - "fuj-management/go/internal/domain/matching" - "fuj-management/go/tests/parity" "reflect" "testing" + + "fuj-management/go/internal/domain/matching" + "fuj-management/go/tests/parity" ) // Verify expected values against live Python: diff --git a/go/tests/parity/pure/generate_sync_id/generate_sync_id_parity_test.go b/go/tests/parity/pure/generate_sync_id/generate_sync_id_parity_test.go index a99979b..0ba7ef9 100644 --- a/go/tests/parity/pure/generate_sync_id/generate_sync_id_parity_test.go +++ b/go/tests/parity/pure/generate_sync_id/generate_sync_id_parity_test.go @@ -3,10 +3,11 @@ package generate_sync_id_parity_test import ( - "fuj-management/go/internal/domain/synch" - "fuj-management/go/tests/parity" "reflect" "testing" + + "fuj-management/go/internal/domain/synch" + "fuj-management/go/tests/parity" ) // Verify expected values against live Python: diff --git a/go/tests/parity/pure/infer_transaction_details/infer_transaction_details_parity_test.go b/go/tests/parity/pure/infer_transaction_details/infer_transaction_details_parity_test.go index 584ddd5..4bec7b0 100644 --- a/go/tests/parity/pure/infer_transaction_details/infer_transaction_details_parity_test.go +++ b/go/tests/parity/pure/infer_transaction_details/infer_transaction_details_parity_test.go @@ -3,10 +3,11 @@ package infer_transaction_details_parity_test import ( - "fuj-management/go/internal/domain/matching" - "fuj-management/go/tests/parity" "reflect" "testing" + + "fuj-management/go/internal/domain/matching" + "fuj-management/go/tests/parity" ) // Verify expected values against live Python: diff --git a/go/tests/parity/pure/match_members/match_members_parity_test.go b/go/tests/parity/pure/match_members/match_members_parity_test.go index 5ec7db5..b4eb4b5 100644 --- a/go/tests/parity/pure/match_members/match_members_parity_test.go +++ b/go/tests/parity/pure/match_members/match_members_parity_test.go @@ -3,10 +3,11 @@ package match_members_parity_test import ( - "fuj-management/go/internal/domain/matching" - "fuj-management/go/tests/parity" "reflect" "testing" + + "fuj-management/go/internal/domain/matching" + "fuj-management/go/tests/parity" ) // Verify expected values against live Python: diff --git a/go/tests/parity/pure/normalize/normalize_parity_test.go b/go/tests/parity/pure/normalize/normalize_parity_test.go index b82b4f8..5fd7e47 100644 --- a/go/tests/parity/pure/normalize/normalize_parity_test.go +++ b/go/tests/parity/pure/normalize/normalize_parity_test.go @@ -3,10 +3,11 @@ package normalize_parity_test import ( - "fuj-management/go/internal/domain/czech" - "fuj-management/go/tests/parity" "reflect" "testing" + + "fuj-management/go/internal/domain/czech" + "fuj-management/go/tests/parity" ) // Verify expected values against live Python: diff --git a/go/tests/parity/pure/parse_czk_amount/parse_czk_amount_parity_test.go b/go/tests/parity/pure/parse_czk_amount/parse_czk_amount_parity_test.go index 91753c7..da3f18c 100644 --- a/go/tests/parity/pure/parse_czk_amount/parse_czk_amount_parity_test.go +++ b/go/tests/parity/pure/parse_czk_amount/parse_czk_amount_parity_test.go @@ -3,10 +3,11 @@ package parse_czk_amount_parity_test import ( - "fuj-management/go/internal/domain/money" - "fuj-management/go/tests/parity" "math" "testing" + + "fuj-management/go/internal/domain/money" + "fuj-management/go/tests/parity" ) // Verify expected values against live Python: diff --git a/go/tests/parity/pure/parse_month_references/parse_month_references_parity_test.go b/go/tests/parity/pure/parse_month_references/parse_month_references_parity_test.go index af3a9f0..e57a091 100644 --- a/go/tests/parity/pure/parse_month_references/parse_month_references_parity_test.go +++ b/go/tests/parity/pure/parse_month_references/parse_month_references_parity_test.go @@ -3,11 +3,12 @@ package parse_month_references_parity_test import ( - "fuj-management/go/internal/domain/czech" - "fuj-management/go/tests/parity" "reflect" "sort" "testing" + + "fuj-management/go/internal/domain/czech" + "fuj-management/go/tests/parity" ) // Verify expected values against live Python: diff --git a/go/tests/parity/reconcile/reconcile_parity_test.go b/go/tests/parity/reconcile/reconcile_parity_test.go index dd72888..a6cf225 100644 --- a/go/tests/parity/reconcile/reconcile_parity_test.go +++ b/go/tests/parity/reconcile/reconcile_parity_test.go @@ -15,12 +15,13 @@ package reconcile_parity_test import ( "encoding/json" "fmt" - "fuj-management/go/internal/domain/czech" - "fuj-management/go/internal/domain/reconcile" "math" "os" "path/filepath" "testing" + + "fuj-management/go/internal/domain/czech" + "fuj-management/go/internal/domain/reconcile" ) // --------------------------------------------------------------------------- -- 2.49.1 From 36a28a40d24d10dd207d547be2e1ccc3dc65a205 Mon Sep 17 00:00:00 2001 From: Jan Novak Date: Thu, 7 May 2026 10:33:55 +0200 Subject: [PATCH 2/4] feat(go): add --dry-run to fuj sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirror fuj infer's read-only mode: SyncOpts.DryRun skips WriteHeader, AppendValues, and SortByDateColumn, printing one "Dry run: would …" line per planned operation instead. ID-dedup still runs so the output reflects exactly what the next real sync would write. Co-Authored-By: Claude Opus 4.7 --- CHANGELOG.md | 7 ++++ .../plans/2026-05-07-1033-fuj-sync-dry-run.md | 36 +++++++++++++++++++ go/cmd/fuj/main.go | 11 ++++-- go/internal/services/banksync/sync.go | 20 +++++++++-- go/internal/services/banksync/sync_test.go | 17 +++++++++ 5 files changed, 86 insertions(+), 5 deletions(-) create mode 100644 docs/plans/2026-05-07-1033-fuj-sync-dry-run.md diff --git a/CHANGELOG.md b/CHANGELOG.md index c315de1..e47f207 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 2026-05-07 10:32 CEST — feat(go): --dry-run for fuj sync + +- `SyncOpts.DryRun bool` added; when true, `SyncToSheets` prints planned writes (`would write header row`, `would append date=… amount=… sender=…`, `would sort by date`) and returns without calling `WriteHeader`, `AppendValues`, or `SortByDateColumn`. +- `fuj sync --dry-run` flag wired in `cmd/fuj/main.go`; mirrors existing `fuj infer --dry-run` behaviour. +- `TestSyncToSheets_DryRun` added to banksync test suite. + ## 2026-05-07 01:03 CEST — feat(go/M4): IO layer behind interfaces - `go/internal/io/attendance`: CSV-over-public-URL client + `Fake` for both adult and junior tabs. @@ -36,6 +42,7 @@ - `scripts/infer_payments.py`: union adults + junior rosters so junior-only members are visible to the matcher. - Root cause: `get_members_with_fees()` reads only the adults sheet; junior-only kids like Jáchym Kubík were absent from `member_names`, causing the exact-match short-circuit to never fire and a different adult sharing the first name to win via fuzzy review. - Two regression tests added to `tests/test_match_members.py`. + ## 2026-05-06 16:05 CEST — feat(go/M2.10): port domain/reconcile.Reconcile - New `go/internal/domain/reconcile` package porting the three-phase payment allocation from `scripts/match_payments.py reconcile()`. diff --git a/docs/plans/2026-05-07-1033-fuj-sync-dry-run.md b/docs/plans/2026-05-07-1033-fuj-sync-dry-run.md new file mode 100644 index 0000000..24fc6c8 --- /dev/null +++ b/docs/plans/2026-05-07-1033-fuj-sync-dry-run.md @@ -0,0 +1,36 @@ +# Plan: add `--dry-run` to `fuj sync` + +## Context + +`fuj infer` already supports `--dry-run` (it builds the planned `BatchUpdateValues` +operations, prints them, and skips the actual write — see +`go/internal/services/banksync/infer.go:136-156` and the +`Dry run: would update N row(s).` line in `go/cmd/fuj/main.go:209-213`). + +`fuj sync` had no equivalent. It always committed three potential writes to the +payments sheet: `WriteHeader` (if the header row is missing/wrong), `AppendValues` +(for each new Fio transaction), and `SortByDateColumn` (if `--sort`, default true). +For inspecting what a sync *would* do — useful when debugging dedupe, sanity-checking +a date window, or wiring up the command for the first time on a new account — the +only options were pointing at a throwaway spreadsheet or reading the diff after the fact. + +This change mirrors `infer`'s read-only mode for `sync`: same flag name, same output +style, same "build the data structures, print instead of writing" shape. + +## Files modified + +1. `go/internal/services/banksync/sync.go` — `DryRun bool` field added to `SyncOpts`; three write points gated on `opts.DryRun` +2. `go/cmd/fuj/main.go` — `--dry-run` flag added to `syncCmd`; final println split on `*dryRun` +3. `go/internal/services/banksync/sync_test.go` — `TestSyncToSheets_DryRun` added +4. `CHANGELOG.md` — entry added + +## Behaviour + +When `--dry-run` is set: + +- If the sheet header is missing/wrong → prints `Dry run: would write header row`; skips `WriteHeader` +- For each non-deduped Fio transaction → prints `Dry run: would append date=… amount=… sender=… vs=… message=…`; skips `AppendValues` +- If `--sort` is true → prints `Dry run: would sort by date`; skips `SortByDateColumn` +- Returns `len(newRows)` so the caller can print `Dry run: would sync N new transaction(s).` + +The existing ID-dedup logic runs in full even during dry-run (reads the sheet, builds `existingIDs`), so the output reflects exactly what the next real sync would do. diff --git a/go/cmd/fuj/main.go b/go/cmd/fuj/main.go index 8c7a439..ebe8d0d 100644 --- a/go/cmd/fuj/main.go +++ b/go/cmd/fuj/main.go @@ -134,8 +134,9 @@ func syncCmd(args []string) { fromStr := fs.String("from", "", "start date YYYY-MM-DD") toStr := fs.String("to", "", "end date YYYY-MM-DD") sort := fs.Bool("sort", true, "sort sheet by date after appending") + dryRun := fs.Bool("dry-run", false, "print planned writes without modifying the sheet") fs.Usage = func() { - fmt.Fprintln(os.Stderr, "usage: fuj sync [--days N] [--from YYYY-MM-DD --to YYYY-MM-DD] [--sort]") + fmt.Fprintln(os.Stderr, "usage: fuj sync [--days N] [--from YYYY-MM-DD --to YYYY-MM-DD] [--sort] [--dry-run]") fs.PrintDefaults() } if err := fs.Parse(args); err != nil { @@ -153,7 +154,7 @@ func syncCmd(args []string) { } fioCli := fio.New(cfg.FioAPIToken, config.IBANAccountNum(cfg.BankAccount), nil) - opts := banksync.SyncOpts{Days: *days, Sort: *sort} + opts := banksync.SyncOpts{Days: *days, Sort: *sort, DryRun: *dryRun} if *fromStr != "" && *toStr != "" { opts.From, err = time.Parse("2006-01-02", *fromStr) if err != nil { @@ -172,7 +173,11 @@ func syncCmd(args []string) { fmt.Fprintf(os.Stderr, "fuj sync: %v\n", err) os.Exit(1) } - fmt.Printf("Synced %d new transaction(s).\n", n) + if *dryRun { + fmt.Printf("Dry run: would sync %d new transaction(s).\n", n) + } else { + fmt.Printf("Synced %d new transaction(s).\n", n) + } } func inferCmd(args []string) { diff --git a/go/internal/services/banksync/sync.go b/go/internal/services/banksync/sync.go index afec8a8..a48b63e 100644 --- a/go/internal/services/banksync/sync.go +++ b/go/internal/services/banksync/sync.go @@ -30,6 +30,7 @@ type SyncOpts struct { Days int // look-back window when From/To are zero From, To time.Time // explicit window (overrides Days) Sort bool // sort the sheet by Date after appending + DryRun bool // print planned writes without modifying the sheet } // SyncToSheets fetches Fio transactions and appends new ones to the payments sheet. @@ -52,8 +53,12 @@ func SyncToSheets( if len(rows) > 0 { header := rows[0] if !headerMatches(header) { - if err := sh.WriteHeader(ctx, spreadsheetID, columnLabels); err != nil { - return 0, fmt.Errorf("sync: write header: %w", err) + if opts.DryRun { + fmt.Println("Dry run: would write header row") + } else { + if err := sh.WriteHeader(ctx, spreadsheetID, columnLabels); err != nil { + return 0, fmt.Errorf("sync: write header: %w", err) + } } } else { for _, row := range rows[1:] { @@ -113,6 +118,17 @@ func SyncToSheets( return 0, nil } + if opts.DryRun { + for _, row := range newRows { + fmt.Printf("Dry run: would append date=%v amount=%v sender=%v vs=%v message=%v\n", + row[0], row[1], row[6], row[7], row[8]) + } + if opts.Sort { + fmt.Println("Dry run: would sort by date") + } + return len(newRows), nil + } + if err := sh.AppendValues(ctx, spreadsheetID, "A2", newRows); err != nil { return 0, fmt.Errorf("sync: append: %w", err) } diff --git a/go/internal/services/banksync/sync_test.go b/go/internal/services/banksync/sync_test.go index d7b1ca1..73f9b8e 100644 --- a/go/internal/services/banksync/sync_test.go +++ b/go/internal/services/banksync/sync_test.go @@ -127,6 +127,23 @@ func TestSyncToSheets_ExplicitDateWindow(t *testing.T) { } } +func TestSyncToSheets_DryRun(t *testing.T) { + sh := &sheets.Fake{Values: map[string][][]any{"SHEETID/A1:K": {}}} + fioFake := &fio.Fake{Transactions: testFioTxns} + + n, err := SyncToSheets(context.Background(), "SHEETID", fioFake, sh, + SyncOpts{Days: 30, Sort: true, DryRun: true}) + if err != nil { + t.Fatal(err) + } + if n != 2 { + t.Errorf("want 2 planned, got %d", n) + } + if len(sh.Appended) != 0 { + t.Error("dry-run must not call AppendValues") + } +} + // syncIDFor mirrors what SyncToSheets computes for a given fio.Transaction. func syncIDFor(tx fio.Transaction) string { currency := tx.Currency -- 2.49.1 From 8275db1a6391bf8b579f1f3966a1576f6112780c Mon Sep 17 00:00:00 2001 From: Jan Novak Date: Thu, 7 May 2026 10:36:20 +0200 Subject: [PATCH 3/4] fix(go/fio): nil http client panic in fio.New When token is empty, New falls back to transparentClient with the caller-supplied hc. main.go passes nil, so the first Do() call panicked. Default to http.DefaultClient when hc is nil. Co-Authored-By: Claude Opus 4.7 --- go/internal/io/fio/client.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/go/internal/io/fio/client.go b/go/internal/io/fio/client.go index 4ff3ae2..db40d54 100644 --- a/go/internal/io/fio/client.go +++ b/go/internal/io/fio/client.go @@ -4,6 +4,7 @@ package fio import ( "context" + "net/http" "time" ) @@ -29,7 +30,11 @@ type Client interface { } // New returns an apiClient when token is non-empty, otherwise a transparentClient. +// hc may be nil, in which case http.DefaultClient is used. func New(token, accountNum string, hc httpDoer) Client { + if hc == nil { + hc = http.DefaultClient + } if token != "" { return &apiClient{token: token, hc: hc} } -- 2.49.1 From fcb83691f5835aa592189012db5a4fd5c7d15d46 Mon Sep 17 00:00:00 2001 From: Jan Novak Date: Thu, 7 May 2026 10:47:54 +0200 Subject: [PATCH 4/4] fix(go/fio): nested-table early exit + non-padded date parsing extractSecondTableRows tracked a boolean inTarget flag and exited on the first
token while inside the target. Any nested (e.g. pagination markup in the real Fio page) would cause an early return before reading any data rows, explaining the 0-transaction report. Fixed by tracking targetDepth instead: depth increments on every
inside the target and we only return when it reaches 0 again. parseCzechDate also only tried zero-padded layouts ("02.01.2006"). The real Fio transparent page emits non-padded dates ("7.5.2026"); added "2.1.2006" and "2/1/2006" as the preferred layouts. Also adds a dry-run diagnostic line ("fetched N transaction(s) from Fio") so the fetch vs dedup split is visible without reading logs. Co-Authored-By: Claude Opus 4.7 --- go/internal/io/fio/fio_test.go | 18 ++++++++++++++++++ go/internal/io/fio/transparent.go | 27 ++++++++++++++++++--------- go/internal/services/banksync/sync.go | 4 ++++ 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/go/internal/io/fio/fio_test.go b/go/internal/io/fio/fio_test.go index 03774ef..7a3e38c 100644 --- a/go/internal/io/fio/fio_test.go +++ b/go/internal/io/fio/fio_test.go @@ -95,6 +95,8 @@ func TestParseCzechDate(t *testing.T) { cases := []struct{ in, want string }{ {"10.04.2026", "2026-04-10"}, {"10/04/2026", "2026-04-10"}, + {"7.5.2026", "2026-05-07"}, // non-padded — real Fio transparent page format + {"3.12.2025", "2025-12-03"}, // non-padded single-digit day, double-digit month {"", ""}, {"invalid", ""}, } @@ -105,6 +107,22 @@ func TestParseCzechDate(t *testing.T) { } } +func TestExtractSecondTableRows_NestedTable(t *testing.T) { + // Regression: a nested
inside the target must not cause early exit. + html := `
nav
+ + + + + + +
Date
7.5.2026
nested
6.5.2026
` + rows := extractSecondTableRows([]byte(html)) + if len(rows) != 2 { + t.Errorf("want 2 data rows, got %d: %v", len(rows), rows) + } +} + func TestParseCzechAmount(t *testing.T) { cases := []struct { in string diff --git a/go/internal/io/fio/transparent.go b/go/internal/io/fio/transparent.go index 9c74023..29ff7df 100644 --- a/go/internal/io/fio/transparent.go +++ b/go/internal/io/fio/transparent.go @@ -91,11 +91,13 @@ func parseTransparentHTML(body []byte) ([]Transaction, error) { // extractSecondTableRows walks the HTML token stream and returns data rows // from the second element, skipping the . +// It tracks nesting depth so that nested
elements inside the target +// do not trigger an early exit. func extractSecondTableRows(body []byte) [][]string { z := ghtml.NewTokenizer(strings.NewReader(string(body))) tableCount := 0 - inTarget := false + targetDepth := 0 // >0 while inside the target table (handles nesting) inThead := false inRow := false inCell := false @@ -113,18 +115,20 @@ func extractSecondTableRows(body []byte) [][]string { t := z.Token() switch t.Data { case "table": - if hasClass(t, "table") { + if targetDepth > 0 { + targetDepth++ // nested table inside target; track so
doesn't exit early + } else if hasClass(t, "table") { tableCount++ if tableCount == 2 { - inTarget = true + targetDepth = 1 } } case "thead": - if inTarget { + if targetDepth > 0 { inThead = true } case "tr": - if inTarget && !inThead { + if targetDepth > 0 && !inThead { inRow = true currentRow = nil } @@ -152,8 +156,11 @@ func extractSecondTableRows(body []byte) [][]string { inRow = false } case "table": - if inTarget { - return rows + if targetDepth > 0 { + targetDepth-- + if targetDepth == 0 { + return rows + } } } case ghtml.TextToken: @@ -178,11 +185,13 @@ func hasClass(t ghtml.Token, cls string) bool { return false } -// parseCzechDate parses "DD.MM.YYYY" or "DD/MM/YYYY" → "YYYY-MM-DD". +// parseCzechDate parses Czech date strings → "YYYY-MM-DD". +// Handles both zero-padded ("07.05.2026") and non-padded ("7.5.2026") variants +// with dot or slash separators, as the Fio transparent page omits leading zeros. // Returns "" on parse error. func parseCzechDate(s string) string { s = strings.TrimSpace(s) - for _, layout := range []string{"02.01.2006", "02/01/2006"} { + for _, layout := range []string{"2.1.2006", "02.01.2006", "2/1/2006", "02/01/2006"} { if t, err := time.Parse(layout, s); err == nil { return t.Format("2006-01-02") } diff --git a/go/internal/services/banksync/sync.go b/go/internal/services/banksync/sync.go index a48b63e..a036c2e 100644 --- a/go/internal/services/banksync/sync.go +++ b/go/internal/services/banksync/sync.go @@ -87,6 +87,10 @@ func SyncToSheets( if err != nil { return 0, fmt.Errorf("sync: fetch fio: %w", err) } + if opts.DryRun { + fmt.Printf("Dry run: window %s to %s, fetched %d transaction(s) from Fio\n", + from.Format("2006-01-02"), to.Format("2006-01-02"), len(txns)) + } // 4. Append new rows. var newRows [][]any -- 2.49.1