refactor: move Python implementation to legacy/, make Go primary
Deploy to K8s / deploy (push) Successful in 20s
Deploy to K8s / deploy (push) Successful in 20s
Go (go/) becomes the primary implementation. The original Python version moves to legacy/ and is kept runnable; new features are still built in both (see the Dual implementation rule in CLAUDE.md). - git mv app.py, scripts/, templates/, tests/, pyproject.toml, uv.lock and build/ into legacy/ - Makefile: Python targets point at legacy/, venv in legacy/.venv, CREDENTIALS_PATH exported for the root .secret/ file - CI and make image build the Python image with legacy/ as context; the Makefile is no longer copied into the image - legacy/.dockerignore moved to the context root so it takes effect - CLAUDE.md, docs/README.md, Go comments, fixture README, capture script and VS Code launch config updated to the new layout Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,565 @@
|
||||
"""Fixture seed registry for capture_fixtures.py.
|
||||
|
||||
Seeds are keyed by (func_name, case_id). Values are dicts whose keys
|
||||
match the fixture input schema defined in docs/plans/2026-05-06-2111-go-m3-fixture-capture.md.
|
||||
|
||||
Real-data seeds for parse_month_references and match_members are loaded
|
||||
from tmp/payments_transactions_cache.json and tmp/attendance_regular_cache.json
|
||||
at hardcoded indices selected once interactively for coverage.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper to load cache files
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
|
||||
def _load_cache(name: str) -> Any:
|
||||
path = os.path.join(_REPO, "tmp", name)
|
||||
if not os.path.exists(path):
|
||||
return None
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Handcrafted seed registry
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SEEDS: dict[tuple[str, str], dict] = {}
|
||||
|
||||
|
||||
# --- normalize ---
|
||||
|
||||
SEEDS[("normalize", "simple_ascii")] = {"text": "hello world"}
|
||||
SEEDS[("normalize", "czech_basic")] = {"text": "štefan čakrtový"}
|
||||
SEEDS[("normalize", "czech_full_set")] = {
|
||||
"text": "áčďéěíňóřšťůúýžÁČĎÉĚÍŇÓŘŠŤŮÚÝŽ"
|
||||
}
|
||||
SEEDS[("normalize", "with_parens")] = {"text": "Pavel Smutný (Štrúdl)"}
|
||||
SEEDS[("normalize", "mixed_case")] = {"text": "Henrietta OTTOVÁ"}
|
||||
SEEDS[("normalize", "empty_string")] = {"text": ""}
|
||||
SEEDS[("normalize", "digits_symbols")] = {"text": "FUJ2026! +3"}
|
||||
|
||||
|
||||
# --- parse_month_references ---
|
||||
|
||||
SEEDS[("parse_month_references", "empty_string")] = {
|
||||
"text": "", "default_year": 2026
|
||||
}
|
||||
SEEDS[("parse_month_references", "single_czech_leden")] = {
|
||||
"text": "leden", "default_year": 2026
|
||||
}
|
||||
SEEDS[("parse_month_references", "single_czech_prosinec_high_month")] = {
|
||||
"text": "prosinec", "default_year": 2026
|
||||
}
|
||||
SEEDS[("parse_month_references", "single_czech_rijen_high_month")] = {
|
||||
"text": "říjen", "default_year": 2026
|
||||
}
|
||||
SEEDS[("parse_month_references", "range_wrap_prosinec_leden")] = {
|
||||
"text": "prosinec-leden", "default_year": 2026
|
||||
}
|
||||
SEEDS[("parse_month_references", "range_wrap_listopad_leden")] = {
|
||||
"text": "listopad-leden", "default_year": 2026
|
||||
}
|
||||
SEEDS[("parse_month_references", "range_no_wrap_leden_unor")] = {
|
||||
"text": "leden-únor", "default_year": 2026
|
||||
}
|
||||
SEEDS[("parse_month_references", "numeric_slash_two_digit_year")] = {
|
||||
"text": "01/26", "default_year": 2026
|
||||
}
|
||||
SEEDS[("parse_month_references", "numeric_slash_four_digit_year")] = {
|
||||
"text": "1/2026", "default_year": 2026
|
||||
}
|
||||
SEEDS[("parse_month_references", "numeric_slash_leading_zero")] = {
|
||||
"text": "03/2026", "default_year": 2026
|
||||
}
|
||||
SEEDS[("parse_month_references", "numeric_plus_multi")] = {
|
||||
"text": "11+12/2025", "default_year": 2026
|
||||
}
|
||||
SEEDS[("parse_month_references", "numeric_dot_format")] = {
|
||||
"text": "12.2025", "default_year": 2026
|
||||
}
|
||||
SEEDS[("parse_month_references", "mixed_czech_numeric")] = {
|
||||
"text": "leden+únor+03/2026", "default_year": 2026
|
||||
}
|
||||
SEEDS[("parse_month_references", "no_month_found")] = {
|
||||
"text": "random text without months", "default_year": 2026
|
||||
}
|
||||
|
||||
|
||||
# --- calculate_fee ---
|
||||
|
||||
SEEDS[("calculate_fee", "zero_sessions")] = {
|
||||
"attendance_count": 0, "month_key": "2026-01"
|
||||
}
|
||||
SEEDS[("calculate_fee", "one_session")] = {
|
||||
"attendance_count": 1, "month_key": "2026-01"
|
||||
}
|
||||
SEEDS[("calculate_fee", "two_sessions_known_rate")] = {
|
||||
"attendance_count": 2, "month_key": "2026-01"
|
||||
}
|
||||
SEEDS[("calculate_fee", "three_sessions_known_rate")] = {
|
||||
"attendance_count": 3, "month_key": "2026-02"
|
||||
}
|
||||
SEEDS[("calculate_fee", "two_sessions_reduced_march")] = {
|
||||
"attendance_count": 2, "month_key": "2026-03"
|
||||
}
|
||||
SEEDS[("calculate_fee", "two_sessions_default_fallback")] = {
|
||||
"attendance_count": 2, "month_key": "2099-01"
|
||||
}
|
||||
|
||||
|
||||
# --- calculate_junior_fee ---
|
||||
|
||||
SEEDS[("calculate_junior_fee", "zero_sessions")] = {
|
||||
"attendance_count": 0, "month_key": "2026-01"
|
||||
}
|
||||
SEEDS[("calculate_junior_fee", "one_session_unknown")] = {
|
||||
"attendance_count": 1, "month_key": "2026-01"
|
||||
}
|
||||
SEEDS[("calculate_junior_fee", "two_sessions_default")] = {
|
||||
"attendance_count": 2, "month_key": "2026-01"
|
||||
}
|
||||
SEEDS[("calculate_junior_fee", "two_sessions_reduced_march")] = {
|
||||
"attendance_count": 2, "month_key": "2026-03"
|
||||
}
|
||||
SEEDS[("calculate_junior_fee", "two_sessions_reduced_sep")] = {
|
||||
"attendance_count": 2, "month_key": "2025-09"
|
||||
}
|
||||
SEEDS[("calculate_junior_fee", "two_sessions_default_fallback")] = {
|
||||
"attendance_count": 2, "month_key": "2099-06"
|
||||
}
|
||||
|
||||
|
||||
# --- parse_czk_amount ---
|
||||
|
||||
SEEDS[("parse_czk_amount", "none_value")] = {
|
||||
"val": {"type": "none"}
|
||||
}
|
||||
SEEDS[("parse_czk_amount", "empty_string")] = {
|
||||
"val": {"type": "string", "value": ""}
|
||||
}
|
||||
SEEDS[("parse_czk_amount", "plain_int")] = {
|
||||
"val": {"type": "int", "value": 750}
|
||||
}
|
||||
SEEDS[("parse_czk_amount", "plain_float")] = {
|
||||
"val": {"type": "float", "value": 750.0}
|
||||
}
|
||||
SEEDS[("parse_czk_amount", "czech_comma_decimal")] = {
|
||||
"val": {"type": "string", "value": "1.500,00"}
|
||||
}
|
||||
SEEDS[("parse_czk_amount", "czech_comma_no_thousands")] = {
|
||||
"val": {"type": "string", "value": "750,00"}
|
||||
}
|
||||
SEEDS[("parse_czk_amount", "dot_decimal")] = {
|
||||
"val": {"type": "string", "value": "1500.00"}
|
||||
}
|
||||
SEEDS[("parse_czk_amount", "dot_thousand_separator")] = {
|
||||
"val": {"type": "string", "value": "1.500"}
|
||||
}
|
||||
SEEDS[("parse_czk_amount", "with_kc_suffix")] = {
|
||||
"val": {"type": "string", "value": "750 Kč"}
|
||||
}
|
||||
SEEDS[("parse_czk_amount", "with_czk_suffix")] = {
|
||||
"val": {"type": "string", "value": "1500CZK"}
|
||||
}
|
||||
SEEDS[("parse_czk_amount", "space_thousands")] = {
|
||||
"val": {"type": "string", "value": "1 500"}
|
||||
}
|
||||
|
||||
|
||||
# --- generate_sync_id ---
|
||||
|
||||
def _sync_tx(date, amount, currency, sender, vs, message, bank_id):
|
||||
"""Build a generate_sync_id input seed."""
|
||||
return {
|
||||
"tx": {
|
||||
"date": date,
|
||||
"amount": amount,
|
||||
"currency": currency,
|
||||
"sender": sender,
|
||||
"vs": vs,
|
||||
"message": message,
|
||||
"bank_id": bank_id,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
SEEDS[("generate_sync_id", "typical_float_amount")] = _sync_tx(
|
||||
"2026-01-15",
|
||||
{"type": "float", "value": 750.0},
|
||||
"CZK",
|
||||
"Test Sender",
|
||||
"123456",
|
||||
"pausal leden",
|
||||
"100000001",
|
||||
)
|
||||
SEEDS[("generate_sync_id", "integer_amount")] = _sync_tx(
|
||||
"2026-01-15",
|
||||
{"type": "int", "value": 750},
|
||||
"CZK",
|
||||
"Test Sender",
|
||||
"123456",
|
||||
"pausal leden",
|
||||
"100000001",
|
||||
)
|
||||
SEEDS[("generate_sync_id", "missing_currency")] = {
|
||||
"tx": {
|
||||
"date": "2026-02-01",
|
||||
"amount": {"type": "float", "value": 500.0},
|
||||
"sender": "Another Person",
|
||||
"vs": "654321",
|
||||
"message": "trenink",
|
||||
"bank_id": "200000002",
|
||||
}
|
||||
}
|
||||
SEEDS[("generate_sync_id", "empty_fields")] = _sync_tx(
|
||||
"2026-03-01",
|
||||
{"type": "float", "value": 0.0},
|
||||
"CZK",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
)
|
||||
SEEDS[("generate_sync_id", "large_amount")] = _sync_tx(
|
||||
"2025-10-05",
|
||||
{"type": "float", "value": 2100.0},
|
||||
"CZK",
|
||||
"Payer Name",
|
||||
"987654",
|
||||
"FUJ treninky",
|
||||
"300000003",
|
||||
)
|
||||
|
||||
|
||||
# --- build_name_variants ---
|
||||
|
||||
SEEDS[("build_name_variants", "full_name_no_nick")] = {
|
||||
"full_name": "Jan Novák"
|
||||
}
|
||||
SEEDS[("build_name_variants", "with_nickname")] = {
|
||||
"full_name": "František Vrbík (Štrúdl)"
|
||||
}
|
||||
SEEDS[("build_name_variants", "three_word_name")] = {
|
||||
"full_name": "Jan Tomášek (Honza)"
|
||||
}
|
||||
SEEDS[("build_name_variants", "single_word")] = {
|
||||
"full_name": "Jáchym"
|
||||
}
|
||||
SEEDS[("build_name_variants", "short_name_filtered")] = {
|
||||
"full_name": "Jo"
|
||||
}
|
||||
SEEDS[("build_name_variants", "common_diacritics")] = {
|
||||
"full_name": "Alžběta Testovická"
|
||||
}
|
||||
|
||||
|
||||
# --- match_members ---
|
||||
|
||||
# Synthetic roster — deliberately NOT real member names.
|
||||
# Tomáš Fiktivný has a nickname (Tov) for nickname-match tests.
|
||||
# Pavel Smutný has a nickname (Štrúdl) for nickname tests.
|
||||
# Adam Novák: normalized last name "novak" is in _COMMON_SURNAMES → common-surname filter test.
|
||||
_ROSTER = [
|
||||
"Alžběta Testovická",
|
||||
"Tomáš Fiktivný (Tov)",
|
||||
"Pavel Smutný (Štrúdl)",
|
||||
"Jana Nováková",
|
||||
"Adam Novák",
|
||||
]
|
||||
|
||||
SEEDS[("match_members", "exact_full_name")] = {
|
||||
"text": "platba od alzbeta testovicka leden",
|
||||
"member_names": _ROSTER,
|
||||
}
|
||||
SEEDS[("match_members", "first_and_last")] = {
|
||||
"text": "jan nový payment tomas fiktivny",
|
||||
"member_names": _ROSTER,
|
||||
}
|
||||
SEEDS[("match_members", "nickname_match")] = {
|
||||
"text": "payment from strudl",
|
||||
"member_names": _ROSTER,
|
||||
}
|
||||
SEEDS[("match_members", "review_lastname_only")] = {
|
||||
"text": "testovicka leden",
|
||||
"member_names": _ROSTER,
|
||||
}
|
||||
SEEDS[("match_members", "common_surname_no_match")] = {
|
||||
"text": "novak leden",
|
||||
"member_names": _ROSTER,
|
||||
}
|
||||
SEEDS[("match_members", "no_match")] = {
|
||||
"text": "xyz platba",
|
||||
"member_names": _ROSTER,
|
||||
}
|
||||
SEEDS[("match_members", "two_members_exact")] = {
|
||||
"text": "pavel smutny a alzbeta testovicka",
|
||||
"member_names": _ROSTER,
|
||||
}
|
||||
|
||||
|
||||
# --- infer_transaction_details ---
|
||||
|
||||
SEEDS[("infer_transaction_details", "member_in_message")] = {
|
||||
"tx": {
|
||||
"sender": "Test Payer",
|
||||
"message": "alzbeta testovicka leden 2026",
|
||||
"user_id": "",
|
||||
"date": {"type": "string", "value": "2026-01-15"},
|
||||
},
|
||||
"member_names": _ROSTER,
|
||||
"default_year": 2026,
|
||||
}
|
||||
SEEDS[("infer_transaction_details", "member_in_sender")] = {
|
||||
"tx": {
|
||||
"sender": "Tomáš Fiktivný",
|
||||
"message": "FUJ trenink",
|
||||
"user_id": "",
|
||||
"date": {"type": "string", "value": "2026-02-01"},
|
||||
},
|
||||
"member_names": _ROSTER,
|
||||
"default_year": 2026,
|
||||
}
|
||||
SEEDS[("infer_transaction_details", "month_fallback_from_date")] = {
|
||||
"tx": {
|
||||
"sender": "Alžběta Testovická",
|
||||
"message": "platba",
|
||||
"user_id": "",
|
||||
"date": {"type": "string", "value": "2026-03-15"},
|
||||
},
|
||||
"member_names": _ROSTER,
|
||||
"default_year": 2026,
|
||||
}
|
||||
SEEDS[("infer_transaction_details", "serial_date")] = {
|
||||
"tx": {
|
||||
"sender": "Jana Nováková",
|
||||
"message": "leden",
|
||||
"user_id": "",
|
||||
"date": {"type": "float", "value": 46027.0}, # 2026-01-15 in Sheets serial
|
||||
},
|
||||
"member_names": _ROSTER,
|
||||
"default_year": 2026,
|
||||
}
|
||||
SEEDS[("infer_transaction_details", "no_member_no_month")] = {
|
||||
"tx": {
|
||||
"sender": "Unknown Person",
|
||||
"message": "random text",
|
||||
"user_id": "",
|
||||
"date": {"type": "none"},
|
||||
},
|
||||
"member_names": _ROSTER,
|
||||
"default_year": 2026,
|
||||
}
|
||||
|
||||
|
||||
# --- format_date ---
|
||||
|
||||
SEEDS[("format_date", "string_iso")] = {"val": {"type": "string", "value": "2026-01-15"}}
|
||||
SEEDS[("format_date", "string_non_iso")] = {"val": {"type": "string", "value": "garbage"}}
|
||||
SEEDS[("format_date", "empty_string")] = {"val": {"type": "string", "value": ""}}
|
||||
SEEDS[("format_date", "none_value")] = {"val": {"type": "none"}}
|
||||
SEEDS[("format_date", "serial_int")] = {"val": {"type": "int", "value": 46027}}
|
||||
SEEDS[("format_date", "serial_float")] = {"val": {"type": "float", "value": 46027.5}}
|
||||
SEEDS[("format_date", "serial_float_exact")] = {"val": {"type": "float", "value": 45957.0}} # 2025-10-01
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Reconcile handcrafted seeds
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _tx(date, amount, person, purpose, sender="Payer", message="", bank_id="", inferred_amount=None):
|
||||
return {
|
||||
"date": date,
|
||||
"amount": amount,
|
||||
"manual_fix": "",
|
||||
"person": person,
|
||||
"purpose": purpose,
|
||||
"inferred_amount": inferred_amount if inferred_amount is not None else amount,
|
||||
"sender": sender,
|
||||
"message": message,
|
||||
"bank_id": bank_id,
|
||||
}
|
||||
|
||||
|
||||
def _member(name, tier, fees: dict):
|
||||
"""fees: {month: (fee, count) or int}. Returns a dict so the scrubber
|
||||
can find the 'name' key and apply deterministic pseudonymisation."""
|
||||
return {"name": name, "tier": tier, "fees": fees}
|
||||
|
||||
|
||||
def _reconcile_seed(members, sorted_months, transactions, exceptions=None, default_year=2026):
|
||||
return {
|
||||
"members": members,
|
||||
"sorted_months": sorted_months,
|
||||
"transactions": transactions,
|
||||
"exceptions": exceptions or [],
|
||||
"default_year": default_year,
|
||||
}
|
||||
|
||||
|
||||
# 01 — greedy exact: Alice pays exactly 750, expected 750
|
||||
SEEDS[("reconcile", "01_greedy_exact")] = _reconcile_seed(
|
||||
members=[_member("Alice Dvořák", "A", {"2026-01": (750, 3)})],
|
||||
sorted_months=["2026-01"],
|
||||
transactions=[_tx("2026-01-20", 750, "Alice Dvořák", "2026-01", sender="Alice Dvořák")],
|
||||
)
|
||||
|
||||
# 02 — greedy overpayment → credit: Alice pays 900, expected 750
|
||||
SEEDS[("reconcile", "02_greedy_overpayment")] = _reconcile_seed(
|
||||
members=[_member("Alice Dvořák", "A", {"2026-01": (750, 3)})],
|
||||
sorted_months=["2026-01"],
|
||||
transactions=[_tx("2026-01-20", 900, "Alice Dvořák", "2026-01", sender="Alice Dvořák")],
|
||||
)
|
||||
|
||||
# 03 — proportional: Alice pays 800 for 3 months (750+750+350=1850 expected)
|
||||
SEEDS[("reconcile", "03_proportional_remainder")] = _reconcile_seed(
|
||||
members=[_member("Alice Dvořák", "A", {
|
||||
"2026-01": (750, 3),
|
||||
"2026-02": (750, 2),
|
||||
"2026-03": (350, 2),
|
||||
})],
|
||||
sorted_months=["2026-01", "2026-02", "2026-03"],
|
||||
transactions=[_tx("2026-03-10", 800, "Alice Dvořák", "2026-01,2026-02,2026-03", sender="Alice Dvořák")],
|
||||
)
|
||||
|
||||
# 04 — even-split: all expected=0, payment spread evenly
|
||||
SEEDS[("reconcile", "04_even_split_prepayment")] = _reconcile_seed(
|
||||
members=[_member("Bob Kratochvíl", "A", {
|
||||
"2026-04": (0, 0),
|
||||
"2026-05": (0, 0),
|
||||
})],
|
||||
sorted_months=["2026-04", "2026-05"],
|
||||
transactions=[_tx("2026-03-25", 700, "Bob Kratochvíl", "2026-04,2026-05", sender="Bob Kratochvíl")],
|
||||
)
|
||||
|
||||
# 05 — out-of-window: payment references 2025-08 which is outside sorted_months
|
||||
SEEDS[("reconcile", "05_out_of_window_credit")] = _reconcile_seed(
|
||||
members=[_member("Alice Dvořák", "A", {"2026-01": (750, 3)})],
|
||||
sorted_months=["2026-01"],
|
||||
transactions=[_tx("2026-01-20", 1500, "Alice Dvořák", "2026-01,2025-08", sender="Alice Dvořák")],
|
||||
)
|
||||
|
||||
# 06 — exception override: Alice's 2026-01 fee overridden from 750 to 300
|
||||
SEEDS[("reconcile", "06_exception_override")] = _reconcile_seed(
|
||||
members=[_member("Alice Dvořák", "A", {"2026-01": (750, 3)})],
|
||||
sorted_months=["2026-01"],
|
||||
transactions=[_tx("2026-01-20", 300, "Alice Dvořák", "2026-01", sender="Alice Dvořák")],
|
||||
# exceptions as list of [name, period, amount, note] (capture_fixtures converts to dict)
|
||||
exceptions=[{"name": "Alice Dvořák", "period": "2026-01", "amount": 300, "note": "injury discount"}],
|
||||
)
|
||||
|
||||
# 07 — other purpose: tournament fee split between Alice and Bob
|
||||
SEEDS[("reconcile", "07_other_purpose_split")] = _reconcile_seed(
|
||||
members=[
|
||||
_member("Alice Dvořák", "A", {"2026-01": (750, 3)}),
|
||||
_member("Bob Kratochvíl", "A", {"2026-01": (750, 2)}),
|
||||
],
|
||||
sorted_months=["2026-01"],
|
||||
transactions=[_tx("2026-01-10", 800, "Alice Dvořák, Bob Kratochvíl", "other:tournament", sender="Alice Dvořák")],
|
||||
)
|
||||
|
||||
# 08 — junior with attendance=1 (expected=0 in reconcile, unknown in UI)
|
||||
SEEDS[("reconcile", "08_junior_question_mark")] = _reconcile_seed(
|
||||
members=[_member("Karel Junior", "A", {"2026-01": (0, 1)})],
|
||||
sorted_months=["2026-01"],
|
||||
transactions=[_tx("2026-01-20", 200, "Karel Junior", "2026-01", sender="Karel Junior")],
|
||||
)
|
||||
|
||||
# 09 — multi-person comma-split: Alice and Bob share a payment for 2 months
|
||||
SEEDS[("reconcile", "09_multiperson_multimonth")] = _reconcile_seed(
|
||||
members=[
|
||||
_member("Alice Dvořák", "A", {"2026-01": (750, 3), "2026-02": (750, 2)}),
|
||||
_member("Bob Kratochvíl", "A", {"2026-01": (750, 2), "2026-02": (350, 2)}),
|
||||
],
|
||||
sorted_months=["2026-01", "2026-02"],
|
||||
transactions=[_tx("2026-02-15", 2000, "Alice Dvořák, Bob Kratochvíl", "2026-01,2026-02", sender="Alice Dvořák")],
|
||||
)
|
||||
|
||||
# 10 — unmatched: no person, garbage message
|
||||
SEEDS[("reconcile", "10_unmatched")] = _reconcile_seed(
|
||||
members=[_member("Alice Dvořák", "A", {"2026-01": (750, 3)})],
|
||||
sorted_months=["2026-01"],
|
||||
transactions=[_tx("2026-01-20", 500, "", "", sender="Unknown Payer", message="garbage xyz 999")],
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Real-data seeds
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Indices into tmp/payments_transactions_cache.json['data'] selected for coverage.
|
||||
# DO NOT change these — they are deliberately frozen to make re-runs deterministic.
|
||||
_REAL_PMR_INDICES = [
|
||||
(16, "real_single_leden"),
|
||||
(17, "real_range_prosinec_leden"),
|
||||
(18, "real_list_prosinec_leden_unor"),
|
||||
(22, "real_martin_prosinec_leden"),
|
||||
(23, "real_range_listopad_leden"),
|
||||
(25, "real_filip_prosinec_leden_unor"),
|
||||
(36, "real_mixed_czech_numeric"),
|
||||
(42, "real_dominika_numeric_multi"),
|
||||
# index 67 removed: the name-sweep scrubber changes the text prefix in a way
|
||||
# that breaks the numeric-slash parser (empty result vs expected "2026-03").
|
||||
(72, "real_tomik_numeric_plus"),
|
||||
(73, "real_franc_numeric_space"),
|
||||
(74, "real_jana_numeric_multi"),
|
||||
(80, "real_alex_numeric_long"),
|
||||
(89, "real_emily_numeric_long"),
|
||||
(90, "real_jachym_numeric_multi"),
|
||||
]
|
||||
|
||||
# Real match_members seeds are intentionally omitted: after PII scrubbing
|
||||
# the member_names pseudonyms are inconsistent with the (un-scrubbed) text,
|
||||
# causing all Go parity assertions to fail. The synthetic seeds below cover
|
||||
# the same code paths without any real data.
|
||||
_REAL_MM_INDICES: list = []
|
||||
|
||||
|
||||
def real_parse_month_references_seeds(default_year: int = 2026):
|
||||
"""Yield (case_id, seed) from real cache messages."""
|
||||
cache = _load_cache("payments_transactions_cache.json")
|
||||
if cache is None:
|
||||
return
|
||||
txs = cache.get("data", [])
|
||||
for idx, case_id in _REAL_PMR_INDICES:
|
||||
if idx >= len(txs):
|
||||
continue
|
||||
msg = str(txs[idx].get("message", ""))
|
||||
yield case_id, {"text": msg, "default_year": default_year}
|
||||
|
||||
|
||||
def _real_member_names():
|
||||
"""Return canonical member names from the regular attendance cache."""
|
||||
cache = _load_cache("attendance_regular_cache.json")
|
||||
if cache is None:
|
||||
return []
|
||||
rows = cache.get("data", [])
|
||||
if rows and isinstance(rows[0], list):
|
||||
rows = rows[0]
|
||||
return [row[0] for row in rows if isinstance(row, (list, tuple)) and len(row) >= 2]
|
||||
|
||||
|
||||
def real_match_members_seeds():
|
||||
"""Yield (case_id, seed) using real senders/messages against real roster."""
|
||||
cache = _load_cache("payments_transactions_cache.json")
|
||||
member_names = _real_member_names()
|
||||
if cache is None or not member_names:
|
||||
return
|
||||
txs = cache.get("data", [])
|
||||
for idx, case_id in _REAL_MM_INDICES:
|
||||
if idx >= len(txs):
|
||||
continue
|
||||
tx = txs[idx]
|
||||
sender = str(tx.get("sender", ""))
|
||||
message = str(tx.get("message", ""))
|
||||
text = f"{sender} {message}"
|
||||
yield case_id, {"text": text, "member_names": member_names}
|
||||
@@ -0,0 +1,291 @@
|
||||
"""Shared attendance/fee logic for FUJ Tuesday practices."""
|
||||
|
||||
import csv
|
||||
import io
|
||||
import urllib.request
|
||||
from datetime import datetime
|
||||
|
||||
from config import ATTENDANCE_SHEET_ID as SHEET_ID, JUNIOR_SHEET_GID
|
||||
|
||||
EXPORT_URL = f"https://docs.google.com/spreadsheets/d/{SHEET_ID}/export?format=csv&gid=0"
|
||||
JUNIOR_EXPORT_URL = f"https://docs.google.com/spreadsheets/d/{SHEET_ID}/export?format=csv&gid={JUNIOR_SHEET_GID}"
|
||||
|
||||
ADULT_FEE_DEFAULT = 700 # CZK, for 2+ practices in a month
|
||||
ADULT_FEE_SINGLE = 200 # CZK, for exactly 1 practice in a month
|
||||
ADULT_FEE_MONTHLY_RATE = {
|
||||
"2025-09": 750,
|
||||
"2025-10": 750,
|
||||
"2025-11": 750,
|
||||
"2025-12": 750,
|
||||
"2026-01": 750,
|
||||
"2026-02": 750,
|
||||
"2026-03": 350,
|
||||
"2026-04": 700,
|
||||
"2026-05": 450,
|
||||
"2026-06": 600,
|
||||
"2026-07": 600,
|
||||
"2026-08": 600,
|
||||
}
|
||||
|
||||
JUNIOR_FEE_DEFAULT = 500 # CZK for 2+ practices
|
||||
JUNIOR_MONTHLY_RATE = {
|
||||
"2025-09": 250,
|
||||
"2026-03": 250 # reduced fee for March 2026
|
||||
}
|
||||
ADULT_MERGED_MONTHS = {
|
||||
"2025-09": "2025-10", # keys are merged into values
|
||||
}
|
||||
|
||||
JUNIOR_MERGED_MONTHS = {
|
||||
"2025-12": "2026-01", # keys are merged into values
|
||||
"2025-09": "2025-10"
|
||||
}
|
||||
|
||||
COL_NAME = 0
|
||||
COL_TIER = 1
|
||||
FIRST_DATE_COL = 3
|
||||
|
||||
|
||||
def fetch_csv(url: str = EXPORT_URL) -> list[list[str]]:
|
||||
"""Fetch the attendance Google Sheet as parsed CSV rows."""
|
||||
req = urllib.request.Request(url)
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
text = resp.read().decode("utf-8")
|
||||
reader = csv.reader(io.StringIO(text))
|
||||
return list(reader)
|
||||
|
||||
|
||||
def parse_dates(header_row: list[str]) -> list[tuple[int, datetime]]:
|
||||
"""Return (column_index, date) pairs for all date columns."""
|
||||
dates = []
|
||||
|
||||
for i in range(FIRST_DATE_COL, len(header_row)):
|
||||
raw = header_row[i].strip()
|
||||
if not raw:
|
||||
continue
|
||||
|
||||
try:
|
||||
# Try DD.MM.YYYY
|
||||
dt = datetime.strptime(raw, "%d.%m.%Y")
|
||||
dates.append((i, dt))
|
||||
except ValueError:
|
||||
try:
|
||||
# Fallback to MM/DD/YYYY
|
||||
dt = datetime.strptime(raw, "%m/%d/%Y")
|
||||
dates.append((i, dt))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return dates
|
||||
|
||||
|
||||
def group_by_month(dates: list[tuple[int, datetime]], merged_months: dict[str, str]) -> dict[str, list[int]]:
|
||||
"""Group column indices by YYYY-MM, handling merged months."""
|
||||
months: dict[str, list[int]] = {}
|
||||
for col, dt in dates:
|
||||
key = dt.strftime("%Y-%m")
|
||||
# Apply merged month mapping if configured
|
||||
target_key = merged_months.get(key, key)
|
||||
months.setdefault(target_key, []).append(col)
|
||||
return months
|
||||
|
||||
|
||||
def calculate_fee(attendance_count: int, month_key: str) -> int:
|
||||
"""Apply fee rules: 0 → 0, 1 → 200, 2+ → configured rate (default 750)."""
|
||||
if attendance_count == 0:
|
||||
return 0
|
||||
if attendance_count == 1:
|
||||
return ADULT_FEE_SINGLE
|
||||
return ADULT_FEE_MONTHLY_RATE.get(month_key, ADULT_FEE_DEFAULT)
|
||||
|
||||
|
||||
def calculate_junior_fee(attendance_count: int, month_key: str) -> str | int:
|
||||
"""Apply junior fee rules: 0 → 0, 1 → '?', 2+ → Configured Rate (default 500)."""
|
||||
if attendance_count == 0:
|
||||
return 0
|
||||
if attendance_count == 1:
|
||||
return "?"
|
||||
return JUNIOR_MONTHLY_RATE.get(month_key, JUNIOR_FEE_DEFAULT)
|
||||
|
||||
|
||||
def get_members(rows: list[list[str]]) -> list[tuple[str, str, list[str]]]:
|
||||
"""Parse member rows. Returns list of (name, tier, row).
|
||||
|
||||
Stopped at row where first column contains '# last line'.
|
||||
Skips rows starting with '#'.
|
||||
"""
|
||||
members = []
|
||||
for row in rows[1:]:
|
||||
if not row or len(row) <= COL_NAME:
|
||||
continue
|
||||
|
||||
first_col = row[COL_NAME].strip()
|
||||
|
||||
# Terminator for rows to process
|
||||
if "# last line" in first_col.lower():
|
||||
break
|
||||
|
||||
# Ignore comments
|
||||
if first_col.startswith("#"):
|
||||
continue
|
||||
|
||||
if not first_col or first_col.lower() in ("jméno", "name", "jmeno"):
|
||||
continue
|
||||
|
||||
tier = row[COL_TIER].strip().upper() if len(row) > COL_TIER else ""
|
||||
members.append((first_col, tier, row))
|
||||
return members
|
||||
|
||||
|
||||
def get_junior_members_from_sheet(rows: list[list[str]]) -> list[tuple[str, str, list[str]]]:
|
||||
"""Parse junior member rows from the junior sheet.
|
||||
|
||||
Stopped at row where first column contains '# Treneri'.
|
||||
Returns list of (name, tier, row) for members where tier is not 'X'.
|
||||
"""
|
||||
members = []
|
||||
for row in rows[1:]:
|
||||
if not row or len(row) <= COL_NAME:
|
||||
continue
|
||||
|
||||
first_col = row[COL_NAME].strip()
|
||||
|
||||
# Terminator for rows to process in junior sheet
|
||||
if "# treneri" in first_col.lower() or "# trenéři" in first_col.lower():
|
||||
break
|
||||
|
||||
# Ignore comments
|
||||
if first_col.startswith("#"):
|
||||
continue
|
||||
|
||||
if not first_col or first_col.lower() in ("jméno", "name", "jmeno"):
|
||||
continue
|
||||
|
||||
tier = row[COL_TIER].strip().upper() if len(row) > COL_TIER else ""
|
||||
if tier == "X":
|
||||
continue
|
||||
|
||||
members.append((first_col, tier, row))
|
||||
return members
|
||||
|
||||
|
||||
def get_members_with_fees() -> tuple[list[tuple[str, str, dict[str, int]]], list[str]]:
|
||||
"""Fetch attendance data and compute fees.
|
||||
|
||||
Returns:
|
||||
(members, sorted_months) where members is a list of
|
||||
(name, tier, {month_key: (fee, count)}) for ALL members (all tiers).
|
||||
sorted_months is the list of YYYY-MM keys in order.
|
||||
"""
|
||||
rows = fetch_csv()
|
||||
if len(rows) < 2:
|
||||
return [], []
|
||||
|
||||
header_row = rows[0]
|
||||
dates = parse_dates(header_row)
|
||||
if not dates:
|
||||
return [], []
|
||||
|
||||
months = group_by_month(dates, ADULT_MERGED_MONTHS)
|
||||
sorted_months = sorted(months.keys())
|
||||
members_raw = get_members(rows)
|
||||
|
||||
members = []
|
||||
for name, tier, row in members_raw:
|
||||
month_fees = {}
|
||||
for month_key in sorted_months:
|
||||
cols = months[month_key]
|
||||
count = sum(
|
||||
1
|
||||
for c in cols
|
||||
if c < len(row) and row[c].strip().upper() == "TRUE"
|
||||
)
|
||||
fee = calculate_fee(count, month_key) if tier == "A" else 0
|
||||
month_fees[month_key] = (fee, count)
|
||||
members.append((name, tier, month_fees))
|
||||
|
||||
return members, sorted_months
|
||||
|
||||
|
||||
def get_junior_members_with_fees() -> tuple[list[tuple[str, str, dict[str, tuple[str | int, int, int, int]]]], list[str]]:
|
||||
"""Fetch attendance data from both sheets and compute junior fees.
|
||||
|
||||
Merges members by exact name match.
|
||||
|
||||
Returns:
|
||||
(members, sorted_months) where members is a list of
|
||||
(name, tier, {month_key: (fee, total_count, adult_count, junior_count)}).
|
||||
"""
|
||||
main_rows = fetch_csv(EXPORT_URL)
|
||||
junior_rows = fetch_csv(JUNIOR_EXPORT_URL)
|
||||
|
||||
if len(main_rows) < 2 or len(junior_rows) < 2:
|
||||
return [], []
|
||||
|
||||
main_dates = parse_dates(main_rows[0])
|
||||
junior_dates = parse_dates(junior_rows[0])
|
||||
|
||||
main_months = group_by_month(main_dates, JUNIOR_MERGED_MONTHS)
|
||||
junior_months = group_by_month(junior_dates, JUNIOR_MERGED_MONTHS)
|
||||
|
||||
# Collect all unique sorted months
|
||||
all_months = set(main_months.keys()).union(set(junior_months.keys()))
|
||||
sorted_months = sorted(list(all_months))
|
||||
|
||||
from typing import Any
|
||||
merged_members: dict[str, Any] = {}
|
||||
|
||||
# Process Junior Tier from Main Sheet (Adult Practices)
|
||||
main_members_raw = get_members(main_rows)
|
||||
for name, tier, row in main_members_raw:
|
||||
if tier != "J":
|
||||
continue
|
||||
|
||||
if name not in merged_members:
|
||||
merged_members[name] = {"tier": tier, "months": {}}
|
||||
|
||||
for month_key in sorted_months:
|
||||
if month_key not in merged_members[name]["months"]:
|
||||
merged_members[name]["months"][month_key] = {"adult": 0, "junior": 0}
|
||||
|
||||
cols = main_months.get(month_key, [])
|
||||
adult_count = sum(
|
||||
1
|
||||
for c in cols
|
||||
if c < len(row) and row[c].strip().upper() == "TRUE"
|
||||
)
|
||||
merged_members[name]["months"][month_key]["adult"] += adult_count
|
||||
|
||||
# Process Junior Sheet (Junior Practices)
|
||||
junior_members_raw = get_junior_members_from_sheet(junior_rows)
|
||||
for name, tier, row in junior_members_raw:
|
||||
if name not in merged_members:
|
||||
merged_members[name] = {"tier": tier, "months": {}}
|
||||
|
||||
for month_key in sorted_months:
|
||||
if month_key not in merged_members[name]["months"]:
|
||||
merged_members[name]["months"][month_key] = {"adult": 0, "junior": 0}
|
||||
|
||||
cols = junior_months.get(month_key, [])
|
||||
junior_count = sum(
|
||||
1
|
||||
for c in cols
|
||||
if c < len(row) and row[c].strip().upper() == "TRUE"
|
||||
)
|
||||
merged_members[name]["months"][month_key]["junior"] += junior_count
|
||||
|
||||
# Compile the final result format
|
||||
members = []
|
||||
for name, data in merged_members.items():
|
||||
month_fees = {}
|
||||
for month_key in sorted_months:
|
||||
adult_count = data["months"].get(month_key, {}).get("adult", 0)
|
||||
junior_count = data["months"].get(month_key, {}).get("junior", 0)
|
||||
total_count = adult_count + junior_count
|
||||
|
||||
fee = calculate_junior_fee(total_count, month_key)
|
||||
month_fees[month_key] = (fee, total_count, adult_count, junior_count)
|
||||
|
||||
members.append((name, data["tier"], month_fees))
|
||||
|
||||
return members, sorted_months
|
||||
@@ -0,0 +1,173 @@
|
||||
import json
|
||||
import socket
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from google.oauth2 import service_account
|
||||
from googleapiclient.discovery import build
|
||||
|
||||
from config import (
|
||||
CACHE_DIR, CREDENTIALS_PATH as CREDS_PATH, DRIVE_TIMEOUT,
|
||||
CACHE_TTL_SECONDS, CACHE_API_CHECK_TTL_SECONDS, CACHE_SHEET_MAP,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Global state to track last Drive API check time per sheet
|
||||
_LAST_CHECKED = {}
|
||||
_DRIVE_SERVICE = None
|
||||
|
||||
def _get_drive_service():
|
||||
global _DRIVE_SERVICE
|
||||
if _DRIVE_SERVICE is not None:
|
||||
return _DRIVE_SERVICE
|
||||
|
||||
if not CREDS_PATH.exists():
|
||||
logger.warning(f"Credentials not found at {CREDS_PATH}. Cannot check Google Drive API.")
|
||||
return None
|
||||
|
||||
try:
|
||||
creds = service_account.Credentials.from_service_account_file(
|
||||
str(CREDS_PATH),
|
||||
scopes=["https://www.googleapis.com/auth/drive.readonly"]
|
||||
)
|
||||
|
||||
# Apply timeout safely to the httplib2 connection without mutating global socket
|
||||
import httplib2
|
||||
import google_auth_httplib2
|
||||
http = httplib2.Http(timeout=DRIVE_TIMEOUT)
|
||||
http = google_auth_httplib2.AuthorizedHttp(creds, http=http)
|
||||
|
||||
_DRIVE_SERVICE = build("drive", "v3", http=http, cache_discovery=False)
|
||||
return _DRIVE_SERVICE
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to build Drive API service: {e}")
|
||||
return None
|
||||
|
||||
import time
|
||||
|
||||
def get_sheet_modified_time(cache_key: str) -> str | None:
|
||||
"""Gets the modifiedTime from Google Drive API for a given cache_key.
|
||||
Returns the ISO timestamp string if successful.
|
||||
If the Drive API fails (e.g., lack of permissions for public sheets),
|
||||
it generates a virtual time bucket string to provide a 5-minute TTL cache.
|
||||
"""
|
||||
sheet_id = CACHE_SHEET_MAP.get(cache_key, cache_key)
|
||||
|
||||
cache_file = CACHE_DIR / f"{cache_key}_cache.json"
|
||||
|
||||
# 1. Check if we should skip the Drive API check entirely (global memory TTL)
|
||||
now = time.time()
|
||||
last_check = _LAST_CHECKED.get(sheet_id, 0)
|
||||
|
||||
if CACHE_API_CHECK_TTL_SECONDS > 0 and (now - last_check) < CACHE_API_CHECK_TTL_SECONDS:
|
||||
# We checked recently. Return cached modifiedTime if cache file exists.
|
||||
if cache_file.exists():
|
||||
try:
|
||||
with open(cache_file, "r", encoding="utf-8") as f:
|
||||
cache_data = json.load(f)
|
||||
cached_time = cache_data.get("modifiedTime")
|
||||
if cached_time:
|
||||
logger.info(f"Skipping Drive API check for {sheet_id} due to {CACHE_API_CHECK_TTL_SECONDS}s API check TTL")
|
||||
return cached_time
|
||||
except Exception as e:
|
||||
logger.warning(f"Error reading existing cache during API skip for {sheet_id}: {e}")
|
||||
|
||||
# 2. Check if the cache file is simply too new (legacy check)
|
||||
if CACHE_TTL_SECONDS > 0 and cache_file.exists():
|
||||
try:
|
||||
file_mtime = cache_file.stat().st_mtime
|
||||
if time.time() - file_mtime < CACHE_TTL_SECONDS:
|
||||
with open(cache_file, "r", encoding="utf-8") as f:
|
||||
cache_data = json.load(f)
|
||||
cached_time = cache_data.get("modifiedTime")
|
||||
if cached_time:
|
||||
logger.info(f"Skipping Drive API check for {sheet_id} due to {CACHE_TTL_SECONDS}s max CACHE_TTL")
|
||||
# We consider this a valid check, update the global state
|
||||
_LAST_CHECKED[sheet_id] = now
|
||||
return cached_time
|
||||
except Exception as e:
|
||||
logger.warning(f"Error checking cache TTL for {sheet_id}: {e}")
|
||||
|
||||
def _fallback_ttl():
|
||||
bucket = int(time.time() // 300)
|
||||
return f"ttl-5m-{bucket}"
|
||||
|
||||
logger.info(f"Checking Drive API for {sheet_id}")
|
||||
drive_service = _get_drive_service()
|
||||
if not drive_service:
|
||||
return _fallback_ttl()
|
||||
|
||||
try:
|
||||
file_meta = drive_service.files().get(fileId=sheet_id, fields="modifiedTime", supportsAllDrives=True).execute()
|
||||
# Successfully checked API, update the global state
|
||||
_LAST_CHECKED[sheet_id] = time.time()
|
||||
return file_meta.get("modifiedTime")
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not get modifiedTime for sheet {sheet_id}: {e}. Falling back to 5-minute TTL.")
|
||||
return _fallback_ttl()
|
||||
|
||||
def read_cache(sheet_id: str, current_modified_time: str) -> list | dict | None:
|
||||
"""Reads the JSON cache for the given sheet_id.
|
||||
Returns the cached data if it exists AND the cached modifiedTime matches
|
||||
current_modified_time.
|
||||
Otherwise, returns None.
|
||||
"""
|
||||
if not current_modified_time:
|
||||
return None
|
||||
|
||||
cache_file = CACHE_DIR / f"{sheet_id}_cache.json"
|
||||
if not cache_file.exists():
|
||||
return None
|
||||
|
||||
try:
|
||||
with open(cache_file, "r", encoding="utf-8") as f:
|
||||
cache_data = json.load(f)
|
||||
|
||||
cached_time = cache_data.get("modifiedTime")
|
||||
if cached_time == current_modified_time:
|
||||
logger.info(f"Cache hit for {sheet_id} ({current_modified_time})")
|
||||
return cache_data.get("data")
|
||||
else:
|
||||
logger.info(f"Cache miss for {sheet_id}. Cached: {cached_time}, Current: {current_modified_time}")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to read cache {cache_file}: {e}")
|
||||
return None
|
||||
|
||||
def write_cache(sheet_id: str, modified_time: str, data: list | dict) -> None:
|
||||
"""Writes the data to a JSON cache file with the given modified_time."""
|
||||
if not modified_time:
|
||||
return
|
||||
|
||||
try:
|
||||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
cache_file = CACHE_DIR / f"{sheet_id}_cache.json"
|
||||
|
||||
cache_data = {
|
||||
"modifiedTime": modified_time,
|
||||
"data": data,
|
||||
"cachedAt": datetime.now().isoformat()
|
||||
}
|
||||
|
||||
with open(cache_file, "w", encoding="utf-8") as f:
|
||||
json.dump(cache_data, f, ensure_ascii=False)
|
||||
|
||||
logger.info(f"Wrote cache for {sheet_id}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to write cache {sheet_id}: {e}")
|
||||
|
||||
def flush_cache():
|
||||
"""Delete all cache files and reset in-memory state. Returns count of deleted files."""
|
||||
global _DRIVE_SERVICE
|
||||
_LAST_CHECKED.clear()
|
||||
_DRIVE_SERVICE = None
|
||||
|
||||
deleted = 0
|
||||
if CACHE_DIR.exists():
|
||||
for f in CACHE_DIR.glob("*_cache.json"):
|
||||
f.unlink()
|
||||
deleted += 1
|
||||
logger.info(f"Deleted cache file: {f.name}")
|
||||
|
||||
logger.info(f"Cache flushed: {deleted} files deleted, timers reset")
|
||||
return deleted
|
||||
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Calculate monthly fees from the FUJ Tuesday practice attendance sheet."""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from attendance import get_members_with_fees
|
||||
|
||||
|
||||
def main():
|
||||
members, sorted_months = get_members_with_fees()
|
||||
if not members:
|
||||
print("No data.")
|
||||
return
|
||||
|
||||
# Filter to adults only for display
|
||||
results = [(name, fees) for name, tier, fees in members if tier == "A"]
|
||||
|
||||
# Format month labels
|
||||
month_labels = {
|
||||
m: datetime.strptime(m, "%Y-%m").strftime("%b %Y") for m in sorted_months
|
||||
}
|
||||
|
||||
# Print table
|
||||
name_width = max((len(r[0]) for r in results), default=20)
|
||||
col_width = 15
|
||||
|
||||
header = f"{'Member':<{name_width}}"
|
||||
for m in sorted_months:
|
||||
header += f" | {month_labels[m]:>{col_width}}"
|
||||
print(header)
|
||||
print("-" * len(header))
|
||||
|
||||
monthly_totals = {m: 0 for m in sorted_months}
|
||||
for name, month_fees in results:
|
||||
line = f"{name:<{name_width}}"
|
||||
for m in sorted_months:
|
||||
fee, count = month_fees[m]
|
||||
monthly_totals[m] += fee
|
||||
cell = f"{fee} CZK ({count})" if count > 0 else "-"
|
||||
line += f" | {cell:>{col_width}}"
|
||||
print(line)
|
||||
|
||||
# Totals row
|
||||
print("-" * len(header))
|
||||
totals_line = f"{'TOTAL':<{name_width}}"
|
||||
for m in sorted_months:
|
||||
cell = f"{monthly_totals[m]} CZK"
|
||||
totals_line += f" | {cell:>{col_width}}"
|
||||
print(totals_line)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+47
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env bash
|
||||
# Regenerate the full fixture corpus.
|
||||
# Safe to re-run — always overwrites.
|
||||
# Requires: legacy/tmp/*_cache.json present (for real-data seeds for parse_month_references and match_members).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
LEGACY="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
REPO="$(cd "$LEGACY/.." && pwd)"
|
||||
FIXTURES="$REPO/go/tests/fixtures"
|
||||
CAPTURE_CMD="PYTHONPATH=$LEGACY/scripts:$LEGACY python3 $LEGACY/scripts/capture_fixtures.py"
|
||||
SCRUB_CMD="python3 $LEGACY/scripts/scrub_fixtures.py"
|
||||
|
||||
run_func() {
|
||||
local func="$1"
|
||||
local dir="$FIXTURES/pure/$func"
|
||||
mkdir -p "$dir"
|
||||
echo " Capturing $func..."
|
||||
eval "$CAPTURE_CMD --func $func --all" | while IFS= read -r line; do
|
||||
case_id="$(python3 -c "import sys,json; print(json.loads('''$line''')['case'])" 2>/dev/null || \
|
||||
echo "$line" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['case'])")"
|
||||
echo "$line" | python3 "$LEGACY/scripts/scrub_fixtures.py" > "$dir/${case_id}.json"
|
||||
done
|
||||
}
|
||||
|
||||
echo "==> Capturing pure-function fixtures..."
|
||||
|
||||
run_func normalize
|
||||
run_func parse_month_references
|
||||
run_func calculate_fee
|
||||
run_func calculate_junior_fee
|
||||
run_func parse_czk_amount
|
||||
run_func generate_sync_id
|
||||
run_func build_name_variants
|
||||
run_func match_members
|
||||
run_func infer_transaction_details
|
||||
run_func format_date
|
||||
|
||||
echo "==> Capturing reconcile fixtures..."
|
||||
mkdir -p "$FIXTURES/reconcile"
|
||||
eval "$CAPTURE_CMD --func reconcile --all" | while IFS= read -r line; do
|
||||
case_id="$(echo "$line" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['case'])")"
|
||||
echo "$line" | python3 "$LEGACY/scripts/scrub_fixtures.py" > "$FIXTURES/reconcile/${case_id}.json"
|
||||
done
|
||||
|
||||
echo "==> Done. Review with: git diff go/tests/fixtures/"
|
||||
echo "==> Audit PII: git ls-files go/tests/fixtures | xargs grep -l '<real name>' should return zero."
|
||||
@@ -0,0 +1,353 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Capture pure-function output as JSON fixtures for parity testing.
|
||||
|
||||
Each invocation emits exactly one JSON object to stdout.
|
||||
Pipe through scrub_fixtures.py before writing to go/tests/fixtures/.
|
||||
|
||||
Usage:
|
||||
# Single case:
|
||||
python capture_fixtures.py --func normalize --case simple_ascii \\
|
||||
--input-seed simple_ascii | python scrub_fixtures.py \\
|
||||
> go/tests/fixtures/pure/normalize/simple_ascii.json
|
||||
|
||||
# All seeds for a function (newline-delimited JSON, one object per line):
|
||||
python capture_fixtures.py --func normalize --all
|
||||
|
||||
# Feed input from stdin (for ad-hoc cases):
|
||||
echo '{"text":"hello"}' | python capture_fixtures.py --func normalize \\
|
||||
--case adhoc --input-stdin
|
||||
|
||||
See scripts/_fixture_seeds.py for the seed registry.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import os
|
||||
import datetime
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from czech_utils import normalize, parse_month_references
|
||||
from attendance import calculate_fee, calculate_junior_fee
|
||||
from infer_payments import parse_czk_amount
|
||||
from sync_fio_to_sheets import generate_sync_id as _py_generate_sync_id
|
||||
from match_payments import (
|
||||
_build_name_variants,
|
||||
match_members,
|
||||
infer_transaction_details,
|
||||
format_date,
|
||||
reconcile,
|
||||
)
|
||||
from czech_utils import normalize as _norm
|
||||
|
||||
import _fixture_seeds as seeds
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Type-envelope helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _decode_envelope(envelope):
|
||||
"""Convert a {type, value} envelope to a Python value for function calls."""
|
||||
if not isinstance(envelope, dict):
|
||||
return envelope
|
||||
t = envelope.get("type", "raw")
|
||||
v = envelope.get("value")
|
||||
if t == "none":
|
||||
return None
|
||||
if t == "int":
|
||||
return int(v)
|
||||
if t == "float":
|
||||
return float(v)
|
||||
if t == "string":
|
||||
return v
|
||||
return v # raw JSON value (for fields that don't use an envelope)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-function capture implementations
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def capture_normalize(inp: dict) -> dict:
|
||||
result = normalize(inp["text"])
|
||||
return {"text": result}
|
||||
|
||||
|
||||
def capture_parse_month_references(inp: dict) -> dict:
|
||||
result = parse_month_references(inp["text"], inp.get("default_year", 2026))
|
||||
return {"months": result}
|
||||
|
||||
|
||||
def capture_calculate_fee(inp: dict) -> dict:
|
||||
result = calculate_fee(inp["attendance_count"], inp["month_key"])
|
||||
return {"fee": result}
|
||||
|
||||
|
||||
def capture_calculate_junior_fee(inp: dict) -> dict:
|
||||
raw = calculate_junior_fee(inp["attendance_count"], inp["month_key"])
|
||||
if raw == "?":
|
||||
return {"value": 0, "unknown": True}
|
||||
return {"value": int(raw), "unknown": False}
|
||||
|
||||
|
||||
def capture_parse_czk_amount(inp: dict) -> dict:
|
||||
val = _decode_envelope(inp["val"])
|
||||
result = parse_czk_amount(val)
|
||||
return {"amount": float(result)}
|
||||
|
||||
|
||||
def capture_generate_sync_id(inp: dict) -> dict:
|
||||
tx_in = inp["tx"]
|
||||
# Build the tx dict that generate_sync_id expects:
|
||||
# amount must be the Python-native type to replicate str(amount) faithfully.
|
||||
tx = {k: v for k, v in tx_in.items() if k != "amount"}
|
||||
tx["amount"] = _decode_envelope(tx_in["amount"])
|
||||
result = _py_generate_sync_id(tx)
|
||||
return {"sync_id": result}
|
||||
|
||||
|
||||
def capture_build_name_variants(inp: dict) -> dict:
|
||||
result = _build_name_variants(inp["full_name"])
|
||||
return {"variants": result}
|
||||
|
||||
|
||||
def capture_match_members(inp: dict) -> dict:
|
||||
matches = match_members(inp["text"], inp["member_names"])
|
||||
return {
|
||||
"matches": [{"name": name, "confidence": conf} for name, conf in matches]
|
||||
}
|
||||
|
||||
|
||||
def capture_infer_transaction_details(inp: dict) -> dict:
|
||||
tx_in = inp["tx"]
|
||||
tx = dict(tx_in)
|
||||
tx["date"] = _decode_envelope(tx_in.get("date"))
|
||||
result = infer_transaction_details(tx, inp["member_names"])
|
||||
return {
|
||||
"matches": [{"name": n, "confidence": c} for n, c in result["members"]],
|
||||
"months": result["months"],
|
||||
"search_text": result.get("search_text", result.get("matched_text", "")),
|
||||
}
|
||||
|
||||
|
||||
def capture_format_date(inp: dict) -> dict:
|
||||
val = _decode_envelope(inp["val"])
|
||||
result = format_date(val)
|
||||
return {"date": result}
|
||||
|
||||
|
||||
def _build_exceptions(exc_list):
|
||||
"""Convert seed exceptions to the dict reconcile() expects.
|
||||
Accepts both the legacy list format [name, period, amount, note] and the
|
||||
new dict format {"name": ..., "period": ..., "amount": ..., "note": ...}."""
|
||||
if not exc_list:
|
||||
return {}
|
||||
result = {}
|
||||
for row in exc_list:
|
||||
if isinstance(row, dict):
|
||||
name = row.get("name", "")
|
||||
period = row.get("period", "")
|
||||
amount = row.get("amount", 0)
|
||||
note = row.get("note", "")
|
||||
else:
|
||||
name, period, amount = row[0], row[1], row[2]
|
||||
note = row[3] if len(row) > 3 else ""
|
||||
result[(_norm(name), _norm(period))] = {"amount": int(amount), "note": note}
|
||||
return result
|
||||
|
||||
|
||||
def _member_fee_dict(fees_raw: dict) -> dict:
|
||||
"""Convert seed fees dict to the form reconcile() expects."""
|
||||
# Seeds store fees as [fee, count] lists (JSON) or (fee, count) tuples.
|
||||
result = {}
|
||||
for month, v in fees_raw.items():
|
||||
if isinstance(v, (list, tuple)) and len(v) == 2:
|
||||
result[month] = (int(v[0]), int(v[1]))
|
||||
else:
|
||||
result[month] = int(v)
|
||||
return result
|
||||
|
||||
|
||||
def _tx_entry_out(tx):
|
||||
"""Convert a reconcile output TxEntry dict to a serializable form."""
|
||||
return {
|
||||
"amount": float(tx.get("amount", 0)),
|
||||
"date": tx.get("date", ""),
|
||||
"sender": tx.get("sender", ""),
|
||||
"message": tx.get("message", ""),
|
||||
"confidence": tx.get("confidence", ""),
|
||||
}
|
||||
|
||||
|
||||
def _other_entry_out(e):
|
||||
return {
|
||||
"amount": float(e.get("amount", 0)),
|
||||
"date": e.get("date", ""),
|
||||
"sender": e.get("sender", ""),
|
||||
"message": e.get("message", ""),
|
||||
"purpose": e.get("purpose", ""),
|
||||
"confidence": e.get("confidence", ""),
|
||||
}
|
||||
|
||||
|
||||
def _month_data_out(md):
|
||||
return {
|
||||
"expected": int(md["expected"]) if isinstance(md["expected"], (int, float)) else 0,
|
||||
"original_expected": int(md["original_expected"]) if isinstance(md.get("original_expected"), (int, float)) else 0,
|
||||
"attendance_count": int(md.get("attendance_count", 0)),
|
||||
"exception": md.get("exception"),
|
||||
"paid": float(md["paid"]),
|
||||
"transactions": [_tx_entry_out(t) for t in md.get("transactions", [])],
|
||||
}
|
||||
|
||||
|
||||
def _unmatched_tx_out(tx):
|
||||
return {
|
||||
"date": tx.get("date", ""),
|
||||
"amount": float(tx.get("amount", 0)),
|
||||
"person": tx.get("person", ""),
|
||||
"purpose": tx.get("purpose", ""),
|
||||
"sender": tx.get("sender", ""),
|
||||
"message": tx.get("message", ""),
|
||||
"bank_id": tx.get("bank_id", ""),
|
||||
}
|
||||
|
||||
|
||||
def capture_reconcile(inp: dict) -> dict:
|
||||
# Convert members from seed format to reconcile() format.
|
||||
# Accepts both the new dict format {"name":..., "tier":..., "fees":{...}}
|
||||
# and the legacy tuple format [name, tier, fees_dict].
|
||||
members_in = inp["members"]
|
||||
members = []
|
||||
for m in members_in:
|
||||
if isinstance(m, dict):
|
||||
name, tier, fees_raw = m["name"], m["tier"], m.get("fees", {})
|
||||
else:
|
||||
name, tier, fees_raw = m[0], m[1], m[2]
|
||||
members.append((name, tier, _member_fee_dict(fees_raw)))
|
||||
|
||||
exceptions = _build_exceptions(inp.get("exceptions") or [])
|
||||
sorted_months = inp["sorted_months"]
|
||||
transactions = inp["transactions"]
|
||||
|
||||
result = reconcile(members, sorted_months, transactions, exceptions)
|
||||
|
||||
members_out = {}
|
||||
for name, mr in result["members"].items():
|
||||
members_out[name] = {
|
||||
"tier": mr["tier"],
|
||||
"months": {m: _month_data_out(md) for m, md in mr["months"].items()},
|
||||
"other_transactions": [_other_entry_out(e) for e in mr.get("other_transactions", [])],
|
||||
"total_balance": int(mr["total_balance"]),
|
||||
}
|
||||
|
||||
return {
|
||||
"members": members_out,
|
||||
"unmatched": [_unmatched_tx_out(tx) for tx in result["unmatched"]],
|
||||
"credits": {k: int(v) for k, v in result["credits"].items()},
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dispatcher
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_DISPATCHERS = {
|
||||
"normalize": capture_normalize,
|
||||
"parse_month_references": capture_parse_month_references,
|
||||
"calculate_fee": capture_calculate_fee,
|
||||
"calculate_junior_fee": capture_calculate_junior_fee,
|
||||
"parse_czk_amount": capture_parse_czk_amount,
|
||||
"generate_sync_id": capture_generate_sync_id,
|
||||
"build_name_variants": capture_build_name_variants,
|
||||
"match_members": capture_match_members,
|
||||
"infer_transaction_details": capture_infer_transaction_details,
|
||||
"format_date": capture_format_date,
|
||||
"reconcile": capture_reconcile,
|
||||
}
|
||||
|
||||
_FUNC_MODULE = {
|
||||
"normalize": "scripts.czech_utils.normalize",
|
||||
"parse_month_references": "scripts.czech_utils.parse_month_references",
|
||||
"calculate_fee": "scripts.attendance.calculate_fee",
|
||||
"calculate_junior_fee": "scripts.attendance.calculate_junior_fee",
|
||||
"parse_czk_amount": "scripts.infer_payments.parse_czk_amount",
|
||||
"generate_sync_id": "scripts.sync_fio_to_sheets.generate_sync_id",
|
||||
"build_name_variants": "scripts.match_payments._build_name_variants",
|
||||
"match_members": "scripts.match_payments.match_members",
|
||||
"infer_transaction_details": "scripts.match_payments.infer_transaction_details",
|
||||
"format_date": "scripts.match_payments.format_date",
|
||||
"reconcile": "scripts.match_payments.reconcile",
|
||||
}
|
||||
|
||||
|
||||
def _emit(func_name: str, case_id: str, inp: dict) -> None:
|
||||
dispatch = _DISPATCHERS[func_name]
|
||||
output = dispatch(inp)
|
||||
doc = {
|
||||
"case": case_id,
|
||||
"func": _FUNC_MODULE[func_name],
|
||||
"captured_at": datetime.date.today().isoformat(),
|
||||
"input": inp,
|
||||
"output": output,
|
||||
}
|
||||
print(json.dumps(doc, ensure_ascii=False))
|
||||
|
||||
|
||||
def _all_seeds(func_name: str):
|
||||
"""Yield (case_id, seed) for all seeds of a function."""
|
||||
for (fn, case_id), seed in seeds.SEEDS.items():
|
||||
if fn == func_name:
|
||||
yield case_id, seed
|
||||
|
||||
# Real-data seeds
|
||||
if func_name == "parse_month_references":
|
||||
yield from seeds.real_parse_month_references_seeds()
|
||||
if func_name == "match_members":
|
||||
yield from seeds.real_match_members_seeds()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Capture pure-function outputs as JSON fixtures."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--func", required=True, choices=list(_DISPATCHERS), help="Function to capture."
|
||||
)
|
||||
group = parser.add_mutually_exclusive_group(required=True)
|
||||
group.add_argument("--case", help="Case ID (file stem). Use with --input-seed or --input-stdin.")
|
||||
group.add_argument("--all", action="store_true", help="Emit all seeds for the function.")
|
||||
parser.add_argument(
|
||||
"--input-seed", metavar="SEED_ID",
|
||||
help="Seed key in _fixture_seeds.SEEDS (required unless --input-stdin or --all).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--input-stdin", action="store_true",
|
||||
help="Read input JSON from stdin instead of seed registry.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.all:
|
||||
for case_id, seed in _all_seeds(args.func):
|
||||
_emit(args.func, case_id, seed)
|
||||
return
|
||||
|
||||
# Single case
|
||||
if args.input_stdin:
|
||||
inp = json.load(sys.stdin)
|
||||
elif args.input_seed:
|
||||
key = (args.func, args.input_seed)
|
||||
if key not in seeds.SEEDS:
|
||||
sys.exit(f"Seed ({args.func!r}, {args.input_seed!r}) not found in _fixture_seeds.SEEDS")
|
||||
inp = seeds.SEEDS[key]
|
||||
else:
|
||||
parser.error("Provide --input-seed SEED_ID or --input-stdin.")
|
||||
|
||||
_emit(args.func, args.case, inp)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Centralized configuration for FUJ management scripts.
|
||||
|
||||
External service IDs, credentials, and tunable parameters.
|
||||
Domain-specific constants (fees, column indices) stay in their respective modules.
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
# Paths
|
||||
PROJECT_ROOT = Path(__file__).parent.parent
|
||||
CREDENTIALS_PATH = Path(os.environ.get(
|
||||
"CREDENTIALS_PATH",
|
||||
str(PROJECT_ROOT / ".secret" / "fuj-management-bot-credentials.json"),
|
||||
))
|
||||
|
||||
# Google Sheets IDs
|
||||
ATTENDANCE_SHEET_ID = "1E2e_gT_K5AwSRCDLDTa2UetZTkHmBOcz0kFbBUNUNBA"
|
||||
PAYMENTS_SHEET_ID = "1Om0YPoDVCH5cV8BrNz5LG5eR5MMU05ypQC7UMN1xn_Y"
|
||||
|
||||
# Attendance sheet tab GIDs
|
||||
JUNIOR_SHEET_GID = "1213318614"
|
||||
|
||||
# Bank accounts — hardcoded list mirroring go/internal/config/config.go.
|
||||
# The entry with primary=True is used for QR codes and as the BANK_ACCOUNT default.
|
||||
ACCOUNTS = [
|
||||
{"iban": "CZ0820100000002502035405", "acct_num": "2502035405", "token_env": "FIO_API_TOKEN_NEW", "primary": True},
|
||||
{"iban": "CZ8520100000002800359168", "acct_num": "2800359168", "token_env": "FIO_API_TOKEN_OLD", "primary": False},
|
||||
]
|
||||
# Resolve API tokens from the environment once at import time.
|
||||
LOADED_ACCOUNTS = [
|
||||
{**a, "token": os.environ.get(a["token_env"], "")}
|
||||
for a in ACCOUNTS
|
||||
]
|
||||
BANK_ACCOUNT = next(a["iban"] for a in ACCOUNTS if a["primary"])
|
||||
|
||||
# Cache settings
|
||||
CACHE_DIR = PROJECT_ROOT / "tmp"
|
||||
DRIVE_TIMEOUT = 10 # seconds
|
||||
CACHE_TTL_SECONDS = int(os.environ.get("CACHE_TTL_SECONDS", 300)) # 5 min default
|
||||
CACHE_API_CHECK_TTL_SECONDS = int(os.environ.get("CACHE_API_CHECK_TTL_SECONDS", 300)) # 5 min default
|
||||
|
||||
# Display settings
|
||||
MONTHS_TO_SHOW = int(os.environ.get("MONTHS_TO_SHOW", 5)) # show last N months; 0 = show all
|
||||
|
||||
# Maps cache keys to their source sheet IDs (used by cache_utils)
|
||||
CACHE_SHEET_MAP = {
|
||||
"attendance_regular": ATTENDANCE_SHEET_ID,
|
||||
"attendance_juniors": ATTENDANCE_SHEET_ID,
|
||||
"exceptions_dict": PAYMENTS_SHEET_ID,
|
||||
"payments_transactions": PAYMENTS_SHEET_ID,
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
"""Czech text utilities — diacritics normalization and month parsing."""
|
||||
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
CZECH_MONTHS = {
|
||||
"leden": 1, "ledna": 1, "lednu": 1,
|
||||
"unor": 2, "unora": 2, "unoru": 2,
|
||||
"brezen": 3, "brezna": 3, "breznu": 3,
|
||||
"duben": 4, "dubna": 4, "dubnu": 4,
|
||||
"kveten": 5, "kvetna": 5, "kvetnu": 5,
|
||||
"cerven": 6, "cervna": 6, "cervnu": 6,
|
||||
"cervenec": 7, "cervnce": 7, "cervenci": 7,
|
||||
"srpen": 8, "srpna": 8, "srpnu": 8,
|
||||
"zari": 9,
|
||||
"rijen": 10, "rijna": 10, "rijnu": 10,
|
||||
"listopad": 11, "listopadu": 11,
|
||||
"prosinec": 12, "prosince": 12, "prosinci": 12,
|
||||
}
|
||||
|
||||
|
||||
def normalize(text: str) -> str:
|
||||
"""Strip diacritics and lowercase."""
|
||||
nfkd = unicodedata.normalize("NFKD", text)
|
||||
return "".join(c for c in nfkd if not unicodedata.combining(c)).lower()
|
||||
|
||||
|
||||
def parse_month_references(text: str, default_year: int = 2026) -> list[str]:
|
||||
"""Extract YYYY-MM month references from Czech free text.
|
||||
|
||||
Handles:
|
||||
- Czech month names: "leden", "únor", "prosinec" (all declensions)
|
||||
- Numeric: "01/26", "1/2026", "11+12/2025"
|
||||
- Ranges: "listopad-leden" (November through January)
|
||||
- Slash-separated numeric months: "11+12/2025"
|
||||
|
||||
Returns sorted list of unique YYYY-MM strings.
|
||||
"""
|
||||
normalized = normalize(text)
|
||||
results: set[str] = set()
|
||||
|
||||
# Pattern: numeric months with year, e.g. "11+12/2025", "01/26", "1/2026"
|
||||
# Match groups of digits separated by + before a /year
|
||||
numeric_pattern = re.findall(
|
||||
r"([\d+]+)\s*/\s*(\d{2,4})", normalized
|
||||
)
|
||||
for months_part, year_str in numeric_pattern:
|
||||
year = int(year_str)
|
||||
if year < 100:
|
||||
year += 2000
|
||||
for m_str in months_part.split("+"):
|
||||
m_str = m_str.strip()
|
||||
if m_str.isdigit():
|
||||
m = int(m_str)
|
||||
if 1 <= m <= 12:
|
||||
results.add(f"{year:04d}-{m:02d}")
|
||||
|
||||
# Pattern: standalone numeric month.year, e.g. "12.2025"
|
||||
dot_pattern = re.findall(r"(\d{1,2})\s*\.\s*(\d{4})", normalized)
|
||||
for m_str, year_str in dot_pattern:
|
||||
m, year = int(m_str), int(year_str)
|
||||
if 1 <= m <= 12:
|
||||
results.add(f"{year:04d}-{m:02d}")
|
||||
|
||||
# Czech month names — handle ranges like "listopad-leden"
|
||||
# First, find range patterns
|
||||
month_name_re = "|".join(sorted(CZECH_MONTHS.keys(), key=len, reverse=True))
|
||||
range_pattern = re.findall(
|
||||
rf"({month_name_re})\s*-\s*({month_name_re})", normalized
|
||||
)
|
||||
found_in_ranges: set[str] = set()
|
||||
for start_name, end_name in range_pattern:
|
||||
found_in_ranges.add(start_name)
|
||||
found_in_ranges.add(end_name)
|
||||
start_m = CZECH_MONTHS[start_name]
|
||||
end_m = CZECH_MONTHS[end_name]
|
||||
# Walk from start to end, wrapping around December→January
|
||||
m = start_m
|
||||
while True:
|
||||
year = default_year if m >= start_m and start_m > end_m else default_year
|
||||
# If range wraps (e.g. Nov-Jan), months >= start are previous year
|
||||
if start_m > end_m and m >= start_m:
|
||||
year = default_year - 1
|
||||
results.add(f"{year:04d}-{m:02d}")
|
||||
if m == end_m:
|
||||
break
|
||||
m = m % 12 + 1
|
||||
|
||||
# Individual Czech month names (not already part of a range)
|
||||
for match in re.finditer(rf"\b({month_name_re})\b", normalized):
|
||||
name = match.group(1)
|
||||
if name in found_in_ranges:
|
||||
continue
|
||||
m = CZECH_MONTHS[name]
|
||||
# Heuristic: if month > 9 and we're early in the year, it's likely previous year
|
||||
year = default_year
|
||||
if m >= 10:
|
||||
year = default_year - 1
|
||||
results.add(f"{year:04d}-{m:02d}")
|
||||
|
||||
return sorted(results)
|
||||
@@ -0,0 +1,252 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shared Fio bank fetching utilities."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.request
|
||||
from datetime import datetime
|
||||
from html.parser import HTMLParser
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Transaction fetching
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class FioTableParser(HTMLParser):
|
||||
"""Parse the second <table class="table"> on the Fio transparent page.
|
||||
|
||||
Columns: Datum | Částka | Typ | Název protiúčtu | Zpráva pro příjemce | KS | VS | SS | Poznámka
|
||||
Indices: 0 1 2 3 4 5 6 7 8
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._table_count = 0
|
||||
self._in_target_table = False
|
||||
self._in_thead = False
|
||||
self._in_row = False
|
||||
self._in_cell = False
|
||||
self._current_row: list[str] = []
|
||||
self._rows: list[list[str]] = []
|
||||
self._cell_text = ""
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
cls = dict(attrs).get("class", "")
|
||||
if tag == "table" and "table" in cls.split():
|
||||
self._table_count += 1
|
||||
if self._table_count == 2:
|
||||
self._in_target_table = True
|
||||
if self._in_target_table:
|
||||
if tag == "thead":
|
||||
self._in_thead = True
|
||||
if tag == "tr" and not self._in_thead:
|
||||
self._in_row = True
|
||||
self._current_row = []
|
||||
if self._in_row and tag in ("td", "th"):
|
||||
self._in_cell = True
|
||||
self._cell_text = ""
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if self._in_cell and tag in ("td", "th"):
|
||||
self._in_cell = False
|
||||
self._current_row.append(self._cell_text.strip())
|
||||
if tag == "thead":
|
||||
self._in_thead = False
|
||||
if self._in_row and tag == "tr":
|
||||
self._in_row = False
|
||||
if self._current_row:
|
||||
self._rows.append(self._current_row)
|
||||
if tag == "table" and self._in_target_table:
|
||||
self._in_target_table = False
|
||||
|
||||
def handle_data(self, data):
|
||||
if self._in_cell:
|
||||
self._cell_text += data
|
||||
|
||||
def get_rows(self) -> list[list[str]]:
|
||||
return self._rows
|
||||
|
||||
|
||||
# Fio transparent table column indices
|
||||
_COL_DATE = 0
|
||||
_COL_AMOUNT = 1
|
||||
_COL_SENDER = 3
|
||||
_COL_MESSAGE = 4
|
||||
_COL_KS = 5
|
||||
_COL_VS = 6
|
||||
_COL_SS = 7
|
||||
_COL_NOTE = 8
|
||||
|
||||
|
||||
def parse_czech_amount(s: str) -> float | None:
|
||||
"""Parse '1 500,00 CZK' to float."""
|
||||
s = s.replace("\xa0", "").replace(" ", "").replace(",", ".")
|
||||
s = re.sub(r"[A-Za-z]+", "", s).strip()
|
||||
try:
|
||||
return float(s)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def parse_czech_date(s: str) -> str | None:
|
||||
"""Parse a Czech date to 'YYYY-MM-DD'. Accepts 4-digit and 2-digit years
|
||||
with dot or slash separators; Fio's transparent page mixes 'DD.MM.YYYY'
|
||||
and 'DD.MM.YY' in the same response."""
|
||||
s = s.strip()
|
||||
for fmt in ("%d.%m.%Y", "%d/%m/%Y", "%d.%m.%y", "%d/%m/%y"):
|
||||
try:
|
||||
return datetime.strptime(s, fmt).strftime("%Y-%m-%d")
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def fetch_transactions_transparent(
|
||||
date_from: str, date_to: str, account_id: str
|
||||
) -> list[dict]:
|
||||
"""Fetch transactions from Fio transparent account HTML page.
|
||||
|
||||
Args:
|
||||
date_from: D.M.YYYY format
|
||||
date_to: D.M.YYYY format
|
||||
"""
|
||||
url = (
|
||||
f"https://ib.fio.cz/ib/transparent?a={account_id}"
|
||||
f"&f={date_from}&t={date_to}"
|
||||
)
|
||||
req = urllib.request.Request(url)
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
html = resp.read().decode("utf-8")
|
||||
|
||||
parser = FioTableParser()
|
||||
parser.feed(html)
|
||||
rows = parser.get_rows()
|
||||
|
||||
transactions = []
|
||||
for row in rows:
|
||||
if len(row) < 5:
|
||||
continue
|
||||
|
||||
def col(i):
|
||||
return row[i].strip() if i < len(row) else ""
|
||||
|
||||
date_str = parse_czech_date(col(_COL_DATE))
|
||||
amount = parse_czech_amount(col(_COL_AMOUNT))
|
||||
|
||||
if date_str is None or amount is None or amount <= 0:
|
||||
continue
|
||||
|
||||
transactions.append({
|
||||
"date": date_str,
|
||||
"amount": amount,
|
||||
"sender": col(_COL_SENDER),
|
||||
"message": col(_COL_MESSAGE),
|
||||
"ks": col(_COL_KS),
|
||||
"vs": col(_COL_VS),
|
||||
"ss": col(_COL_SS),
|
||||
"note": col(_COL_NOTE),
|
||||
"bank_id": "", # HTML scraping doesn't give stable ID
|
||||
})
|
||||
|
||||
print(f"fio: transparent fetched {len(rows)} raw rows, {len(transactions)} transaction(s) after filtering", file=sys.stderr)
|
||||
return transactions
|
||||
|
||||
|
||||
def fetch_transactions_api(
|
||||
token: str, date_from: str, date_to: str
|
||||
) -> list[dict]:
|
||||
"""Fetch transactions via Fio REST API (JSON).
|
||||
|
||||
Args:
|
||||
token: Fio API token
|
||||
date_from: YYYY-MM-DD format
|
||||
date_to: YYYY-MM-DD format
|
||||
"""
|
||||
url = (
|
||||
f"https://fioapi.fio.cz/v1/rest/periods/{token}"
|
||||
f"/{date_from}/{date_to}/transactions.json"
|
||||
)
|
||||
req = urllib.request.Request(url)
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
|
||||
transactions = []
|
||||
tx_list = data.get("accountStatement", {}).get("transactionList", {})
|
||||
raw_list = tx_list.get("transaction") or []
|
||||
for tx in raw_list:
|
||||
# Each field is {"value": ..., "name": ..., "id": ...} or null
|
||||
def val(col_id):
|
||||
col = tx.get(f"column{col_id}")
|
||||
return col["value"] if col else ""
|
||||
|
||||
amount = float(val(1) or 0)
|
||||
if amount <= 0:
|
||||
continue # Skip outgoing
|
||||
|
||||
date_raw = val(0) or ""
|
||||
# API returns date as "YYYY-MM-DD+HHMM" or ISO format
|
||||
date_str = date_raw[:10] if date_raw else ""
|
||||
|
||||
transactions.append({
|
||||
"date": date_str,
|
||||
"amount": amount,
|
||||
"sender": str(val(10) or ""), # column10 = sender name
|
||||
"message": str(val(16) or ""), # column16 = message for recipient
|
||||
"vs": str(val(5) or ""), # column5 = VS
|
||||
"ks": str(val(4) or ""), # column4 = KS
|
||||
"ss": str(val(6) or ""), # column6 = SS
|
||||
"user_id": str(val(7) or ""), # column7 = user identification
|
||||
"sender_account": str(val(2) or ""), # column2 = sender account
|
||||
"bank_id": str(val(22) or ""), # column22 = ID operace
|
||||
"currency": str(val(14) or "CZK"), # column14 = Currency
|
||||
})
|
||||
|
||||
print(f"fio: api fetched {len(raw_list)} raw transaction(s), {len(transactions)} after filtering", file=sys.stderr)
|
||||
return transactions
|
||||
|
||||
|
||||
def fetch_transactions_for(account: dict, date_from: str, date_to: str) -> list[dict]:
|
||||
"""Fetch transactions for a single loaded account dict (from config.LOADED_ACCOUNTS).
|
||||
|
||||
Uses the API path if the account has a token, otherwise the transparent scraper.
|
||||
date_from/date_to: YYYY-MM-DD.
|
||||
"""
|
||||
token = (account.get("token") or "").strip()
|
||||
acct_num = account["acct_num"]
|
||||
if token:
|
||||
print(f"fio: account {acct_num}: using authenticated API, window {date_from}..{date_to}", file=sys.stderr)
|
||||
return fetch_transactions_api(token, date_from, date_to)
|
||||
|
||||
print(
|
||||
f"fio: account {acct_num}: using transparent page (no token — expect publishing lag), "
|
||||
f"window {date_from}..{date_to}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
from_dt = datetime.strptime(date_from, "%Y-%m-%d")
|
||||
to_dt = datetime.strptime(date_to, "%Y-%m-%d")
|
||||
return fetch_transactions_transparent(
|
||||
from_dt.strftime("%d.%m.%Y"),
|
||||
to_dt.strftime("%d.%m.%Y"),
|
||||
account_id=acct_num,
|
||||
)
|
||||
|
||||
|
||||
def fetch_transactions_all(
|
||||
date_from: str, date_to: str, accounts: list[dict] | None = None
|
||||
) -> list[dict]:
|
||||
"""Fetch and combine transactions from all configured accounts.
|
||||
|
||||
accounts: list of loaded account dicts (defaults to config.LOADED_ACCOUNTS).
|
||||
Returns a flat list of all transactions across all accounts.
|
||||
"""
|
||||
if accounts is None:
|
||||
from config import LOADED_ACCOUNTS
|
||||
accounts = LOADED_ACCOUNTS
|
||||
all_txns: list[dict] = []
|
||||
for account in accounts:
|
||||
txns = fetch_transactions_for(account, date_from, date_to)
|
||||
print(f"fio: account {account['acct_num']}: {len(txns)} transaction(s)", file=sys.stderr)
|
||||
all_txns.extend(txns)
|
||||
print(f"fio: total {len(all_txns)} transaction(s) across {len(accounts)} account(s)", file=sys.stderr)
|
||||
return all_txns
|
||||
@@ -0,0 +1,200 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Infer 'Person', 'Purpose', and 'Amount' for transactions in Google Sheets."""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
# Add the current directory to sys.path to import local modules
|
||||
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from googleapiclient.discovery import build
|
||||
from sync_fio_to_sheets import get_sheets_service, DEFAULT_SPREADSHEET_ID
|
||||
from match_payments import infer_transaction_details, canonical_member_key
|
||||
from attendance import get_members_with_fees, get_junior_members_with_fees
|
||||
|
||||
def parse_czk_amount(val) -> float:
|
||||
"""Parse Czech currency string or handle raw numeric value."""
|
||||
if val is None or val == "":
|
||||
return 0.0
|
||||
if isinstance(val, (int, float)):
|
||||
return float(val)
|
||||
|
||||
val = str(val)
|
||||
# Strip currency symbol and spaces
|
||||
val = val.replace("Kč", "").replace("CZK", "").strip()
|
||||
# Remove thousand separators (often space or dot)
|
||||
# Heuristic: if there's a comma, it's the decimal separator.
|
||||
# If there's a dot, it might be a thousand separator OR decimal separator.
|
||||
if "," in val:
|
||||
# 1.500,00 -> 1500.00
|
||||
val = val.replace(".", "").replace(" ", "").replace(",", ".")
|
||||
else:
|
||||
# 1 500.00 -> 1500.00 or 1.500.00 -> ???
|
||||
# If there are multiple dots, it's thousand separator.
|
||||
if val.count(".") > 1:
|
||||
val = val.replace(".", "").replace(" ", "")
|
||||
# If there's one dot, it might be decimal separator.
|
||||
else:
|
||||
val = val.replace(" ", "")
|
||||
|
||||
try:
|
||||
return float(val)
|
||||
except ValueError:
|
||||
return 0.0
|
||||
|
||||
# Column names as requested by the user
|
||||
COL_MANUAL = "manual fix"
|
||||
COL_PERSON = "Person"
|
||||
COL_PURPOSE = "Purpose"
|
||||
COL_AMOUNT = "Inferred Amount"
|
||||
|
||||
def infer_payments(spreadsheet_id: str, credentials_path: str, dry_run: bool = False):
|
||||
print(f"Connecting to Google Sheets...")
|
||||
service = get_sheets_service(credentials_path)
|
||||
sheet = service.spreadsheets()
|
||||
|
||||
# 1. Fetch all data from the sheet
|
||||
print("Reading sheet data...")
|
||||
result = sheet.values().get(
|
||||
spreadsheetId=spreadsheet_id,
|
||||
range="A1:Z", # Read a broad range to find existing columns
|
||||
valueRenderOption="UNFORMATTED_VALUE"
|
||||
).execute()
|
||||
rows = result.get("values", [])
|
||||
if not rows:
|
||||
print("Sheet is empty.")
|
||||
return
|
||||
|
||||
header = rows[0]
|
||||
|
||||
# Identify indices of existing columns
|
||||
def get_col_index(label):
|
||||
normalized_label = label.lower().strip()
|
||||
for i, h in enumerate(header):
|
||||
if h.lower().strip() == normalized_label:
|
||||
return i
|
||||
return -1
|
||||
|
||||
idx_date = get_col_index("Date")
|
||||
idx_amount_raw = get_col_index("Amount") # Bank Amount
|
||||
idx_sender = get_col_index("Sender")
|
||||
idx_message = get_col_index("Message")
|
||||
idx_vs = get_col_index("VS")
|
||||
|
||||
target_labels = [COL_MANUAL, COL_PERSON, COL_PURPOSE, COL_AMOUNT]
|
||||
|
||||
# Refresh indices
|
||||
idx_manual = get_col_index(COL_MANUAL)
|
||||
idx_inferred_person = get_col_index(COL_PERSON)
|
||||
idx_inferred_purpose = get_col_index(COL_PURPOSE)
|
||||
idx_inferred_amount = get_col_index(COL_AMOUNT)
|
||||
|
||||
if idx_inferred_person == -1 or idx_inferred_purpose == -1 or idx_inferred_amount == -1:
|
||||
print(f"Error: Required columns {target_labels[1:]} not found in sheet.")
|
||||
print(f"Current header: {header}")
|
||||
return
|
||||
|
||||
# 2. Fetch members for matching — union adults + juniors so junior-only
|
||||
# members (e.g. kids not on the adult sheet) are visible to the matcher.
|
||||
print("Fetching member list for matching...")
|
||||
adult_members, _ = get_members_with_fees()
|
||||
junior_members, _ = get_junior_members_with_fees()
|
||||
|
||||
seen: set[str] = set()
|
||||
member_names: list[str] = []
|
||||
for m in adult_members + junior_members:
|
||||
key = canonical_member_key(m[0])
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
member_names.append(m[0])
|
||||
|
||||
# 3. Process rows
|
||||
print("Inferring details for empty rows...")
|
||||
updates = []
|
||||
|
||||
for i, row in enumerate(rows[1:], start=2):
|
||||
# Extend row if it's shorter than existing header
|
||||
while len(row) < len(header):
|
||||
row.append("")
|
||||
|
||||
# Check if already filled (manual override)
|
||||
val_manual = str(row[idx_manual]) if idx_manual != -1 and idx_manual < len(row) else ""
|
||||
val_person = str(row[idx_inferred_person]) if idx_inferred_person < len(row) else ""
|
||||
val_purpose = str(row[idx_inferred_purpose]) if idx_inferred_purpose < len(row) else ""
|
||||
|
||||
if val_manual.strip() or val_person.strip() or val_purpose.strip():
|
||||
continue
|
||||
|
||||
# Prepare transaction dict for matching logic
|
||||
tx = {
|
||||
"date": row[idx_date] if idx_date != -1 and idx_date < len(row) else "",
|
||||
"amount": parse_czk_amount(row[idx_amount_raw]) if idx_amount_raw != -1 and idx_amount_raw < len(row) and row[idx_amount_raw] else 0,
|
||||
"sender": row[idx_sender] if idx_sender != -1 and idx_sender < len(row) else "",
|
||||
"message": row[idx_message] if idx_message != -1 and idx_message < len(row) else "",
|
||||
"vs": row[idx_vs] if idx_vs != -1 and idx_vs < len(row) else "",
|
||||
}
|
||||
|
||||
inference = infer_transaction_details(tx, member_names)
|
||||
|
||||
# Sort members by confidence and add markers
|
||||
peeps = []
|
||||
for name, conf in inference["members"]:
|
||||
prefix = "[?] " if conf == "review" else ""
|
||||
peeps.append(f"{prefix}{name}")
|
||||
|
||||
matched_months = inference["months"]
|
||||
|
||||
if peeps or matched_months:
|
||||
person_val = ", ".join(peeps)
|
||||
purpose_val = ", ".join(matched_months)
|
||||
amount_val = str(tx["amount"]) # For now, use total amount
|
||||
|
||||
print(f"Row {i}: Inferred {person_val} for {purpose_val} ({amount_val} CZK)")
|
||||
|
||||
# Update the row in memory (for terminal output/dry run)
|
||||
row[idx_inferred_person] = person_val
|
||||
row[idx_inferred_purpose] = purpose_val
|
||||
row[idx_inferred_amount] = amount_val
|
||||
|
||||
# Prepare batch update
|
||||
updates.append({
|
||||
"range": f"R{i}C{idx_inferred_person+1}:R{i}C{idx_inferred_amount+1}",
|
||||
"values": [[person_val, purpose_val, amount_val]]
|
||||
})
|
||||
|
||||
if not updates:
|
||||
print("No new inferences to make.")
|
||||
return
|
||||
|
||||
if dry_run:
|
||||
print(f"Dry run: would update {len(updates)} rows.")
|
||||
else:
|
||||
print(f"Applying {len(updates)} updates to the sheet...")
|
||||
body = {
|
||||
"valueInputOption": "USER_ENTERED",
|
||||
"data": updates
|
||||
}
|
||||
sheet.values().batchUpdate(
|
||||
spreadsheetId=spreadsheet_id,
|
||||
body=body
|
||||
).execute()
|
||||
print("Update completed successfully.")
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Infer payment details in Google Sheets.")
|
||||
parser.add_argument("--sheet-id", default=DEFAULT_SPREADSHEET_ID, help="Google Sheet ID")
|
||||
parser.add_argument("--credentials", default="credentials.json", help="Path to Google API credentials JSON")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Print updates without applying them")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
infer_payments(args.sheet_id, args.credentials, args.dry_run)
|
||||
except Exception as e:
|
||||
print(f"Inference failed: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,712 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Match Fio bank payments against expected attendance fees."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import urllib.request
|
||||
from datetime import datetime, timedelta
|
||||
from html.parser import HTMLParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
from attendance import get_members_with_fees
|
||||
from czech_utils import normalize, parse_month_references
|
||||
from sync_fio_to_sheets import get_sheets_service, DEFAULT_SPREADSHEET_ID
|
||||
|
||||
|
||||
def canonical_member_key(name: str) -> str:
|
||||
"""Diacritic-, case-, and whitespace-insensitive key for member-name matching.
|
||||
|
||||
Used to resolve `Person`-column values from the payments sheet to canonical
|
||||
attendance-sheet names, tolerating cells like "Maria Maco" vs "Mária Maco".
|
||||
"""
|
||||
return re.sub(r"\s+", " ", normalize(name)).strip()
|
||||
|
||||
|
||||
def _expected_amount(value) -> float:
|
||||
"""Numeric value of an expected fee; non-numeric markers like '?' → 0.
|
||||
|
||||
Juniors with exactly 1 session get expected='?' (manual-review marker).
|
||||
Treat those as 0 for arithmetic so payments become surplus/credit.
|
||||
"""
|
||||
return value if isinstance(value, (int, float)) else 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Name matching
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _build_name_variants(name: str) -> list[str]:
|
||||
"""Build searchable name variants from a member name.
|
||||
|
||||
E.g. 'František Vrbík (Štrúdl)' → ['frantisek vrbik', 'strudl', 'vrbik']
|
||||
"""
|
||||
# Extract nickname from parentheses
|
||||
nickname_match = re.search(r"\(([^)]+)\)", name)
|
||||
nickname = nickname_match.group(1) if nickname_match else ""
|
||||
|
||||
# Base name without nickname
|
||||
base = re.sub(r"\s*\([^)]*\)\s*", " ", name).strip()
|
||||
normalized_base = normalize(base)
|
||||
normalized_nick = normalize(nickname)
|
||||
|
||||
variants = [normalized_base]
|
||||
if normalized_nick:
|
||||
variants.append(normalized_nick)
|
||||
|
||||
# Also add last name alone (for matching in messages)
|
||||
parts = normalized_base.split()
|
||||
if len(parts) >= 2:
|
||||
variants.append(parts[-1]) # last name
|
||||
variants.append(parts[0]) # first name
|
||||
|
||||
return [v for v in variants if len(v) >= 3]
|
||||
|
||||
|
||||
def _word_in(needle: str, haystack: str) -> bool:
|
||||
"""Return True if needle appears as a whole word in haystack."""
|
||||
return bool(re.search(rf"\b{re.escape(needle)}\b", haystack))
|
||||
|
||||
|
||||
def match_members(
|
||||
text: str, member_names: list[str]
|
||||
) -> list[tuple[str, str]]:
|
||||
"""Find members mentioned in text.
|
||||
|
||||
Returns list of (member_name, confidence) where confidence is 'auto' or 'review'.
|
||||
"""
|
||||
normalized_text = normalize(text)
|
||||
|
||||
# Short-circuit: if any member's full canonical name appears verbatim (whole words),
|
||||
# return only those matches and skip all fuzzy/nickname checks. This prevents a
|
||||
# nickname that is a substring of another member's surname from producing false hits.
|
||||
exact_matches = []
|
||||
for name in member_names:
|
||||
variants = _build_name_variants(name)
|
||||
full_name = variants[0] if variants else ""
|
||||
if full_name and _word_in(full_name, normalized_text):
|
||||
exact_matches.append((name, "auto"))
|
||||
if exact_matches:
|
||||
return exact_matches
|
||||
|
||||
matches = []
|
||||
|
||||
for name in member_names:
|
||||
variants = _build_name_variants(name)
|
||||
full_name = variants[0] if variants else ""
|
||||
parts = full_name.split()
|
||||
|
||||
# 1. Full name match (exact sequence) = high confidence
|
||||
if full_name and full_name in normalized_text:
|
||||
matches.append((name, "auto"))
|
||||
continue
|
||||
|
||||
# 2. Both first and last name present (any order) = high confidence
|
||||
if len(parts) >= 2:
|
||||
if _word_in(parts[0], normalized_text) and _word_in(parts[-1], normalized_text):
|
||||
matches.append((name, "auto"))
|
||||
continue
|
||||
|
||||
# 3. Nickname present = high confidence
|
||||
nickname = ""
|
||||
nickname_match = re.search(r"\(([^)]+)\)", name)
|
||||
if nickname_match:
|
||||
nickname = normalize(nickname_match.group(1))
|
||||
if nickname and _word_in(nickname, normalized_text):
|
||||
matches.append((name, "auto"))
|
||||
continue
|
||||
|
||||
# 4. Partial matches = review confidence
|
||||
if len(parts) >= 2:
|
||||
first_name = parts[0]
|
||||
last_name = parts[-1]
|
||||
_COMMON_SURNAMES = {"novak", "novakova", "prach"}
|
||||
|
||||
if len(last_name) >= 4 and last_name not in _COMMON_SURNAMES and _word_in(last_name, normalized_text):
|
||||
matches.append((name, "review"))
|
||||
continue
|
||||
|
||||
if len(first_name) >= 3 and _word_in(first_name, normalized_text):
|
||||
matches.append((name, "review"))
|
||||
continue
|
||||
elif len(parts) == 1:
|
||||
if len(parts[0]) >= 4 and _word_in(parts[0], normalized_text):
|
||||
matches.append((name, "review"))
|
||||
continue
|
||||
|
||||
# --- Filtering ---
|
||||
# If we have any "auto" matches, discard all "review" matches
|
||||
auto_matches = [m for m in matches if m[1] == "auto"]
|
||||
if auto_matches:
|
||||
return auto_matches
|
||||
|
||||
return matches
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Reconciliation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def infer_transaction_details(tx: dict, member_names: list[str]) -> dict:
|
||||
"""Infer member(s) and month(s) for a single transaction.
|
||||
|
||||
Returns:
|
||||
{
|
||||
'members': [(name, confidence)],
|
||||
'months': [YYYY-MM],
|
||||
'matched_text': str
|
||||
}
|
||||
"""
|
||||
# Combine sender + message for searching
|
||||
search_text = f"{tx.get('sender', '')} {tx.get('message', '')} {tx.get('user_id', '')}"
|
||||
matched_members = match_members(search_text, member_names)
|
||||
matched_months = parse_month_references(
|
||||
tx.get("message", "") + " " + tx.get("user_id", "")
|
||||
)
|
||||
|
||||
if not matched_members:
|
||||
# Try matching sender name alone with more lenient matching
|
||||
matched_members = match_members(tx.get("sender", ""), member_names)
|
||||
|
||||
if not matched_months:
|
||||
# If no month specified, try to infer from payment date
|
||||
tx_date = tx.get("date")
|
||||
if tx_date:
|
||||
try:
|
||||
if isinstance(tx_date, (int, float)):
|
||||
# Handle Google Sheets serial date
|
||||
dt = datetime(1899, 12, 30) + timedelta(days=tx_date)
|
||||
else:
|
||||
dt = datetime.strptime(str(tx_date), "%Y-%m-%d")
|
||||
# Assume payment is for the current month
|
||||
matched_months = [dt.strftime("%Y-%m")]
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
return {
|
||||
"members": matched_members,
|
||||
"months": matched_months,
|
||||
"search_text": search_text
|
||||
}
|
||||
|
||||
|
||||
def format_date(val) -> str:
|
||||
"""Normalize date from Google Sheet (handles serial numbers and strings)."""
|
||||
if val is None or val == "":
|
||||
return ""
|
||||
|
||||
# Handle Google Sheets serial dates (number of days since 1899-12-30)
|
||||
if isinstance(val, (int, float)):
|
||||
base_date = datetime(1899, 12, 30)
|
||||
dt = base_date + timedelta(days=val)
|
||||
return dt.strftime("%Y-%m-%d")
|
||||
|
||||
val_str = str(val).strip()
|
||||
if not val_str:
|
||||
return ""
|
||||
|
||||
# If already YYYY-MM-DD, return as is
|
||||
if len(val_str) == 10 and val_str[4] == "-" and val_str[7] == "-":
|
||||
return val_str
|
||||
|
||||
return val_str
|
||||
|
||||
def fetch_sheet_data(spreadsheet_id: str, credentials_path: str) -> list[dict]:
|
||||
"""Fetch all rows from the Google Sheet and convert to a list of dicts."""
|
||||
service = get_sheets_service(credentials_path)
|
||||
sheet = service.spreadsheets()
|
||||
|
||||
result = sheet.values().get(
|
||||
spreadsheetId=spreadsheet_id,
|
||||
range="A1:Z",
|
||||
valueRenderOption="UNFORMATTED_VALUE"
|
||||
).execute()
|
||||
rows = result.get("values", [])
|
||||
if not rows:
|
||||
return []
|
||||
|
||||
header = rows[0]
|
||||
def get_col_index(label):
|
||||
normalized_label = label.lower().strip()
|
||||
for i, h in enumerate(header):
|
||||
if h.lower().strip() == normalized_label:
|
||||
return i
|
||||
return -1
|
||||
|
||||
idx_date = get_col_index("Date")
|
||||
idx_amount = get_col_index("Amount")
|
||||
idx_manual = get_col_index("manual fix")
|
||||
idx_person = get_col_index("Person")
|
||||
idx_purpose = get_col_index("Purpose")
|
||||
idx_inferred_amount = get_col_index("Inferred Amount")
|
||||
idx_sender = get_col_index("Sender")
|
||||
idx_message = get_col_index("Message")
|
||||
idx_bank_id = get_col_index("Bank ID")
|
||||
idx_vs = get_col_index("VS")
|
||||
idx_sync_id = get_col_index("Sync ID")
|
||||
|
||||
required = {"Date": idx_date, "Amount": idx_amount, "Person": idx_person, "Purpose": idx_purpose}
|
||||
missing = [name for name, idx in required.items() if idx == -1]
|
||||
if missing:
|
||||
raise ValueError(f"Required columns missing from payments sheet: {', '.join(missing)}. Found headers: {header}")
|
||||
|
||||
transactions = []
|
||||
for row in rows[1:]:
|
||||
def get_val(idx):
|
||||
return row[idx] if idx != -1 and idx < len(row) else ""
|
||||
|
||||
def get_str(idx):
|
||||
v = get_val(idx)
|
||||
if isinstance(v, float) and v.is_integer():
|
||||
return str(int(v))
|
||||
return str(v)
|
||||
|
||||
def get_float(idx):
|
||||
v = get_val(idx)
|
||||
if isinstance(v, (int, float)):
|
||||
return float(v)
|
||||
try:
|
||||
return float(str(v).strip())
|
||||
except (ValueError, TypeError):
|
||||
return 0.0
|
||||
|
||||
tx = {
|
||||
"date": format_date(get_val(idx_date)),
|
||||
"amount": get_float(idx_amount),
|
||||
"manual_fix": get_val(idx_manual),
|
||||
"person": get_val(idx_person),
|
||||
"purpose": get_val(idx_purpose),
|
||||
"inferred_amount": get_val(idx_inferred_amount),
|
||||
"sender": get_val(idx_sender),
|
||||
"vs": get_str(idx_vs),
|
||||
"message": get_str(idx_message),
|
||||
"bank_id": get_val(idx_bank_id),
|
||||
"sync_id": get_val(idx_sync_id),
|
||||
}
|
||||
transactions.append(tx)
|
||||
|
||||
return transactions
|
||||
|
||||
|
||||
def fetch_exceptions(spreadsheet_id: str, credentials_path: str) -> dict[tuple[str, str], dict]:
|
||||
"""Fetch manual fee overrides from the 'exceptions' sheet.
|
||||
|
||||
Returns a dict mapping (member_name, period_YYYYMM) to {'amount': int, 'note': str}.
|
||||
"""
|
||||
service = get_sheets_service(credentials_path)
|
||||
try:
|
||||
result = service.spreadsheets().values().get(
|
||||
spreadsheetId=spreadsheet_id,
|
||||
range="'exceptions'!A2:D",
|
||||
valueRenderOption="UNFORMATTED_VALUE"
|
||||
).execute()
|
||||
rows = result.get("values", [])
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not fetch exceptions: {e}")
|
||||
return {}
|
||||
|
||||
exceptions = {}
|
||||
for row in rows:
|
||||
if len(row) < 3 or str(row[0]).lower().startswith("name"):
|
||||
continue
|
||||
|
||||
name = str(row[0]).strip()
|
||||
period = str(row[1]).strip()
|
||||
# Robust normalization using czech_utils.normalize
|
||||
norm_name = normalize(name)
|
||||
norm_period = normalize(period)
|
||||
|
||||
try:
|
||||
amount = int(row[2])
|
||||
note = str(row[3]).strip() if len(row) > 3 else ""
|
||||
exceptions[(norm_name, norm_period)] = {"amount": amount, "note": note}
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
|
||||
return exceptions
|
||||
|
||||
|
||||
def reconcile(
|
||||
members: list[tuple[str, str, dict[str, int]]],
|
||||
sorted_months: list[str],
|
||||
transactions: list[dict],
|
||||
exceptions: dict[tuple[str, str], dict] = None,
|
||||
) -> dict:
|
||||
"""Match transactions to members and months.
|
||||
|
||||
Returns a dict with:
|
||||
- 'members': {name: {'tier': str, 'months': {YYYY-MM: {'expected': int, 'paid': int, 'transactions': list}}}}
|
||||
- 'unmatched': list of transactions that couldn't be matched
|
||||
- 'credits': {name: int} — excess payments tracked as credit
|
||||
"""
|
||||
member_names = [name for name, _, _ in members]
|
||||
member_tiers = {name: tier for name, tier, _ in members}
|
||||
member_fees = {name: fees for name, _, fees in members}
|
||||
|
||||
# Map canonical key → first attendance-sheet name with that key, so a
|
||||
# `Person` cell that drifts in diacritics/case/whitespace still resolves.
|
||||
canonical_by_key: dict[str, str] = {}
|
||||
for name in member_names:
|
||||
canonical_by_key.setdefault(canonical_member_key(name), name)
|
||||
|
||||
# Initialize ledger
|
||||
ledger: dict[str, dict[str, dict]] = {}
|
||||
other_ledger: dict[str, list] = {}
|
||||
exceptions = exceptions or {}
|
||||
for name in member_names:
|
||||
ledger[name] = {}
|
||||
other_ledger[name] = []
|
||||
for m in sorted_months:
|
||||
# Robust normalization for lookup
|
||||
norm_name = normalize(name)
|
||||
norm_period = normalize(m)
|
||||
fee_data = member_fees[name].get(m, (0, 0))
|
||||
original_expected = fee_data[0] if isinstance(fee_data, (tuple, list)) else fee_data
|
||||
attendance_count = fee_data[1] if isinstance(fee_data, (tuple, list)) else 0
|
||||
|
||||
ex_data = exceptions.get((norm_name, norm_period))
|
||||
if ex_data is not None:
|
||||
expected = ex_data["amount"]
|
||||
exception_info = ex_data
|
||||
else:
|
||||
expected = original_expected
|
||||
exception_info = None
|
||||
|
||||
ledger[name][m] = {
|
||||
"expected": expected,
|
||||
"original_expected": original_expected,
|
||||
"attendance_count": attendance_count,
|
||||
"exception": exception_info,
|
||||
"paid": 0,
|
||||
"transactions": [],
|
||||
}
|
||||
|
||||
unmatched = []
|
||||
credits: dict[str, int] = {}
|
||||
|
||||
for tx in transactions:
|
||||
# Use sheet columns if they exist, otherwise fallback to inference
|
||||
person_str = str(tx.get("person", "")).strip()
|
||||
purpose_str = str(tx.get("purpose", "")).strip()
|
||||
|
||||
# Strip markers like [?]
|
||||
person_str = re.sub(r"\[\?\]\s*", "", person_str)
|
||||
is_other = purpose_str.lower().startswith("other:")
|
||||
|
||||
if person_str and purpose_str:
|
||||
# We have pre-matched data (either from script or manual)
|
||||
# Support multiple people/months in the comma-separated string
|
||||
matched_members = [(p.strip(), "auto") for p in person_str.split(",") if p.strip()]
|
||||
matched_months = [purpose_str] if is_other else [m.strip() for m in purpose_str.split(",") if m.strip()]
|
||||
|
||||
# Use Inferred Amount if available, otherwise bank Amount
|
||||
amount = tx.get("inferred_amount")
|
||||
if amount is None or amount == "":
|
||||
amount = tx.get("amount", 0)
|
||||
try:
|
||||
amount = float(amount)
|
||||
except (ValueError, TypeError):
|
||||
amount = 0
|
||||
else:
|
||||
# Fallback to inference (for rows not yet processed by infer_payments.py)
|
||||
inference = infer_transaction_details(tx, member_names)
|
||||
matched_members = inference["members"]
|
||||
matched_months = inference["months"]
|
||||
amount = tx.get("amount", 0)
|
||||
try:
|
||||
amount = float(amount)
|
||||
except (ValueError, TypeError):
|
||||
amount = 0
|
||||
|
||||
if not matched_members or not matched_months:
|
||||
unmatched.append(tx)
|
||||
continue
|
||||
|
||||
# Allocate payment across matched members and months
|
||||
if is_other:
|
||||
num_allocations = len(matched_members)
|
||||
per_allocation = amount / num_allocations if num_allocations > 0 else 0
|
||||
for raw_member_name, confidence in matched_members:
|
||||
member_name = canonical_by_key.get(canonical_member_key(raw_member_name))
|
||||
if member_name is not None:
|
||||
other_ledger[member_name].append({
|
||||
"amount": per_allocation,
|
||||
"date": tx["date"],
|
||||
"sender": tx["sender"],
|
||||
"message": tx["message"],
|
||||
"purpose": purpose_str,
|
||||
"confidence": confidence,
|
||||
})
|
||||
continue
|
||||
|
||||
member_share = amount / len(matched_members) if matched_members else 0
|
||||
|
||||
for raw_member_name, confidence in matched_members:
|
||||
member_name = canonical_by_key.get(canonical_member_key(raw_member_name))
|
||||
if member_name is None:
|
||||
logger.warning(
|
||||
"Payment matched to unknown member %r (tx: %s, %s) — adding to unmatched",
|
||||
raw_member_name, tx.get("date", "?"), tx.get("message", "?"),
|
||||
)
|
||||
unmatched.append(tx)
|
||||
continue
|
||||
if member_name != raw_member_name:
|
||||
logger.info(
|
||||
"Person cell %r resolved to canonical member %r — consider fixing the sheet",
|
||||
raw_member_name, member_name,
|
||||
)
|
||||
|
||||
in_window = [(m, ledger[member_name][m]["expected"]) for m in matched_months if m in ledger[member_name]]
|
||||
out_of_window = [m for m in matched_months if m not in ledger[member_name]]
|
||||
|
||||
# Out-of-window months (outside display range): even split → credit, same as before.
|
||||
n_total = len(matched_months)
|
||||
if out_of_window and n_total > 0:
|
||||
out_credit = member_share / n_total * len(out_of_window)
|
||||
credits[member_name] = credits.get(member_name, 0) + int(out_credit)
|
||||
else:
|
||||
out_credit = 0.0
|
||||
|
||||
in_window_share = member_share - out_credit
|
||||
|
||||
if not in_window:
|
||||
continue
|
||||
|
||||
total_expected = sum(_expected_amount(e) for _, e in in_window)
|
||||
|
||||
if total_expected > 0:
|
||||
# Fill-first: iterate in_window in matched_months order (chronological by
|
||||
# convention from infer_payments.py), allocating min(remaining, deficit) to
|
||||
# each month. Deficit is net of what prior transactions already paid, so a
|
||||
# second payment on the same months correctly fills only what remains due.
|
||||
# Any surplus after all deficits are covered goes to the credit bucket.
|
||||
remaining = in_window_share
|
||||
for m, exp in in_window:
|
||||
paid_so_far = ledger[member_name][m]["paid"]
|
||||
deficit = max(0.0, float(_expected_amount(exp)) - paid_so_far)
|
||||
alloc = min(remaining, deficit)
|
||||
if alloc <= 0:
|
||||
continue
|
||||
ledger[member_name][m]["paid"] += alloc
|
||||
ledger[member_name][m]["transactions"].append({
|
||||
"amount": alloc,
|
||||
"date": tx["date"],
|
||||
"sender": tx["sender"],
|
||||
"message": tx["message"],
|
||||
"confidence": confidence,
|
||||
})
|
||||
remaining -= alloc
|
||||
if remaining > 0:
|
||||
credits[member_name] = credits.get(member_name, 0) + int(remaining)
|
||||
else:
|
||||
# Fallback: no expected fees (prepayment before attendance recorded); even split.
|
||||
per_month = in_window_share / len(in_window)
|
||||
for m, _ in in_window:
|
||||
ledger[member_name][m]["paid"] += per_month
|
||||
ledger[member_name][m]["transactions"].append({
|
||||
"amount": per_month,
|
||||
"date": tx["date"],
|
||||
"sender": tx["sender"],
|
||||
"message": tx["message"],
|
||||
"confidence": confidence,
|
||||
})
|
||||
|
||||
# Calculate final total balances (window + off-window credits)
|
||||
final_balances: dict[str, int] = {}
|
||||
for name in member_names:
|
||||
window_balance = sum(
|
||||
int(mdata["paid"]) - _expected_amount(mdata["expected"])
|
||||
for mdata in ledger[name].values()
|
||||
)
|
||||
final_balances[name] = window_balance + credits.get(name, 0)
|
||||
|
||||
return {
|
||||
"members": {
|
||||
name: {
|
||||
"tier": member_tiers[name],
|
||||
"months": ledger[name],
|
||||
"other_transactions": other_ledger[name],
|
||||
"total_balance": final_balances[name]
|
||||
}
|
||||
for name in member_names
|
||||
},
|
||||
"unmatched": unmatched,
|
||||
"credits": final_balances, # Redefine credits as any positive total balance
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Report output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def print_report(result: dict, sorted_months: list[str]):
|
||||
month_labels = {
|
||||
m: datetime.strptime(m, "%Y-%m").strftime("%b %Y") for m in sorted_months
|
||||
}
|
||||
|
||||
# --- Per-member breakdown (adults only) ---
|
||||
print("=" * 80)
|
||||
print("PAYMENT RECONCILIATION REPORT")
|
||||
print("=" * 80)
|
||||
|
||||
adults = {
|
||||
name: data
|
||||
for name, data in result["members"].items()
|
||||
if data["tier"] == "A"
|
||||
}
|
||||
|
||||
total_expected = 0
|
||||
total_paid = 0
|
||||
|
||||
# Summary table
|
||||
name_width = max((len(n) for n in adults), default=20)
|
||||
header = f"{'Member':<{name_width}}"
|
||||
for m in sorted_months:
|
||||
header += f" | {month_labels[m]:>10}"
|
||||
header += " | {'Balance':>10}"
|
||||
print(f"\n{'Member':<{name_width}}", end="")
|
||||
for m in sorted_months:
|
||||
print(f" | {month_labels[m]:>10}", end="")
|
||||
print(f" | {'Balance':>10}")
|
||||
print("-" * (name_width + (len(sorted_months) + 1) * 13))
|
||||
|
||||
for name in sorted(adults.keys()):
|
||||
data = adults[name]
|
||||
line = f"{name:<{name_width}}"
|
||||
member_balance = 0
|
||||
for m in sorted_months:
|
||||
mdata = data["months"].get(m, {"expected": 0, "paid": 0})
|
||||
expected = mdata["expected"]
|
||||
original = mdata["original_expected"]
|
||||
paid = int(mdata["paid"])
|
||||
total_expected += expected
|
||||
total_paid += paid
|
||||
|
||||
cell_status = ""
|
||||
if expected == 0 and paid == 0:
|
||||
cell = "-"
|
||||
elif paid >= expected and expected > 0:
|
||||
cell = "OK"
|
||||
elif paid > 0:
|
||||
cell = f"{paid}/{expected}"
|
||||
else:
|
||||
cell = f"UNPAID {expected}"
|
||||
|
||||
member_balance += paid - expected
|
||||
line += f" | {cell:>10}"
|
||||
balance_str = f"{member_balance:+d}" if member_balance != 0 else "0"
|
||||
line += f" | {balance_str:>10}"
|
||||
print(line)
|
||||
|
||||
print("-" * (name_width + (len(sorted_months) + 1) * 13))
|
||||
print(f"{'TOTAL':<{name_width}}", end="")
|
||||
for _ in sorted_months:
|
||||
print(f" | {'':>10}", end="")
|
||||
balance = total_paid - total_expected
|
||||
print(f" | {f'Expected: {total_expected}, Paid: {int(total_paid)}, Balance: {balance:+d}'}")
|
||||
|
||||
# --- Credits (Total Surplus) ---
|
||||
all_credits = {
|
||||
name: data["total_balance"]
|
||||
for name, data in result["members"].items()
|
||||
if data["total_balance"] > 0
|
||||
}
|
||||
|
||||
if all_credits:
|
||||
print(f"\n{'TOTAL CREDITS (advance payments or surplus):'}")
|
||||
for name, amount in sorted(all_credits.items()):
|
||||
print(f" {name}: {amount} CZK")
|
||||
|
||||
# --- Debts (Missing Payments) ---
|
||||
all_debts = {
|
||||
name: data["total_balance"]
|
||||
for name, data in result["members"].items()
|
||||
if data["total_balance"] < 0
|
||||
}
|
||||
|
||||
if all_debts:
|
||||
print(f"\n{'TOTAL DEBTS (missing payments):'}")
|
||||
for name, amount in sorted(all_debts.items()):
|
||||
print(f" {name}: {abs(amount)} CZK")
|
||||
|
||||
# --- Unmatched transactions ---
|
||||
if result["unmatched"]:
|
||||
print(f"\n{'UNMATCHED TRANSACTIONS (need manual review)':}")
|
||||
print(f" {'Date':<12} {'Amount':>10} {'Sender':<30} {'Message'}")
|
||||
print(f" {'-'*12} {'-'*10} {'-'*30} {'-'*30}")
|
||||
for tx in result["unmatched"]:
|
||||
print(
|
||||
f" {tx['date']:<12} {tx['amount']:>10.0f} "
|
||||
f"{tx['sender']:<30} {tx['message']}"
|
||||
)
|
||||
|
||||
# --- Detailed matched transactions ---
|
||||
print(f"\n{'MATCHED TRANSACTION DETAILS':}")
|
||||
for name in sorted(adults.keys()):
|
||||
data = adults[name]
|
||||
has_payments = any(
|
||||
data["months"].get(m, {}).get("transactions")
|
||||
for m in sorted_months
|
||||
)
|
||||
if not has_payments:
|
||||
continue
|
||||
print(f"\n {name}:")
|
||||
for m in sorted_months:
|
||||
mdata = data["months"].get(m, {})
|
||||
for tx in mdata.get("transactions", []):
|
||||
conf = " [REVIEW]" if tx["confidence"] == "review" else ""
|
||||
print(
|
||||
f" {month_labels[m]}: {tx['amount']:.0f} CZK "
|
||||
f"from {tx['sender']} — \"{tx['message']}\"{conf}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Match bank payments against expected attendance fees."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--sheet-id", default=DEFAULT_SPREADSHEET_ID, help="Google Sheet ID"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--credentials", default=".secret/fuj-management-bot-credentials.json",
|
||||
help="Path to Google API credentials JSON"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--bank", action="store_true", help="Scrape bank instead of using Sheet data"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"Fetching attendance data...")
|
||||
members, sorted_months = get_members_with_fees()
|
||||
if not members:
|
||||
print("No attendance data found.")
|
||||
return
|
||||
|
||||
if args.bank:
|
||||
print(f"Fetching transactions from Fio bank ({args.date_from} to {args.date_to})...")
|
||||
from fio_utils import fetch_transactions
|
||||
transactions = fetch_transactions(args.date_from, args.date_to)
|
||||
else:
|
||||
print(f"Fetching transactions from Google Sheet ({args.sheet_id})...")
|
||||
transactions = fetch_sheet_data(args.sheet_id, args.credentials)
|
||||
|
||||
print(f"Processing {len(transactions)} transactions.\n")
|
||||
|
||||
exceptions = fetch_exceptions(args.sheet_id, args.credentials)
|
||||
if exceptions:
|
||||
print(f"Loaded {len(exceptions)} fee exceptions.")
|
||||
|
||||
result = reconcile(members, sorted_months, transactions, exceptions)
|
||||
print_report(result, sorted_months)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,330 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Scrub PII from fixture JSON.
|
||||
|
||||
Reads one JSON fixture from stdin (as produced by capture_fixtures.py),
|
||||
replaces PII fields with deterministic pseudonyms, writes scrubbed JSON
|
||||
to stdout.
|
||||
|
||||
Run in the two-step pipeline:
|
||||
python capture_fixtures.py ... | python scrub_fixtures.py > fixture.json
|
||||
|
||||
Or process multiple lines (--multi for newline-delimited input):
|
||||
python capture_fixtures.py --func foo --all | python scrub_fixtures.py --multi \\
|
||||
| while read line; do ...
|
||||
|
||||
PII handling:
|
||||
- Member names: replaced with Member_<8hex> (sha256(name)[:8]), deterministic.
|
||||
- Senders / account numbers / VS / bank_id / user_id: stable digit-preserving hash.
|
||||
- Notes (exception text): replaced with "<scrubbed>".
|
||||
- Messages: name-substring sweep applied; rest preserved.
|
||||
- All other fields (dates, amounts, months, fees): preserved verbatim.
|
||||
|
||||
Function-specific exceptions:
|
||||
- match_members / infer_transaction_details: these functions are tested with
|
||||
synthetic member names only. Only real-roster message sweeping is applied;
|
||||
field-key scrubbing is skipped so Go can perform genuine name matching.
|
||||
- generate_sync_id: after normal field-key scrubbing the output sync_id is
|
||||
recomputed from the now-scrubbed inputs so the hash remains consistent.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
_REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bijection helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _sha256_hex(s: str) -> str:
|
||||
return hashlib.sha256(s.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def scrub_name(name: str) -> str:
|
||||
"""Deterministic pseudonym for a member name."""
|
||||
if not name:
|
||||
return name
|
||||
return f"Member_{_sha256_hex(name)[:8]}"
|
||||
|
||||
|
||||
def scrub_id_digits(s: str) -> str:
|
||||
"""Length-preserving digit hash for VS, bank_id, user_id, etc."""
|
||||
s = str(s)
|
||||
if not s:
|
||||
return s
|
||||
if re.match(r"^\d+$", s):
|
||||
n = len(s)
|
||||
hashed = int(_sha256_hex(s), 16) % (10 ** n)
|
||||
return f"{hashed:0{n}d}"
|
||||
return f"id_{_sha256_hex(s)[:8]}"
|
||||
|
||||
|
||||
def scrub_account(s: str) -> str:
|
||||
"""Preserve Czech bank account format PREFIX/BANKCODE."""
|
||||
s = str(s)
|
||||
if not s:
|
||||
return s
|
||||
m = re.match(r"^(\d+)/(\d{4})$", s)
|
||||
if m:
|
||||
prefix, bank = m.group(1), m.group(2)
|
||||
n = len(prefix)
|
||||
new_prefix = int(_sha256_hex(prefix), 16) % (10 ** n)
|
||||
new_bank = int(_sha256_hex(bank), 16) % 10000
|
||||
return f"{new_prefix:0{n}d}/{new_bank:04d}"
|
||||
return scrub_id_digits(s)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Name roster for message sweeps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _load_member_names() -> list[str]:
|
||||
"""Load canonical names from the attendance cache (may not exist)."""
|
||||
path = os.path.join(_REPO, "tmp", "attendance_regular_cache.json")
|
||||
if not os.path.exists(path):
|
||||
return []
|
||||
try:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
cache = json.load(f)
|
||||
rows = cache.get("data", [])
|
||||
if rows and isinstance(rows[0], list):
|
||||
rows = rows[0]
|
||||
names = []
|
||||
for row in rows:
|
||||
if isinstance(row, (list, tuple)) and len(row) >= 1:
|
||||
names.append(str(row[0]))
|
||||
return names
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def _build_name_map(names: list[str]) -> dict[str, str]:
|
||||
"""Map each real name (and its normalized form) to its pseudonym."""
|
||||
mapping: dict[str, str] = {}
|
||||
for name in names:
|
||||
pseudo = scrub_name(name)
|
||||
mapping[name] = pseudo
|
||||
# Also add first+last without parenthetical nicknames
|
||||
base = re.sub(r"\s*\([^)]*\)\s*", " ", name).strip()
|
||||
if base != name:
|
||||
mapping[base] = pseudo
|
||||
return mapping
|
||||
|
||||
|
||||
def _sweep_names_in_text(text: str, name_map: dict[str, str]) -> str:
|
||||
"""Replace real-name substrings in free text, longest match first."""
|
||||
# Sort descending by length so longer names replace before their substrings
|
||||
for real in sorted(name_map, key=len, reverse=True):
|
||||
if real and real in text:
|
||||
text = text.replace(real, name_map[real])
|
||||
return text
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scramble whitelist — only these keys are scrambled; everything else is kept
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_SCRAMBLE_KEYS = {
|
||||
"name",
|
||||
"member_names",
|
||||
"person",
|
||||
"sender",
|
||||
"sender_account",
|
||||
"account",
|
||||
"vs",
|
||||
"bank_id",
|
||||
"user_id",
|
||||
"note",
|
||||
}
|
||||
|
||||
# Dict keys whose *child keys* (not values) are member names and need scrubbing.
|
||||
# e.g. the reconcile output: {"members": {"Alice Dvořák": {...}}, "credits": {"Alice Dvořák": 0}}
|
||||
_MEMBER_KEY_DICTS = {"members", "credits"}
|
||||
|
||||
_MESSAGE_KEYS = {"message", "text", "search_text"}
|
||||
|
||||
|
||||
def _scrub_value(key: str, value: Any, name_map: dict[str, str]) -> Any:
|
||||
"""Scrub a single value based on its field key."""
|
||||
if isinstance(value, list):
|
||||
if key == "member_names":
|
||||
return [scrub_name(str(v)) for v in value]
|
||||
# Don't propagate parent key into list elements — each element is an
|
||||
# independent document. Propagating would incorrectly flag nested dicts
|
||||
# (e.g. the fees dict inside a member tuple) as member-name-keyed dicts.
|
||||
return [_scrub_doc(v, name_map) for v in value]
|
||||
if isinstance(value, dict):
|
||||
# Pass the current key as parent context so dicts like
|
||||
# {"members": {"Real Name": ...}} get their keys scrubbed too.
|
||||
return _scrub_doc(value, name_map, _parent_key=key)
|
||||
if key not in _SCRAMBLE_KEYS and key not in _MESSAGE_KEYS:
|
||||
return value
|
||||
if not isinstance(value, str):
|
||||
value = str(value)
|
||||
if key in _MESSAGE_KEYS:
|
||||
return _sweep_names_in_text(value, name_map)
|
||||
if key == "name":
|
||||
return scrub_name(value)
|
||||
if key in ("sender_account", "account"):
|
||||
return scrub_account(value)
|
||||
if key == "note":
|
||||
return "<scrubbed>"
|
||||
if key == "person":
|
||||
# "person" may contain comma-separated member names (e.g. "Alice, Bob").
|
||||
# Sweep with name_map so each name gets its own consistent pseudonym,
|
||||
# matching what the output.members keys will look like.
|
||||
return _sweep_names_in_text(value, name_map) if value else value
|
||||
# vs, bank_id, user_id, sender
|
||||
return scrub_id_digits(value) if re.match(r"^\d+$", value) else scrub_name(value) if value else value
|
||||
|
||||
|
||||
def _scrub_doc(doc: Any, name_map: dict[str, str], _parent_key: str = "") -> Any:
|
||||
"""Recursively scrub a JSON document."""
|
||||
if isinstance(doc, dict):
|
||||
if _parent_key in _MEMBER_KEY_DICTS:
|
||||
# Keys of this dict are member names — scrub the keys and recurse.
|
||||
return {
|
||||
scrub_name(k): _scrub_doc(v, name_map)
|
||||
for k, v in doc.items()
|
||||
}
|
||||
return {k: _scrub_value(k, v, name_map) for k, v in doc.items()}
|
||||
if isinstance(doc, list):
|
||||
return [_scrub_doc(item, name_map) for item in doc]
|
||||
return doc
|
||||
|
||||
|
||||
# Functions where field-key scrubbing would break parity (name matching tests).
|
||||
# Only real-roster message sweep is applied for these.
|
||||
_NO_FIELD_SCRUB_FUNCS = {
|
||||
"scripts.match_payments.match_members",
|
||||
"scripts.match_payments.infer_transaction_details",
|
||||
}
|
||||
|
||||
|
||||
def _scrub_messages_only(doc: Any, name_map: dict[str, str]) -> Any:
|
||||
"""Sweep only message/text/search_text fields; leave all other values unchanged."""
|
||||
if isinstance(doc, dict):
|
||||
return {
|
||||
k: (_sweep_names_in_text(v, name_map) if k in _MESSAGE_KEYS and isinstance(v, str)
|
||||
else _scrub_messages_only(v, name_map))
|
||||
for k, v in doc.items()
|
||||
}
|
||||
if isinstance(doc, list):
|
||||
return [_scrub_messages_only(item, name_map) for item in doc]
|
||||
return doc
|
||||
|
||||
|
||||
def _recompute_sync_id(tx_scrubbed: dict) -> str:
|
||||
"""Recompute generate_sync_id hash from already-scrubbed tx fields.
|
||||
|
||||
After the scrubber changes sender/vs/bank_id the original hash is invalid.
|
||||
Replicates the Python generate_sync_id formula (pipe-separated, lowercased)
|
||||
and always treats amount as float64 to match Go's formatAmount behaviour.
|
||||
"""
|
||||
envelope = tx_scrubbed.get("amount", {})
|
||||
if isinstance(envelope, dict):
|
||||
t = envelope.get("type", "")
|
||||
v = envelope.get("value")
|
||||
if t in ("int", "float"):
|
||||
amount = float(v) # always float — matches Go's formatAmount
|
||||
else:
|
||||
amount = ""
|
||||
else:
|
||||
amount = float(envelope) if envelope not in (None, "") else ""
|
||||
|
||||
currency = tx_scrubbed.get("currency", "") or "CZK"
|
||||
components = [
|
||||
str(tx_scrubbed.get("date", "")),
|
||||
str(amount),
|
||||
currency,
|
||||
str(tx_scrubbed.get("sender", "")),
|
||||
str(tx_scrubbed.get("vs", "")),
|
||||
str(tx_scrubbed.get("message", "")),
|
||||
str(tx_scrubbed.get("bank_id", "")),
|
||||
]
|
||||
raw_str = "|".join(components).lower()
|
||||
return hashlib.sha256(raw_str.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _extract_inline_names(doc: Any) -> list[str]:
|
||||
"""Extract names from member_names and 'name' fields in the fixture itself."""
|
||||
names: list[str] = []
|
||||
if isinstance(doc, dict):
|
||||
for k, v in doc.items():
|
||||
if k == "member_names" and isinstance(v, list):
|
||||
names.extend(str(n) for n in v)
|
||||
elif k == "name" and isinstance(v, str):
|
||||
names.append(v)
|
||||
else:
|
||||
names.extend(_extract_inline_names(v))
|
||||
elif isinstance(doc, list):
|
||||
for item in doc:
|
||||
names.extend(_extract_inline_names(item))
|
||||
return names
|
||||
|
||||
|
||||
def scrub_fixture(doc: dict) -> dict:
|
||||
"""Scrub a single fixture document in-place (returns new dict)."""
|
||||
roster_names = _load_member_names()
|
||||
inline_names = _extract_inline_names(doc)
|
||||
all_names = list(dict.fromkeys(roster_names + inline_names))
|
||||
name_map = _build_name_map(all_names)
|
||||
|
||||
func = doc.get("func", "")
|
||||
|
||||
# match_members / infer_transaction_details: tested with synthetic names only.
|
||||
# Field-key scrubbing would make member_names pseudonyms inconsistent with
|
||||
# the text, breaking Go's name-matching assertions. Only sweep messages.
|
||||
if func in _NO_FIELD_SCRUB_FUNCS:
|
||||
# Synthetic member names only — no field scrubbing, no message sweep.
|
||||
# Any sweep would create inconsistency between scrubbed output fields
|
||||
# (search_text) and un-scrubbed input fields (sender, member_names).
|
||||
return _scrub_messages_only(doc, {})
|
||||
|
||||
result = _scrub_doc(doc, name_map)
|
||||
|
||||
# generate_sync_id: recompute hash from the now-scrubbed inputs so the
|
||||
# fixture is self-consistent (scrubbed fields → Go hashes scrubbed values).
|
||||
if func.endswith("generate_sync_id"):
|
||||
result["output"]["sync_id"] = _recompute_sync_id(result["input"].get("tx", {}))
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Scrub PII from fixture JSON.")
|
||||
parser.add_argument(
|
||||
"--multi", action="store_true",
|
||||
help="Process newline-delimited JSON (one object per line) from stdin.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.multi:
|
||||
for line in sys.stdin:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
doc = json.loads(line)
|
||||
print(json.dumps(scrub_fixture(doc), ensure_ascii=False))
|
||||
else:
|
||||
doc = json.load(sys.stdin)
|
||||
out = scrub_fixture(doc)
|
||||
print(json.dumps(out, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,289 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sync Fio bank transactions to a Google Sheet intermediary ledger."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import pickle
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from google.auth.transport.requests import Request
|
||||
from google_auth_oauthlib.flow import InstalledAppFlow
|
||||
from google.oauth2 import service_account
|
||||
from googleapiclient.discovery import build
|
||||
|
||||
from fio_utils import fetch_transactions_all
|
||||
|
||||
from config import PAYMENTS_SHEET_ID as DEFAULT_SPREADSHEET_ID
|
||||
SCOPES = ["https://www.googleapis.com/auth/spreadsheets"]
|
||||
TOKEN_FILE = "token.pickle"
|
||||
COLUMN_LABELS = ["Date", "Amount", "manual fix", "Person", "Purpose", "Inferred Amount", "Sender", "VS", "Message", "Bank ID", "Sync ID"]
|
||||
_SHEETS_SERVICE = None
|
||||
|
||||
def get_sheets_service(credentials_path: str):
|
||||
"""Authenticate and return the Google Sheets API service."""
|
||||
global _SHEETS_SERVICE
|
||||
if _SHEETS_SERVICE is not None:
|
||||
return _SHEETS_SERVICE
|
||||
|
||||
if not os.path.exists(credentials_path):
|
||||
raise FileNotFoundError(f"Credentials file not found: {credentials_path}")
|
||||
|
||||
# Check if it's a service account
|
||||
import json
|
||||
with open(credentials_path, "r") as f:
|
||||
creds_data = json.load(f)
|
||||
|
||||
if creds_data.get("type") == "service_account":
|
||||
creds = service_account.Credentials.from_service_account_file(
|
||||
credentials_path, scopes=SCOPES
|
||||
)
|
||||
else:
|
||||
# Fallback to OAuth2 flow
|
||||
creds = None
|
||||
if os.path.exists(TOKEN_FILE):
|
||||
with open(TOKEN_FILE, "rb") as token:
|
||||
creds = pickle.load(token)
|
||||
|
||||
if not creds or not creds.valid:
|
||||
if creds and creds.expired and creds.refresh_token:
|
||||
creds.refresh(Request())
|
||||
else:
|
||||
flow = InstalledAppFlow.from_client_secrets_file(credentials_path, SCOPES)
|
||||
creds = flow.run_local_server(port=0)
|
||||
|
||||
with open(TOKEN_FILE, "wb") as token:
|
||||
pickle.dump(creds, token)
|
||||
|
||||
_SHEETS_SERVICE = build("sheets", "v4", credentials=creds)
|
||||
return _SHEETS_SERVICE
|
||||
|
||||
|
||||
def generate_sync_id(tx: dict) -> str:
|
||||
"""Generate a unique SHA-256 hash for a transaction.
|
||||
|
||||
Hash components: date|amount|currency|sender|vs|message|bank_id
|
||||
"""
|
||||
components = [
|
||||
str(tx.get("date", "")),
|
||||
str(tx.get("amount", "")),
|
||||
str(tx.get("currency", "CZK")),
|
||||
str(tx.get("sender", "")),
|
||||
str(tx.get("vs", "")),
|
||||
str(tx.get("message", "")),
|
||||
str(tx.get("bank_id", "")),
|
||||
]
|
||||
raw_str = "|".join(components).lower()
|
||||
return hashlib.sha256(raw_str.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _trunc(s: str, n: int = 40) -> str:
|
||||
s = str(s)
|
||||
return s if len(s) <= n else s[: n - 1] + "…"
|
||||
|
||||
|
||||
def _print_fio_table(transactions: list[dict], statuses: list[str]) -> None:
|
||||
headers = ["DATE", "AMOUNT", "SENDER", "VS", "MESSAGE", "BANKID", "STATUS"]
|
||||
rows = [
|
||||
[
|
||||
str(tx.get("date", "")),
|
||||
f"{float(tx.get('amount', 0)):.2f}",
|
||||
str(tx.get("sender", "")),
|
||||
str(tx.get("vs", "")),
|
||||
_trunc(str(tx.get("message", ""))),
|
||||
str(tx.get("bank_id", "")),
|
||||
status,
|
||||
]
|
||||
for tx, status in zip(transactions, statuses)
|
||||
]
|
||||
widths = [
|
||||
max(len(headers[i]), max((len(r[i]) for r in rows), default=0))
|
||||
for i in range(len(headers))
|
||||
]
|
||||
sep = " "
|
||||
print(sep.join(h.ljust(w) for h, w in zip(headers, widths)))
|
||||
for row in rows:
|
||||
print(sep.join(cell.ljust(w) for cell, w in zip(row, widths)))
|
||||
|
||||
|
||||
def sort_sheet_by_date(service, spreadsheet_id):
|
||||
"""Sort the sheet by the Date column (Column B)."""
|
||||
# Get the sheet ID (gid) of the first sheet
|
||||
spreadsheet = service.spreadsheets().get(spreadsheetId=spreadsheet_id).execute()
|
||||
sheet_id = spreadsheet['sheets'][0]['properties']['sheetId']
|
||||
|
||||
requests = [{
|
||||
"sortRange": {
|
||||
"range": {
|
||||
"sheetId": sheet_id,
|
||||
"startRowIndex": 1, # Skip header
|
||||
"endRowIndex": 10000
|
||||
},
|
||||
"sortSpecs": [{
|
||||
"dimensionIndex": 0, # Column A (Date)
|
||||
"sortOrder": "ASCENDING"
|
||||
}]
|
||||
}
|
||||
}]
|
||||
|
||||
service.spreadsheets().batchUpdate(
|
||||
spreadsheetId=spreadsheet_id,
|
||||
body={"requests": requests}
|
||||
).execute()
|
||||
print("Sheet sorted by date.")
|
||||
|
||||
|
||||
def sync_to_sheets(
|
||||
spreadsheet_id: str,
|
||||
credentials_path: str,
|
||||
days: int = None,
|
||||
date_from_str: str = None,
|
||||
date_to_str: str = None,
|
||||
sort_by_date: bool = False,
|
||||
dry_run: bool = False,
|
||||
print_fio_table: bool = False,
|
||||
):
|
||||
print(f"Connecting to Google Sheets using {credentials_path}...")
|
||||
service = get_sheets_service(credentials_path)
|
||||
sheet = service.spreadsheets()
|
||||
|
||||
# 1. Read existing sync IDs from Column K
|
||||
print(f"Reading existing sync IDs from sheet...")
|
||||
try:
|
||||
result = sheet.values().get(
|
||||
spreadsheetId=spreadsheet_id,
|
||||
range="A1:K" # Include header and all columns to check Sync ID
|
||||
).execute()
|
||||
values = result.get("values", [])
|
||||
|
||||
# Check and insert labels if missing
|
||||
if not values or values[0] != COLUMN_LABELS:
|
||||
if dry_run:
|
||||
print("Dry run: would write header row")
|
||||
else:
|
||||
print("Inserting column labels...")
|
||||
sheet.values().update(
|
||||
spreadsheetId=spreadsheet_id,
|
||||
range="A1",
|
||||
valueInputOption="USER_ENTERED",
|
||||
body={"values": [COLUMN_LABELS]}
|
||||
).execute()
|
||||
existing_ids = set()
|
||||
else:
|
||||
# Sync ID is the last column (index 10)
|
||||
existing_ids = {row[10] for row in values[1:] if len(row) > 10}
|
||||
except Exception as e:
|
||||
print(f"Error reading sheet (maybe empty?): {e}")
|
||||
existing_ids = set()
|
||||
|
||||
# 2. Fetch Fio transactions
|
||||
if date_from_str and date_to_str:
|
||||
df_str = date_from_str
|
||||
dt_str = date_to_str
|
||||
else:
|
||||
now = datetime.now()
|
||||
date_to = now
|
||||
date_from = now - timedelta(days=days or 30)
|
||||
df_str = date_from.strftime("%Y-%m-%d")
|
||||
dt_str = date_to.strftime("%Y-%m-%d")
|
||||
|
||||
print(f"Fetching Fio transactions from {df_str} to {dt_str}...")
|
||||
transactions = fetch_transactions_all(df_str, dt_str)
|
||||
print(f"Found {len(transactions)} transactions.")
|
||||
|
||||
if dry_run:
|
||||
print(f"Dry run: window {df_str} to {dt_str}, fetched {len(transactions)} transaction(s) from Fio")
|
||||
|
||||
# 3. Determine NEW/DUP for each transaction
|
||||
new_rows = []
|
||||
tx_statuses = []
|
||||
for tx in transactions:
|
||||
sync_id = generate_sync_id(tx)
|
||||
if sync_id not in existing_ids:
|
||||
# Schema: Date | Amount | Manual | Person | Purpose | Inferred Amount | Sender | VS | Message | Bank ID | Sync ID
|
||||
new_rows.append([
|
||||
tx.get("date", ""),
|
||||
tx.get("amount", ""),
|
||||
"", # Manual
|
||||
"", # Person
|
||||
"", # Purpose
|
||||
"", # Inferred Amount
|
||||
tx.get("sender", ""),
|
||||
tx.get("vs", ""),
|
||||
tx.get("message", ""),
|
||||
tx.get("bank_id", ""),
|
||||
sync_id,
|
||||
])
|
||||
tx_statuses.append("NEW")
|
||||
else:
|
||||
tx_statuses.append("DUP")
|
||||
|
||||
# 4. Print table (before early-return so all transactions are shown including DUPs)
|
||||
if print_fio_table and transactions:
|
||||
_print_fio_table(transactions, tx_statuses)
|
||||
|
||||
if not new_rows:
|
||||
if dry_run:
|
||||
print("Dry run: would sync 0 new transaction(s).")
|
||||
else:
|
||||
print("No new transactions to sync.")
|
||||
return
|
||||
|
||||
# 5. Append to sheet or print dry-run would-write lines
|
||||
if dry_run:
|
||||
for tx, status in zip(transactions, tx_statuses):
|
||||
if status == "NEW":
|
||||
print(
|
||||
f"Dry run: would append"
|
||||
f" date={tx.get('date', '')}"
|
||||
f" amount={tx.get('amount', '')}"
|
||||
f" sender={tx.get('sender', '')}"
|
||||
f" vs={tx.get('vs', '')}"
|
||||
f" message={tx.get('message', '')}"
|
||||
)
|
||||
if sort_by_date:
|
||||
print("Dry run: would sort by date")
|
||||
print(f"Dry run: would sync {len(new_rows)} new transaction(s).")
|
||||
else:
|
||||
print(f"Appending {len(new_rows)} new transactions to the sheet...")
|
||||
body = {"values": new_rows}
|
||||
sheet.values().append(
|
||||
spreadsheetId=spreadsheet_id,
|
||||
range="A2", # Appends to the end of the sheet
|
||||
valueInputOption="USER_ENTERED",
|
||||
body=body
|
||||
).execute()
|
||||
print("Sync completed successfully.")
|
||||
if sort_by_date:
|
||||
sort_sheet_by_date(service, spreadsheet_id)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Sync Fio transactions to Google Sheets.")
|
||||
parser.add_argument("--days", type=int, default=30, help="Days to look back (default: 30)")
|
||||
parser.add_argument("--sheet-id", default=DEFAULT_SPREADSHEET_ID, help="Google Sheet ID")
|
||||
parser.add_argument("--credentials", default="credentials.json", help="Path to Google API credentials JSON")
|
||||
parser.add_argument("--from", dest="date_from", help="Start date YYYY-MM-DD")
|
||||
parser.add_argument("--to", dest="date_to", help="End date YYYY-MM-DD")
|
||||
parser.add_argument("--sort-by-date", action="store_true", help="Sort the sheet by date after sync")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Fetch and dedup without writing to the sheet")
|
||||
parser.add_argument("--print-fio-table", action="store_true", help="Print aligned table of all fetched transactions with NEW/DUP status (use with --dry-run)")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
sync_to_sheets(
|
||||
spreadsheet_id=args.sheet_id,
|
||||
credentials_path=args.credentials,
|
||||
days=args.days,
|
||||
date_from_str=args.date_from,
|
||||
date_to_str=args.date_to,
|
||||
sort_by_date=args.sort_by_date,
|
||||
dry_run=args.dry_run,
|
||||
print_fio_table=args.print_fio_table,
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"Sync failed: {e}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,446 @@
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime
|
||||
|
||||
from attendance import ADULT_MERGED_MONTHS, JUNIOR_MERGED_MONTHS
|
||||
from match_payments import canonical_member_key
|
||||
|
||||
|
||||
def get_month_labels(sorted_months, merged_months):
|
||||
labels = {}
|
||||
for m in sorted_months:
|
||||
dt = datetime.strptime(m, "%Y-%m")
|
||||
merged_in = sorted([k for k, v in merged_months.items() if v == m])
|
||||
if merged_in:
|
||||
all_dts = [datetime.strptime(x, "%Y-%m") for x in sorted(merged_in + [m])]
|
||||
years = {d.year for d in all_dts}
|
||||
if len(years) > 1:
|
||||
parts = [d.strftime("%b %Y") for d in all_dts]
|
||||
labels[m] = "+".join(parts)
|
||||
else:
|
||||
parts = [d.strftime("%b") for d in all_dts]
|
||||
labels[m] = f"{'+'.join(parts)} {dt.strftime('%Y')}"
|
||||
else:
|
||||
labels[m] = dt.strftime("%b %Y")
|
||||
return labels
|
||||
|
||||
|
||||
def group_payments_by_person(transactions, member_names=None):
|
||||
canonical_by_key = (
|
||||
{canonical_member_key(n): n for n in member_names} if member_names else {}
|
||||
)
|
||||
grouped = {}
|
||||
for tx in transactions:
|
||||
person = str(tx.get("person", "")).strip()
|
||||
if not person:
|
||||
continue
|
||||
for p in person.split(","):
|
||||
p = re.sub(r"\[\?\]\s*", "", p).strip()
|
||||
if not p:
|
||||
continue
|
||||
key = canonical_by_key.get(canonical_member_key(p), p)
|
||||
grouped.setdefault(key, []).append(tx)
|
||||
for rows in grouped.values():
|
||||
rows.sort(key=lambda t: str(t.get("date", "")), reverse=True)
|
||||
return grouped
|
||||
|
||||
|
||||
def adapt_junior_members(junior_members):
|
||||
"""Convert 4-tuple junior fee data to (fee, total_count) for reconcile."""
|
||||
adapted = []
|
||||
for name, tier, fees_dict in junior_members:
|
||||
adapted_fees = {}
|
||||
for m, fee_data in fees_dict.items():
|
||||
if len(fee_data) == 4:
|
||||
fee, total_count, _, _ = fee_data
|
||||
adapted_fees[m] = (fee, total_count)
|
||||
else:
|
||||
fee, count = fee_data
|
||||
adapted_fees[m] = (fee, count)
|
||||
adapted.append((name, tier, adapted_fees))
|
||||
return adapted
|
||||
|
||||
|
||||
def build_adults_view_model(
|
||||
members,
|
||||
sorted_months,
|
||||
result,
|
||||
transactions,
|
||||
current_month,
|
||||
*,
|
||||
attendance_url,
|
||||
payments_url,
|
||||
bank_account,
|
||||
):
|
||||
month_labels = get_month_labels(sorted_months, ADULT_MERGED_MONTHS)
|
||||
adult_names = sorted([name for name, tier, _ in members if tier == "A"])
|
||||
|
||||
monthly_totals = {m: {"expected": 0, "paid": 0} for m in sorted_months}
|
||||
formatted_results = []
|
||||
for name in adult_names:
|
||||
data = result["members"][name]
|
||||
row = {
|
||||
"name": name,
|
||||
"months": [],
|
||||
"balance": data["total_balance"],
|
||||
"unpaid_periods": "",
|
||||
"raw_unpaid_periods": "",
|
||||
}
|
||||
unpaid_months = []
|
||||
raw_unpaid_months = []
|
||||
for m in sorted_months:
|
||||
mdata = data["months"].get(m, {
|
||||
"expected": 0, "original_expected": 0,
|
||||
"attendance_count": 0, "paid": 0, "exception": None,
|
||||
})
|
||||
expected = mdata.get("expected", 0)
|
||||
original_expected = mdata.get("original_expected", 0)
|
||||
count = mdata.get("attendance_count", 0)
|
||||
paid = int(mdata.get("paid", 0))
|
||||
exception_info = mdata.get("exception", None)
|
||||
|
||||
monthly_totals[m]["expected"] += expected
|
||||
monthly_totals[m]["paid"] += paid
|
||||
|
||||
override_amount = exception_info["amount"] if exception_info else None
|
||||
|
||||
if override_amount is not None and override_amount != original_expected:
|
||||
is_overridden = True
|
||||
fee_display = (
|
||||
f"{override_amount} ({original_expected}) CZK ({count})"
|
||||
if count > 0
|
||||
else f"{override_amount} ({original_expected}) CZK"
|
||||
)
|
||||
else:
|
||||
is_overridden = False
|
||||
fee_display = (
|
||||
f"{expected} CZK ({count})" if count > 0 else f"{expected} CZK"
|
||||
)
|
||||
|
||||
status = "empty"
|
||||
cell_text = "-"
|
||||
amount_to_pay = 0
|
||||
|
||||
if expected > 0:
|
||||
amount_to_pay = max(0, expected - paid)
|
||||
if paid >= expected:
|
||||
status = "ok"
|
||||
cell_text = f"{paid}/{fee_display}"
|
||||
elif paid > 0:
|
||||
status = "partial"
|
||||
cell_text = f"{paid}/{fee_display}"
|
||||
if m < current_month:
|
||||
unpaid_months.append(month_labels[m])
|
||||
raw_unpaid_months.append(
|
||||
datetime.strptime(m, "%Y-%m").strftime("%m/%Y")
|
||||
)
|
||||
else:
|
||||
status = "unpaid"
|
||||
cell_text = f"0/{fee_display}"
|
||||
if m < current_month:
|
||||
unpaid_months.append(month_labels[m])
|
||||
raw_unpaid_months.append(
|
||||
datetime.strptime(m, "%Y-%m").strftime("%m/%Y")
|
||||
)
|
||||
elif paid > 0:
|
||||
status = "surplus"
|
||||
cell_text = f"PAID {paid}"
|
||||
else:
|
||||
cell_text = "-"
|
||||
amount_to_pay = 0
|
||||
|
||||
if expected > 0 or paid > 0:
|
||||
tooltip = f"Received: {paid}, Expected: {expected}"
|
||||
else:
|
||||
tooltip = ""
|
||||
|
||||
row["months"].append({
|
||||
"text": cell_text,
|
||||
"overridden": is_overridden,
|
||||
"status": status,
|
||||
"amount": amount_to_pay,
|
||||
"month": month_labels[m],
|
||||
"raw_month": m,
|
||||
"tooltip": tooltip,
|
||||
})
|
||||
|
||||
settled_balance = 0
|
||||
for m, mdata in data["months"].items():
|
||||
if m >= current_month:
|
||||
continue
|
||||
exp = mdata.get("expected", 0)
|
||||
if isinstance(exp, int):
|
||||
settled_balance += int(mdata.get("paid", 0)) - exp
|
||||
|
||||
payable_amount = max(0, -settled_balance)
|
||||
row["unpaid_periods"] = ", ".join(unpaid_months)
|
||||
row["raw_unpaid_periods"] = "+".join(raw_unpaid_months)
|
||||
row["balance"] = settled_balance
|
||||
row["payable_amount"] = payable_amount
|
||||
formatted_results.append(row)
|
||||
|
||||
formatted_totals = []
|
||||
for m in sorted_months:
|
||||
t = monthly_totals[m]
|
||||
status = "empty"
|
||||
if t["expected"] > 0 or t["paid"] > 0:
|
||||
if t["paid"] == t["expected"]:
|
||||
status = "ok"
|
||||
elif t["paid"] < t["expected"]:
|
||||
status = "unpaid"
|
||||
else:
|
||||
status = "surplus"
|
||||
formatted_totals.append({
|
||||
"text": f"{t['paid']} / {t['expected']} CZK",
|
||||
"status": status,
|
||||
})
|
||||
|
||||
def _settled_balance(name):
|
||||
data = result["members"][name]
|
||||
total = 0
|
||||
for m, mdata in data["months"].items():
|
||||
if m >= current_month:
|
||||
continue
|
||||
exp = mdata.get("expected", 0)
|
||||
if isinstance(exp, int):
|
||||
total += int(mdata.get("paid", 0)) - exp
|
||||
return total
|
||||
|
||||
credits = sorted(
|
||||
[{"name": n, "amount": _settled_balance(n)} for n in adult_names if _settled_balance(n) > 0],
|
||||
key=lambda x: x["name"],
|
||||
)
|
||||
debts = sorted(
|
||||
[{"name": n, "amount": abs(_settled_balance(n))} for n in adult_names if _settled_balance(n) < 0],
|
||||
key=lambda x: x["name"],
|
||||
)
|
||||
|
||||
raw_payments_by_person = group_payments_by_person(
|
||||
transactions, [name for name, _, _ in members]
|
||||
)
|
||||
|
||||
return dict(
|
||||
months=[month_labels[m] for m in sorted_months],
|
||||
raw_months=sorted_months,
|
||||
results=formatted_results,
|
||||
totals=formatted_totals,
|
||||
member_data=json.dumps(result["members"]),
|
||||
month_labels_json=json.dumps(month_labels),
|
||||
raw_payments_json=json.dumps(raw_payments_by_person),
|
||||
credits=credits,
|
||||
debts=debts,
|
||||
unmatched=result["unmatched"],
|
||||
attendance_url=attendance_url,
|
||||
payments_url=payments_url,
|
||||
bank_account=bank_account,
|
||||
current_month=current_month,
|
||||
)
|
||||
|
||||
|
||||
def build_juniors_view_model(
|
||||
junior_members,
|
||||
adapted_members,
|
||||
sorted_months,
|
||||
result,
|
||||
transactions,
|
||||
current_month,
|
||||
*,
|
||||
attendance_url,
|
||||
payments_url,
|
||||
bank_account,
|
||||
):
|
||||
month_labels = get_month_labels(sorted_months, JUNIOR_MERGED_MONTHS)
|
||||
junior_names = sorted([name for name, tier, _ in adapted_members])
|
||||
junior_members_dict = {name: fees_dict for name, _, fees_dict in junior_members}
|
||||
|
||||
monthly_totals = {m: {"expected": 0, "paid": 0} for m in sorted_months}
|
||||
formatted_results = []
|
||||
for name in junior_names:
|
||||
data = result["members"][name]
|
||||
row = {
|
||||
"name": name,
|
||||
"months": [],
|
||||
"balance": data["total_balance"],
|
||||
"unpaid_periods": "",
|
||||
"raw_unpaid_periods": "",
|
||||
}
|
||||
unpaid_months = []
|
||||
raw_unpaid_months = []
|
||||
for m in sorted_months:
|
||||
mdata = data["months"].get(m, {
|
||||
"expected": 0, "original_expected": 0,
|
||||
"attendance_count": 0, "paid": 0, "exception": None,
|
||||
})
|
||||
expected = mdata.get("expected", 0)
|
||||
original_expected = mdata.get("original_expected", 0)
|
||||
count = mdata.get("attendance_count", 0)
|
||||
paid = int(mdata.get("paid", 0))
|
||||
exception_info = mdata.get("exception", None)
|
||||
|
||||
if expected != "?" and isinstance(expected, int):
|
||||
monthly_totals[m]["expected"] += expected
|
||||
monthly_totals[m]["paid"] += paid
|
||||
|
||||
orig_fee_data = junior_members_dict.get(name, {}).get(m)
|
||||
adult_count = 0
|
||||
junior_count = 0
|
||||
if orig_fee_data and len(orig_fee_data) == 4:
|
||||
_, _, adult_count, junior_count = orig_fee_data
|
||||
|
||||
breakdown = ""
|
||||
if adult_count > 0 and junior_count > 0:
|
||||
breakdown = f":{junior_count}J,{adult_count}A"
|
||||
elif junior_count > 0:
|
||||
breakdown = f":{junior_count}J"
|
||||
elif adult_count > 0:
|
||||
breakdown = f":{adult_count}A"
|
||||
|
||||
count_str = f" ({count}{breakdown})" if count > 0 else ""
|
||||
|
||||
override_amount = exception_info["amount"] if exception_info else None
|
||||
|
||||
if override_amount is not None and override_amount != original_expected:
|
||||
is_overridden = True
|
||||
fee_display = f"{override_amount} ({original_expected}) CZK{count_str}"
|
||||
else:
|
||||
is_overridden = False
|
||||
fee_display = f"{expected} CZK{count_str}"
|
||||
|
||||
status = "empty"
|
||||
cell_text = "-"
|
||||
amount_to_pay = 0
|
||||
|
||||
is_unknown = original_expected == "?"
|
||||
if is_unknown or (isinstance(expected, int) and expected > 0):
|
||||
if is_unknown:
|
||||
status = "empty"
|
||||
cell_text = f"?{count_str}"
|
||||
elif paid >= expected:
|
||||
status = "ok"
|
||||
cell_text = f"{paid}/{fee_display}"
|
||||
elif paid > 0:
|
||||
status = "partial"
|
||||
cell_text = f"{paid}/{fee_display}"
|
||||
amount_to_pay = expected - paid
|
||||
if m < current_month:
|
||||
unpaid_months.append(month_labels[m])
|
||||
raw_unpaid_months.append(
|
||||
datetime.strptime(m, "%Y-%m").strftime("%m/%Y")
|
||||
)
|
||||
else:
|
||||
status = "unpaid"
|
||||
cell_text = f"0/{fee_display}"
|
||||
amount_to_pay = expected
|
||||
if m < current_month:
|
||||
unpaid_months.append(month_labels[m])
|
||||
raw_unpaid_months.append(
|
||||
datetime.strptime(m, "%Y-%m").strftime("%m/%Y")
|
||||
)
|
||||
elif paid > 0:
|
||||
status = "surplus"
|
||||
cell_text = f"PAID {paid}"
|
||||
|
||||
if (not is_unknown and isinstance(expected, int) and expected > 0) or paid > 0:
|
||||
tooltip = f"Received: {paid}, Expected: {expected}"
|
||||
else:
|
||||
tooltip = ""
|
||||
|
||||
row["months"].append({
|
||||
"text": cell_text,
|
||||
"overridden": is_overridden,
|
||||
"status": status,
|
||||
"amount": amount_to_pay,
|
||||
"month": month_labels[m],
|
||||
"raw_month": m,
|
||||
"tooltip": tooltip,
|
||||
})
|
||||
|
||||
settled_balance = 0
|
||||
for m, mdata in data["months"].items():
|
||||
if m >= current_month:
|
||||
continue
|
||||
exp = mdata.get("expected", 0)
|
||||
if isinstance(exp, int):
|
||||
settled_balance += int(mdata.get("paid", 0)) - exp
|
||||
|
||||
payable_amount = max(0, -settled_balance)
|
||||
row["unpaid_periods"] = ", ".join(unpaid_months)
|
||||
row["raw_unpaid_periods"] = "+".join(raw_unpaid_months)
|
||||
row["balance"] = settled_balance
|
||||
row["payable_amount"] = payable_amount
|
||||
formatted_results.append(row)
|
||||
|
||||
formatted_totals = []
|
||||
for m in sorted_months:
|
||||
t = monthly_totals[m]
|
||||
status = "empty"
|
||||
if t["expected"] > 0 or t["paid"] > 0:
|
||||
if t["paid"] == t["expected"]:
|
||||
status = "ok"
|
||||
elif t["paid"] < t["expected"]:
|
||||
status = "unpaid"
|
||||
else:
|
||||
status = "surplus"
|
||||
formatted_totals.append({
|
||||
"text": f"{t['paid']} / {t['expected']} CZK",
|
||||
"status": status,
|
||||
})
|
||||
|
||||
junior_all_names = [name for name, _, _ in adapted_members]
|
||||
|
||||
def _junior_settled_balance(name):
|
||||
data = result["members"][name]
|
||||
total = 0
|
||||
for m, mdata in data["months"].items():
|
||||
if m >= current_month:
|
||||
continue
|
||||
exp = mdata.get("expected", 0)
|
||||
if isinstance(exp, int):
|
||||
total += int(mdata.get("paid", 0)) - exp
|
||||
return total
|
||||
|
||||
credits = sorted(
|
||||
[{"name": n, "amount": _junior_settled_balance(n)} for n in junior_all_names if _junior_settled_balance(n) > 0],
|
||||
key=lambda x: x["name"],
|
||||
)
|
||||
debts = sorted(
|
||||
[{"name": n, "amount": abs(_junior_settled_balance(n))} for n in junior_all_names if _junior_settled_balance(n) < 0],
|
||||
key=lambda x: x["name"],
|
||||
)
|
||||
|
||||
raw_payments_by_person = group_payments_by_person(
|
||||
transactions, [name for name, _, _ in adapted_members]
|
||||
)
|
||||
|
||||
return dict(
|
||||
months=[month_labels[m] for m in sorted_months],
|
||||
raw_months=sorted_months,
|
||||
results=formatted_results,
|
||||
totals=formatted_totals,
|
||||
member_data=json.dumps(result["members"]),
|
||||
month_labels_json=json.dumps(month_labels),
|
||||
raw_payments_json=json.dumps(raw_payments_by_person),
|
||||
credits=credits,
|
||||
debts=debts,
|
||||
unmatched=result["unmatched"],
|
||||
attendance_url=attendance_url,
|
||||
payments_url=payments_url,
|
||||
bank_account=bank_account,
|
||||
current_month=current_month,
|
||||
)
|
||||
|
||||
|
||||
def build_payments_view_model(transactions, member_names, *, attendance_url, payments_url):
|
||||
grouped = group_payments_by_person(transactions, member_names)
|
||||
for tx in transactions:
|
||||
if not str(tx.get("person", "")).strip():
|
||||
grouped.setdefault("Unmatched / Unknown", []).append(tx)
|
||||
for rows in grouped.values():
|
||||
rows.sort(key=lambda t: str(t.get("date", "")), reverse=True)
|
||||
sorted_people = sorted(grouped.keys())
|
||||
return dict(
|
||||
grouped_payments=grouped,
|
||||
sorted_people=sorted_people,
|
||||
attendance_url=attendance_url,
|
||||
payments_url=payments_url,
|
||||
)
|
||||
Reference in New Issue
Block a user