Compare commits
11 Commits
feat/m4-io
...
59223c0da4
| Author | SHA1 | Date | |
|---|---|---|---|
| 59223c0da4 | |||
| 32a16ff50d | |||
| 2eec51bb34 | |||
| b562ce3201 | |||
| f0de300292 | |||
| 2164e99866 | |||
| b41b8ef29c | |||
| 80db33945d | |||
| f87adeff9f | |||
| a7cf45fc95 | |||
| f0a0f79475 |
14
CHANGELOG.md
14
CHANGELOG.md
@@ -1,5 +1,19 @@
|
||||
# Changelog
|
||||
|
||||
## 2026-05-07 15:26 CEST — refactor(app): extract view-model builders into scripts/views.py
|
||||
|
||||
- Pulled ~350 lines of inline per-row computation out of `adults_view`, `juniors_view`, and `payments` into three pure functions in `scripts/views.py`: `build_adults_view_model`, `build_juniors_view_model`, `build_payments_view_model`.
|
||||
- Moved `get_month_labels`, `group_payments_by_person`, `adapt_junior_members` from `app.py` to `scripts/views.py`. Route handlers now ~25 lines each.
|
||||
- Hotfixed missing `import re` that caused 500 on `/qr` after the refactor.
|
||||
- No behaviour change; all 27 tests pass. Prep for `/api/*` shadow endpoints (M5).
|
||||
|
||||
## 2026-05-07 14:13 CEST — feat(go): --print-fio-table + Fio debug logging + date parser fix
|
||||
|
||||
- Added `--print-fio-table` flag to `fuj sync --dry-run`: prints an aligned table of every Fio transaction in the window with `STATUS=NEW/DUP`, using `text/tabwriter`. Key files: `go/internal/services/banksync/fio_table.go`, `sync.go`, `cmd/fuj/main.go`.
|
||||
- Added `LOG_LEVEL=DEBUG` debug logging on the Fio fetch path: client variant selected, full GET URL (token redacted on API path), HTTP status, body bytes, and per-parse drop-reason counters (`raw_rows`, `kept`, `dropped_bad_date`, `dropped_nonpositive_amount`). Key files: `go/internal/io/fio/{client,api,transparent}.go`.
|
||||
- Fixed `parseCzechDate` to accept `DD.MM.YY` (2-digit year) in addition to the 4-digit variant — Fio's transparent page now serves this format. Key file: `go/internal/io/fio/transparent.go`.
|
||||
- Added `make go-sync-debug [DAYS=N]` Makefile target (default 30 days).
|
||||
|
||||
## 2026-05-07 10:32 CEST — feat(go): --dry-run for fuj sync
|
||||
|
||||
- `SyncOpts.DryRun bool` added; when true, `SyncToSheets` prints planned writes (`would write header row`, `would append date=… amount=… sender=…`, `would sort by date`) and returns without calling `WriteHeader`, `AppendValues`, or `SortByDateColumn`.
|
||||
|
||||
7
Makefile
7
Makefile
@@ -1,4 +1,4 @@
|
||||
.PHONY: help fees match web web-py web-debug web-go go-build go-test go-test-all go-parity go-run go-lint capture-fixtures image run sync sync-2026 test test-v docs
|
||||
.PHONY: help fees match web web-py web-debug web-go go-build go-test go-test-all go-parity go-run go-sync-debug go-lint capture-fixtures image run sync sync-2026 test test-v docs
|
||||
|
||||
export PYTHONPATH := scripts:$(PYTHONPATH)
|
||||
VENV := .venv
|
||||
@@ -27,6 +27,7 @@ help:
|
||||
@echo " make go-parity - Run Go parity tests (requires -tags=parity fixture corpus)"
|
||||
@echo " make go-test-all - Run both unit and parity tests"
|
||||
@echo " make go-lint - Run golangci-lint on Go code"
|
||||
@echo " make go-sync-debug [DAYS=N] - Dry-run Go sync with Fio debug logs and txn table (default DAYS=30)"
|
||||
@echo " make capture-fixtures - Regenerate parity fixture corpus from live Python"
|
||||
@echo " make image - Build Python OCI container image"
|
||||
@echo " make run - Run the built Python Docker image locally"
|
||||
@@ -91,6 +92,10 @@ capture-fixtures: $(PYTHON)
|
||||
go-run: go-build
|
||||
./$(GO_BIN) $(ARGS)
|
||||
|
||||
DAYS ?= 30
|
||||
go-sync-debug: go-build
|
||||
LOG_LEVEL=DEBUG ./$(GO_BIN) sync -dry-run -print-fio-table -days $(DAYS)
|
||||
|
||||
go-lint:
|
||||
cd $(GO_SRC) && golangci-lint run ./...
|
||||
|
||||
|
||||
414
app.py
414
app.py
@@ -21,8 +21,14 @@ from config import (
|
||||
ATTENDANCE_SHEET_ID, PAYMENTS_SHEET_ID, JUNIOR_SHEET_GID,
|
||||
BANK_ACCOUNT, CREDENTIALS_PATH,
|
||||
)
|
||||
from attendance import get_members_with_fees, get_junior_members_with_fees, ADULT_MERGED_MONTHS, JUNIOR_MERGED_MONTHS
|
||||
from match_payments import reconcile, fetch_sheet_data, fetch_exceptions, normalize, canonical_member_key
|
||||
from attendance import get_members_with_fees, get_junior_members_with_fees
|
||||
from match_payments import reconcile, fetch_sheet_data, fetch_exceptions
|
||||
from views import (
|
||||
build_adults_view_model,
|
||||
build_juniors_view_model,
|
||||
build_payments_view_model,
|
||||
adapt_junior_members,
|
||||
)
|
||||
from cache_utils import get_sheet_modified_time, read_cache, write_cache, _LAST_CHECKED, flush_cache
|
||||
from sync_fio_to_sheets import sync_to_sheets
|
||||
from infer_payments import infer_payments
|
||||
@@ -38,44 +44,6 @@ def get_cached_data(cache_key, sheet_id, fetch_func, *args, serialize=None, dese
|
||||
write_cache(cache_key, mod_time, serialize(data) if serialize else data)
|
||||
return data
|
||||
|
||||
def get_month_labels(sorted_months, merged_months):
|
||||
labels = {}
|
||||
for m in sorted_months:
|
||||
dt = datetime.strptime(m, "%Y-%m")
|
||||
# Find which months were merged into m (e.g. 2026-01 is merged into 2026-02)
|
||||
merged_in = sorted([k for k, v in merged_months.items() if v == m])
|
||||
if merged_in:
|
||||
all_dts = [datetime.strptime(x, "%Y-%m") for x in sorted(merged_in + [m])]
|
||||
years = {d.year for d in all_dts}
|
||||
if len(years) > 1:
|
||||
parts = [d.strftime("%b %Y") for d in all_dts]
|
||||
labels[m] = "+".join(parts)
|
||||
else:
|
||||
parts = [d.strftime("%b") for d in all_dts]
|
||||
labels[m] = f"{'+'.join(parts)} {dt.strftime('%Y')}"
|
||||
else:
|
||||
labels[m] = dt.strftime("%b %Y")
|
||||
return labels
|
||||
|
||||
def group_payments_by_person(transactions, member_names=None):
|
||||
canonical_by_key = (
|
||||
{canonical_member_key(n): n for n in member_names} if member_names else {}
|
||||
)
|
||||
grouped = {}
|
||||
for tx in transactions:
|
||||
person = str(tx.get("person", "")).strip()
|
||||
if not person:
|
||||
continue
|
||||
for p in person.split(","):
|
||||
p = re.sub(r"\[\?\]\s*", "", p).strip()
|
||||
if not p:
|
||||
continue
|
||||
key = canonical_by_key.get(canonical_member_key(p), p)
|
||||
grouped.setdefault(key, []).append(tx)
|
||||
for rows in grouped.values():
|
||||
rows.sort(key=lambda t: str(t.get("date", "")), reverse=True)
|
||||
return grouped
|
||||
|
||||
def warmup_cache():
|
||||
"""Pre-fetch all cached data so first request is fast."""
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -191,164 +159,29 @@ def adults_view():
|
||||
transactions = get_cached_data("payments_transactions", PAYMENTS_SHEET_ID, fetch_sheet_data, PAYMENTS_SHEET_ID, credentials_path)
|
||||
record_step("fetch_payments")
|
||||
exceptions = get_cached_data(
|
||||
"exceptions_dict", PAYMENTS_SHEET_ID, fetch_exceptions,
|
||||
PAYMENTS_SHEET_ID, credentials_path,
|
||||
serialize=lambda d: [[list(k), v] for k, v in d.items()],
|
||||
deserialize=lambda c: {tuple(k): v for k, v in c},
|
||||
)
|
||||
"exceptions_dict", PAYMENTS_SHEET_ID, fetch_exceptions,
|
||||
PAYMENTS_SHEET_ID, credentials_path,
|
||||
serialize=lambda d: [[list(k), v] for k, v in d.items()],
|
||||
deserialize=lambda c: {tuple(k): v for k, v in c},
|
||||
)
|
||||
record_step("fetch_exceptions")
|
||||
result = reconcile(members, sorted_months, transactions, exceptions)
|
||||
record_step("reconcile")
|
||||
|
||||
month_labels = get_month_labels(sorted_months, ADULT_MERGED_MONTHS)
|
||||
adult_names = sorted([name for name, tier, _ in members if tier == "A"])
|
||||
current_month = datetime.now().strftime("%Y-%m")
|
||||
|
||||
monthly_totals = {m: {"expected": 0, "paid": 0} for m in sorted_months}
|
||||
formatted_results = []
|
||||
for name in adult_names:
|
||||
data = result["members"][name]
|
||||
row = {"name": name, "months": [], "balance": data["total_balance"], "unpaid_periods": "", "raw_unpaid_periods": ""}
|
||||
unpaid_months = []
|
||||
raw_unpaid_months = []
|
||||
for m in sorted_months:
|
||||
mdata = data["months"].get(m, {"expected": 0, "original_expected": 0, "attendance_count": 0, "paid": 0, "exception": None})
|
||||
expected = mdata.get("expected", 0)
|
||||
original_expected = mdata.get("original_expected", 0)
|
||||
count = mdata.get("attendance_count", 0)
|
||||
paid = int(mdata.get("paid", 0))
|
||||
exception_info = mdata.get("exception", None)
|
||||
|
||||
monthly_totals[m]["expected"] += expected
|
||||
monthly_totals[m]["paid"] += paid
|
||||
|
||||
override_amount = exception_info["amount"] if exception_info else None
|
||||
|
||||
if override_amount is not None and override_amount != original_expected:
|
||||
is_overridden = True
|
||||
fee_display = f"{override_amount} ({original_expected}) CZK ({count})" if count > 0 else f"{override_amount} ({original_expected}) CZK"
|
||||
else:
|
||||
is_overridden = False
|
||||
fee_display = f"{expected} CZK ({count})" if count > 0 else f"{expected} CZK"
|
||||
|
||||
status = "empty"
|
||||
cell_text = "-"
|
||||
amount_to_pay = 0
|
||||
|
||||
if expected > 0:
|
||||
amount_to_pay = max(0, expected - paid)
|
||||
if paid >= expected:
|
||||
status = "ok"
|
||||
cell_text = f"{paid}/{fee_display}"
|
||||
elif paid > 0:
|
||||
status = "partial"
|
||||
cell_text = f"{paid}/{fee_display}"
|
||||
if m < current_month:
|
||||
unpaid_months.append(month_labels[m])
|
||||
raw_unpaid_months.append(datetime.strptime(m, "%Y-%m").strftime("%m/%Y"))
|
||||
else:
|
||||
status = "unpaid"
|
||||
cell_text = f"0/{fee_display}"
|
||||
if m < current_month:
|
||||
unpaid_months.append(month_labels[m])
|
||||
raw_unpaid_months.append(datetime.strptime(m, "%Y-%m").strftime("%m/%Y"))
|
||||
elif paid > 0:
|
||||
status = "surplus"
|
||||
cell_text = f"PAID {paid}"
|
||||
else:
|
||||
cell_text = "-"
|
||||
amount_to_pay = 0
|
||||
|
||||
if expected > 0 or paid > 0:
|
||||
tooltip = f"Received: {paid}, Expected: {expected}"
|
||||
else:
|
||||
tooltip = ""
|
||||
|
||||
row["months"].append({
|
||||
"text": cell_text,
|
||||
"overridden": is_overridden,
|
||||
"status": status,
|
||||
"amount": amount_to_pay,
|
||||
"month": month_labels[m],
|
||||
"raw_month": m,
|
||||
"tooltip": tooltip
|
||||
})
|
||||
|
||||
# Balance = sum of (paid - expected) for past months only; current/future months ignored.
|
||||
settled_balance = 0
|
||||
for m, mdata in data["months"].items():
|
||||
if m >= current_month:
|
||||
continue
|
||||
exp = mdata.get("expected", 0)
|
||||
if isinstance(exp, int):
|
||||
settled_balance += int(mdata.get("paid", 0)) - exp
|
||||
|
||||
payable_amount = max(0, -settled_balance)
|
||||
row["unpaid_periods"] = ", ".join(unpaid_months)
|
||||
row["raw_unpaid_periods"] = "+".join(raw_unpaid_months)
|
||||
row["balance"] = settled_balance
|
||||
row["payable_amount"] = payable_amount
|
||||
formatted_results.append(row)
|
||||
|
||||
formatted_totals = []
|
||||
for m in sorted_months:
|
||||
t = monthly_totals[m]
|
||||
status = "empty"
|
||||
if t["expected"] > 0 or t["paid"] > 0:
|
||||
if t["paid"] == t["expected"]:
|
||||
status = "ok"
|
||||
elif t["paid"] < t["expected"]:
|
||||
status = "unpaid"
|
||||
else:
|
||||
status = "surplus"
|
||||
|
||||
formatted_totals.append({
|
||||
"text": f"{t['paid']} / {t['expected']} CZK",
|
||||
"status": status
|
||||
})
|
||||
|
||||
def settled_balance(name):
|
||||
data = result["members"][name]
|
||||
total = 0
|
||||
for m, mdata in data["months"].items():
|
||||
if m >= current_month:
|
||||
continue
|
||||
exp = mdata.get("expected", 0)
|
||||
if isinstance(exp, int):
|
||||
total += int(mdata.get("paid", 0)) - exp
|
||||
return total
|
||||
|
||||
credits = sorted([{"name": n, "amount": settled_balance(n)} for n in adult_names if settled_balance(n) > 0], key=lambda x: x["name"])
|
||||
debts = sorted([{"name": n, "amount": abs(settled_balance(n))} for n in adult_names if settled_balance(n) < 0], key=lambda x: x["name"])
|
||||
unmatched = result["unmatched"]
|
||||
import json
|
||||
|
||||
raw_payments_by_person = group_payments_by_person(transactions, [name for name, _, _ in members])
|
||||
record_step("process_data")
|
||||
|
||||
return render_template(
|
||||
"adults.html",
|
||||
months=[month_labels[m] for m in sorted_months],
|
||||
raw_months=sorted_months,
|
||||
results=formatted_results,
|
||||
totals=formatted_totals,
|
||||
member_data=json.dumps(result["members"]),
|
||||
month_labels_json=json.dumps(month_labels),
|
||||
raw_payments_json=json.dumps(raw_payments_by_person),
|
||||
credits=credits,
|
||||
debts=debts,
|
||||
unmatched=unmatched,
|
||||
vm = build_adults_view_model(
|
||||
members, sorted_months, result, transactions,
|
||||
datetime.now().strftime("%Y-%m"),
|
||||
attendance_url=attendance_url,
|
||||
payments_url=payments_url,
|
||||
bank_account=BANK_ACCOUNT,
|
||||
current_month=current_month
|
||||
)
|
||||
record_step("process_data")
|
||||
return render_template("adults.html", **vm)
|
||||
|
||||
@app.route("/juniors")
|
||||
def juniors_view():
|
||||
attendance_url = f"https://docs.google.com/spreadsheets/d/{ATTENDANCE_SHEET_ID}/edit#gid={JUNIOR_SHEET_GID}"
|
||||
payments_url = f"https://docs.google.com/spreadsheets/d/{PAYMENTS_SHEET_ID}/edit"
|
||||
|
||||
credentials_path = CREDENTIALS_PATH
|
||||
|
||||
junior_members_data = get_cached_data("attendance_juniors", ATTENDANCE_SHEET_ID, get_junior_members_with_fees)
|
||||
@@ -360,195 +193,26 @@ def juniors_view():
|
||||
transactions = get_cached_data("payments_transactions", PAYMENTS_SHEET_ID, fetch_sheet_data, PAYMENTS_SHEET_ID, credentials_path)
|
||||
record_step("fetch_payments")
|
||||
exceptions = get_cached_data(
|
||||
"exceptions_dict", PAYMENTS_SHEET_ID, fetch_exceptions,
|
||||
PAYMENTS_SHEET_ID, credentials_path,
|
||||
serialize=lambda d: [[list(k), v] for k, v in d.items()],
|
||||
deserialize=lambda c: {tuple(k): v for k, v in c},
|
||||
)
|
||||
"exceptions_dict", PAYMENTS_SHEET_ID, fetch_exceptions,
|
||||
PAYMENTS_SHEET_ID, credentials_path,
|
||||
serialize=lambda d: [[list(k), v] for k, v in d.items()],
|
||||
deserialize=lambda c: {tuple(k): v for k, v in c},
|
||||
)
|
||||
record_step("fetch_exceptions")
|
||||
|
||||
# Adapt junior tuple format (name, tier, {month: (fee, total_count, adult_count, junior_count)})
|
||||
# to what match_payments expects: (name, tier, {month: (expected_fee, attendance_count)})
|
||||
adapted_members = []
|
||||
for name, tier, fees_dict in junior_members:
|
||||
adapted_fees = {}
|
||||
for m, fee_data in fees_dict.items():
|
||||
if len(fee_data) == 4:
|
||||
fee, total_count, _, _ = fee_data
|
||||
adapted_fees[m] = (fee, total_count)
|
||||
else:
|
||||
fee, count = fee_data
|
||||
adapted_fees[m] = (fee, count)
|
||||
adapted_members.append((name, tier, adapted_fees))
|
||||
|
||||
adapted_members = adapt_junior_members(junior_members)
|
||||
result = reconcile(adapted_members, sorted_months, transactions, exceptions)
|
||||
record_step("reconcile")
|
||||
|
||||
# Format month labels
|
||||
month_labels = get_month_labels(sorted_months, JUNIOR_MERGED_MONTHS)
|
||||
junior_names = sorted([name for name, tier, _ in adapted_members])
|
||||
junior_members_dict = {name: fees_dict for name, _, fees_dict in junior_members}
|
||||
current_month = datetime.now().strftime("%Y-%m")
|
||||
|
||||
monthly_totals = {m: {"expected": 0, "paid": 0} for m in sorted_months}
|
||||
formatted_results = []
|
||||
for name in junior_names:
|
||||
data = result["members"][name]
|
||||
row = {"name": name, "months": [], "balance": data["total_balance"], "unpaid_periods": "", "raw_unpaid_periods": ""}
|
||||
unpaid_months = []
|
||||
raw_unpaid_months = []
|
||||
for m in sorted_months:
|
||||
mdata = data["months"].get(m, {"expected": 0, "original_expected": 0, "attendance_count": 0, "paid": 0, "exception": None})
|
||||
expected = mdata.get("expected", 0)
|
||||
original_expected = mdata.get("original_expected", 0)
|
||||
count = mdata.get("attendance_count", 0)
|
||||
paid = int(mdata.get("paid", 0))
|
||||
exception_info = mdata.get("exception", None)
|
||||
|
||||
if expected != "?" and isinstance(expected, int):
|
||||
monthly_totals[m]["expected"] += expected
|
||||
monthly_totals[m]["paid"] += paid
|
||||
|
||||
orig_fee_data = junior_members_dict.get(name, {}).get(m)
|
||||
adult_count = 0
|
||||
junior_count = 0
|
||||
if orig_fee_data and len(orig_fee_data) == 4:
|
||||
_, _, adult_count, junior_count = orig_fee_data
|
||||
|
||||
breakdown = ""
|
||||
if adult_count > 0 and junior_count > 0:
|
||||
breakdown = f":{junior_count}J,{adult_count}A"
|
||||
elif junior_count > 0:
|
||||
breakdown = f":{junior_count}J"
|
||||
elif adult_count > 0:
|
||||
breakdown = f":{adult_count}A"
|
||||
|
||||
count_str = f" ({count}{breakdown})" if count > 0 else ""
|
||||
|
||||
override_amount = exception_info["amount"] if exception_info else None
|
||||
|
||||
if override_amount is not None and override_amount != original_expected:
|
||||
is_overridden = True
|
||||
fee_display = f"{override_amount} ({original_expected}) CZK{count_str}"
|
||||
else:
|
||||
is_overridden = False
|
||||
fee_display = f"{expected} CZK{count_str}"
|
||||
|
||||
status = "empty"
|
||||
cell_text = "-"
|
||||
amount_to_pay = 0
|
||||
|
||||
if expected == "?" or (isinstance(expected, int) and expected > 0):
|
||||
if expected == "?":
|
||||
status = "empty"
|
||||
cell_text = f"?{count_str}"
|
||||
elif paid >= expected:
|
||||
status = "ok"
|
||||
cell_text = f"{paid}/{fee_display}"
|
||||
elif paid > 0:
|
||||
status = "partial"
|
||||
cell_text = f"{paid}/{fee_display}"
|
||||
amount_to_pay = expected - paid
|
||||
if m < current_month:
|
||||
unpaid_months.append(month_labels[m])
|
||||
raw_unpaid_months.append(datetime.strptime(m, "%Y-%m").strftime("%m/%Y"))
|
||||
else:
|
||||
status = "unpaid"
|
||||
cell_text = f"0/{fee_display}"
|
||||
amount_to_pay = expected
|
||||
if m < current_month:
|
||||
unpaid_months.append(month_labels[m])
|
||||
raw_unpaid_months.append(datetime.strptime(m, "%Y-%m").strftime("%m/%Y"))
|
||||
elif paid > 0:
|
||||
status = "surplus"
|
||||
cell_text = f"PAID {paid}"
|
||||
|
||||
if (isinstance(expected, int) and expected > 0) or paid > 0:
|
||||
tooltip = f"Received: {paid}, Expected: {expected}"
|
||||
else:
|
||||
tooltip = ""
|
||||
|
||||
row["months"].append({
|
||||
"text": cell_text,
|
||||
"overridden": is_overridden,
|
||||
"status": status,
|
||||
"amount": amount_to_pay,
|
||||
"month": month_labels[m],
|
||||
"raw_month": m,
|
||||
"tooltip": tooltip
|
||||
})
|
||||
|
||||
# Balance = sum of (paid - expected) for past months only; current/future months ignored.
|
||||
settled_balance = 0
|
||||
for m, mdata in data["months"].items():
|
||||
if m >= current_month:
|
||||
continue
|
||||
exp = mdata.get("expected", 0)
|
||||
if isinstance(exp, int):
|
||||
settled_balance += int(mdata.get("paid", 0)) - exp
|
||||
|
||||
payable_amount = max(0, -settled_balance)
|
||||
row["unpaid_periods"] = ", ".join(unpaid_months)
|
||||
row["raw_unpaid_periods"] = "+".join(raw_unpaid_months)
|
||||
row["balance"] = settled_balance
|
||||
row["payable_amount"] = payable_amount
|
||||
formatted_results.append(row)
|
||||
|
||||
formatted_totals = []
|
||||
for m in sorted_months:
|
||||
t = monthly_totals[m]
|
||||
status = "empty"
|
||||
if t["expected"] > 0 or t["paid"] > 0:
|
||||
if t["paid"] == t["expected"]:
|
||||
status = "ok"
|
||||
elif t["paid"] < t["expected"]:
|
||||
status = "unpaid"
|
||||
else:
|
||||
status = "surplus"
|
||||
|
||||
formatted_totals.append({
|
||||
"text": f"{t['paid']} / {t['expected']} CZK",
|
||||
"status": status
|
||||
})
|
||||
|
||||
# Format credits and debts
|
||||
def junior_settled_balance(name):
|
||||
data = result["members"][name]
|
||||
total = 0
|
||||
for m, mdata in data["months"].items():
|
||||
if m >= current_month:
|
||||
continue
|
||||
exp = mdata.get("expected", 0)
|
||||
if isinstance(exp, int):
|
||||
total += int(mdata.get("paid", 0)) - exp
|
||||
return total
|
||||
|
||||
junior_all_names = [name for name, _, _ in adapted_members]
|
||||
credits = sorted([{"name": n, "amount": junior_settled_balance(n)} for n in junior_all_names if junior_settled_balance(n) > 0], key=lambda x: x["name"])
|
||||
debts = sorted([{"name": n, "amount": abs(junior_settled_balance(n))} for n in junior_all_names if junior_settled_balance(n) < 0], key=lambda x: x["name"])
|
||||
unmatched = result["unmatched"]
|
||||
raw_payments_by_person = group_payments_by_person(transactions, [name for name, _, _ in adapted_members])
|
||||
import json
|
||||
|
||||
record_step("process_data")
|
||||
|
||||
return render_template(
|
||||
"juniors.html",
|
||||
months=[month_labels[m] for m in sorted_months],
|
||||
raw_months=sorted_months,
|
||||
results=formatted_results,
|
||||
totals=formatted_totals,
|
||||
member_data=json.dumps(result["members"]),
|
||||
month_labels_json=json.dumps(month_labels),
|
||||
raw_payments_json=json.dumps(raw_payments_by_person),
|
||||
credits=credits,
|
||||
debts=debts,
|
||||
unmatched=unmatched,
|
||||
vm = build_juniors_view_model(
|
||||
junior_members, adapted_members, sorted_months, result, transactions,
|
||||
datetime.now().strftime("%Y-%m"),
|
||||
attendance_url=attendance_url,
|
||||
payments_url=payments_url,
|
||||
bank_account=BANK_ACCOUNT,
|
||||
current_month=current_month
|
||||
)
|
||||
record_step("process_data")
|
||||
return render_template("juniors.html", **vm)
|
||||
|
||||
@app.route("/payments")
|
||||
def payments():
|
||||
@@ -567,23 +231,13 @@ def payments():
|
||||
if juniors_data:
|
||||
member_names.extend(name for name, _, _ in juniors_data[0])
|
||||
|
||||
grouped = group_payments_by_person(transactions, member_names)
|
||||
# payments page also groups unmatched rows under a fallback key
|
||||
for tx in transactions:
|
||||
if not str(tx.get("person", "")).strip():
|
||||
grouped.setdefault("Unmatched / Unknown", []).append(tx)
|
||||
for rows in grouped.values():
|
||||
rows.sort(key=lambda t: str(t.get("date", "")), reverse=True)
|
||||
sorted_people = sorted(grouped.keys())
|
||||
|
||||
record_step("process_data")
|
||||
return render_template(
|
||||
"payments.html",
|
||||
grouped_payments=grouped,
|
||||
sorted_people=sorted_people,
|
||||
vm = build_payments_view_model(
|
||||
transactions, member_names,
|
||||
attendance_url=attendance_url,
|
||||
payments_url=payments_url
|
||||
payments_url=payments_url,
|
||||
)
|
||||
record_step("process_data")
|
||||
return render_template("payments.html", **vm)
|
||||
|
||||
@app.route("/qr")
|
||||
def qr_code():
|
||||
|
||||
29
docs/plans/2026-05-07-1321-fuj-sync-print-fio-table.md
Normal file
29
docs/plans/2026-05-07-1321-fuj-sync-print-fio-table.md
Normal file
@@ -0,0 +1,29 @@
|
||||
# Add `--print-fio-table` debug flag to `fuj sync`
|
||||
|
||||
## Context
|
||||
|
||||
The Go port of `fuj sync --dry-run` currently prints only the **new**
|
||||
transactions — i.e. rows that will be appended to the payments sheet after
|
||||
deduping against existing Sync IDs (see [sync.go:125-129](../../go/internal/services/banksync/sync.go#L125-L129)).
|
||||
When debugging Fio sync issues ("why isn't transaction X showing up?",
|
||||
"is the dedup working?"), there's no way to see what Fio actually
|
||||
returned versus what got filtered as a duplicate.
|
||||
|
||||
This change adds a `--print-fio-table` flag that, **only when combined
|
||||
with `--dry-run`**, prints an aligned table of every Fio transaction in
|
||||
the window with each row marked `NEW` (would be appended) or `DUP`
|
||||
(already in sheet, skipped). The flag is silently ignored without
|
||||
`--dry-run`, so it can't accidentally fire during a real sync.
|
||||
|
||||
## Decisions
|
||||
|
||||
- Flag name: `--print-fio-table` (specific, not generic `--verbose`).
|
||||
- Columns: `DATE | AMOUNT | SENDER | VS | MESSAGE | BANKID | STATUS`,
|
||||
with MESSAGE truncated and STATUS = `NEW` / `DUP`.
|
||||
- Scope: only effective when `--dry-run` is also set.
|
||||
|
||||
## Files modified
|
||||
|
||||
- [go/cmd/fuj/main.go](../../go/cmd/fuj/main.go) — new flag + SyncOpts field
|
||||
- [go/internal/services/banksync/sync.go](../../go/internal/services/banksync/sync.go) — SyncOpts struct + refactored step 4
|
||||
- [go/internal/services/banksync/debug.go](../../go/internal/services/banksync/debug.go) — printFioTable helper (new)
|
||||
185
docs/plans/2026-05-07-1431-m5-json-api-parity.md
Normal file
185
docs/plans/2026-05-07-1431-m5-json-api-parity.md
Normal file
@@ -0,0 +1,185 @@
|
||||
# Python view-model cleanup (M5 prep — Python side only)
|
||||
|
||||
Scoped-down precursor to M5 of the Go rewrite. See:
|
||||
- [2026-05-03-2349-go-backend-rewrite.md](2026-05-03-2349-go-backend-rewrite.md)
|
||||
- [2026-05-03-2349-go-backend-rewrite-progress.md](2026-05-03-2349-go-backend-rewrite-progress.md)
|
||||
|
||||
## Context
|
||||
|
||||
The Python app is still production. Before adding `/api/X` shadow
|
||||
routes, JSON DTOs, parity tooling, and Go handlers, get the Python
|
||||
side into a clean shape. Today the `/adults` and `/juniors` routes
|
||||
each carry 150–200 lines of inline view-model construction inside
|
||||
the Flask handler:
|
||||
|
||||
- [adults_view](app.py#L179) — 167 LOC, computes per-row
|
||||
status/cell_text/balance/credits/debts inline.
|
||||
- [juniors_view](app.py#L347) — 205 LOC, same plus `"?"` sentinel
|
||||
branching and `:NJ,MA` breakdown.
|
||||
- [payments](app.py#L553) — 35 LOC, lighter but still mixes IO and
|
||||
grouping.
|
||||
|
||||
Pulling that into pure builder functions:
|
||||
- shrinks `app.py` and makes each route handler do only what a route
|
||||
handler should (IO + cache + step timing + render);
|
||||
- gives us **already-tested** pure functions that future M5 work can
|
||||
call from a `/api/X` shadow endpoint with one line of `jsonify(...)`;
|
||||
- has zero behavioural change (existing tests in
|
||||
[test_app.py](tests/test_app.py) act as the regression guard).
|
||||
|
||||
This is a Python-only change. No Go work, no JSON contract, no shadow
|
||||
routes — those come after.
|
||||
|
||||
## Approach
|
||||
|
||||
Three pure builder functions in a new `scripts/views.py` module. Each
|
||||
takes already-loaded, deserialized inputs (no Flask globals, no IO,
|
||||
no cache calls) and returns the exact dict that today's route passes
|
||||
to `render_template`.
|
||||
|
||||
```python
|
||||
def build_adults_view_model(
|
||||
members, sorted_months, transactions, exceptions, current_month,
|
||||
*, attendance_url, payments_url, bank_account,
|
||||
) -> dict: ...
|
||||
|
||||
def build_juniors_view_model(
|
||||
junior_members, sorted_months, transactions, exceptions, current_month,
|
||||
*, attendance_url, payments_url, bank_account,
|
||||
) -> dict: ...
|
||||
|
||||
def build_payments_view_model(
|
||||
transactions, member_names,
|
||||
*, attendance_url, payments_url,
|
||||
) -> dict: ...
|
||||
```
|
||||
|
||||
The route handlers shrink to: load data (with `get_cached_data` and
|
||||
`record_step` calls staying in `app.py`), call the builder, render.
|
||||
|
||||
```python
|
||||
@app.route("/adults")
|
||||
def adults_view():
|
||||
members, sorted_months = ... # cached loads + record_step calls
|
||||
transactions = ...
|
||||
exceptions = ...
|
||||
result_meta = ...
|
||||
record_step("process_data")
|
||||
vm = build_adults_view_model(
|
||||
members, sorted_months, transactions, exceptions,
|
||||
datetime.now().strftime("%Y-%m"),
|
||||
attendance_url=..., payments_url=..., bank_account=BANK_ACCOUNT,
|
||||
)
|
||||
return render_template("adults.html", **vm)
|
||||
```
|
||||
|
||||
## Decisions
|
||||
|
||||
1. **Pure builders, no Flask state.** No `record_step`, no `g.*`, no
|
||||
`get_cached_data` inside builders. They take plain args, return
|
||||
plain dicts. This is what makes them trivially unit-testable and,
|
||||
later, trivially reusable from `/api/X`.
|
||||
2. **Preserve byte-equal behaviour.** The dicts returned must match
|
||||
today's `render_template(...)` kwargs key-for-key, value-for-value
|
||||
— including the existing `json.dumps(member_data)` /
|
||||
`json.dumps(month_labels_json)` / `json.dumps(raw_payments_json)`
|
||||
wrappers. Those wrappers exist for inline JS in templates; the
|
||||
refactor doesn't touch them. (When `/api/X` lands later, that route
|
||||
will produce a sibling dict with the wrappers stripped, but that's
|
||||
future work.)
|
||||
3. **`reconcile()` stays inside the route, not the builder.** It
|
||||
crosses the IO/cache boundary in spirit (its inputs come from
|
||||
cache); but it's also pure-domain and called by both adults and
|
||||
juniors. Keep the call site in the route so `record_step("reconcile")`
|
||||
timing isn't lost. Builder takes `result` as an argument.
|
||||
4. **Shared helpers stay in `app.py`** for now —
|
||||
[get_month_labels](app.py#L41) and
|
||||
[group_payments_by_person](app.py#L60) are already module-level
|
||||
pure functions, used by routes and now by builders. Either leave
|
||||
them in `app.py` and import into `scripts/views.py`, or move them
|
||||
into `scripts/views.py`. **Choose: move into `scripts/views.py`**
|
||||
— they're view-model concerns, not Flask concerns, and `app.py`
|
||||
should keep shrinking.
|
||||
5. **No new test file needed.** Existing
|
||||
[tests/test_app.py](tests/test_app.py) tests
|
||||
`test_adults_route`, `test_juniors_route`, `test_payments_route`
|
||||
exercise the rendered HTML end-to-end. If they pass after the
|
||||
refactor, behaviour is preserved. Adding builder-level unit tests
|
||||
is a *nice-to-have* but not required for this iteration.
|
||||
|
||||
## Tasks
|
||||
|
||||
### 1. Create `scripts/views.py` with the three builders + shared helpers
|
||||
|
||||
- Move `get_month_labels` and `group_payments_by_person` from
|
||||
[app.py:41–77](app.py#L41-L77) into `scripts/views.py`. Update the
|
||||
import in `app.py`.
|
||||
- Implement `build_adults_view_model` by extracting
|
||||
[app.py:200–344](app.py#L200-L344) (everything between `result =
|
||||
reconcile(...)` and `return render_template(...)`). Take `result`
|
||||
as a parameter; emit the same dict that's currently passed as
|
||||
`**kwargs` to `render_template`.
|
||||
- Implement `build_juniors_view_model` by extracting
|
||||
[app.py:370–550](app.py#L370-L550). Same shape — including the
|
||||
`adapted_members` adapter loop, the junior `?`-sentinel branches,
|
||||
and the `:NJ,MA` breakdown.
|
||||
- Implement `build_payments_view_model` by extracting
|
||||
[app.py:570–586](app.py#L570-L586) (the `group_payments_by_person`
|
||||
call + `Unmatched / Unknown` bucket + sort).
|
||||
|
||||
### 2. Slim down the route handlers in `app.py`
|
||||
|
||||
Each handler keeps:
|
||||
- `attendance_url`/`payments_url` URL building
|
||||
- `get_cached_data` calls (with `record_step` between)
|
||||
- `reconcile(...)` call (adults/juniors) with `record_step("reconcile")`
|
||||
- `record_step("process_data")` after the builder call
|
||||
- `return render_template("X.html", **view_model)`
|
||||
|
||||
Each handler **drops** all the per-row computation, totals
|
||||
formatting, credits/debts sorting, `raw_payments_by_person` building,
|
||||
and `import json` lines.
|
||||
|
||||
Target post-refactor LOC for each route handler: ~25–30 lines
|
||||
(currently 167 / 205 / 35).
|
||||
|
||||
### 3. Run the existing test suite + manual smoke
|
||||
|
||||
```
|
||||
make test # all tests green
|
||||
make web-py # browse /adults /juniors /payments visually
|
||||
```
|
||||
|
||||
## Critical files
|
||||
|
||||
- [app.py](app.py) — routes shrink dramatically; `get_month_labels`
|
||||
and `group_payments_by_person` get moved out.
|
||||
- New: `scripts/views.py` — three builders + the two helpers.
|
||||
- [tests/test_app.py](tests/test_app.py) — unchanged; serves as the
|
||||
regression guard.
|
||||
|
||||
## Verification
|
||||
|
||||
1. `make test` — all four `TestWebApp` tests pass unchanged.
|
||||
2. `make web-py` and visit `/adults`, `/juniors`, `/payments` — each
|
||||
renders identically to before (same table contents, same totals,
|
||||
same credits/debts, same `?` rendering on juniors).
|
||||
3. `git diff app.py` shows substantial deletions (route bodies
|
||||
shrink) and only thin glue calling the new builders.
|
||||
4. Optional sanity check: temporarily add `print(repr(view_model))`
|
||||
in the route before `render_template` on `main` and on the branch,
|
||||
diff for one fixture run — should be byte-identical dicts.
|
||||
|
||||
## Out of scope (future iterations, not this plan)
|
||||
|
||||
- `/api/<route>` shadow endpoints in Flask
|
||||
- Go `internal/web/api/` DTOs and JSON Schemas
|
||||
- Go `internal/services/membership/views.go` aggregators
|
||||
- Go HTTP handlers for `/api/X`
|
||||
- `cmd/parity/main.go` and `make parity` target
|
||||
- Junior breakdown sidecar work in
|
||||
`go/internal/services/membership/sources.go`
|
||||
|
||||
These are the original M5 tasks (M5.1–M5.4 in the progress tracker).
|
||||
They become much easier once today's refactor lands, because the
|
||||
shadow `/api/X` routes will be one-liners over the new builders.
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"fuj-management/go/internal/services/banksync"
|
||||
"fuj-management/go/internal/services/membership"
|
||||
"fuj-management/go/internal/web"
|
||||
"log/slog"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
@@ -28,6 +29,9 @@ func main() {
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
// Honour LOG_LEVEL for slog calls in any package (e.g. internal/io/fio debug logs).
|
||||
slog.SetDefault(logging.New(os.Getenv("LOG_LEVEL")))
|
||||
|
||||
cmd, args := os.Args[1], os.Args[2:]
|
||||
|
||||
switch cmd {
|
||||
@@ -135,8 +139,9 @@ func syncCmd(args []string) {
|
||||
toStr := fs.String("to", "", "end date YYYY-MM-DD")
|
||||
sort := fs.Bool("sort", true, "sort sheet by date after appending")
|
||||
dryRun := fs.Bool("dry-run", false, "print planned writes without modifying the sheet")
|
||||
printFioTable := fs.Bool("print-fio-table", false, "with --dry-run: print aligned table of every Fio transaction with NEW/DUP status")
|
||||
fs.Usage = func() {
|
||||
fmt.Fprintln(os.Stderr, "usage: fuj sync [--days N] [--from YYYY-MM-DD --to YYYY-MM-DD] [--sort] [--dry-run]")
|
||||
fmt.Fprintln(os.Stderr, "usage: fuj sync [--days N] [--from YYYY-MM-DD --to YYYY-MM-DD] [--sort] [--dry-run] [--print-fio-table]")
|
||||
fs.PrintDefaults()
|
||||
}
|
||||
if err := fs.Parse(args); err != nil {
|
||||
@@ -154,7 +159,7 @@ func syncCmd(args []string) {
|
||||
}
|
||||
fioCli := fio.New(cfg.FioAPIToken, config.IBANAccountNum(cfg.BankAccount), nil)
|
||||
|
||||
opts := banksync.SyncOpts{Days: *days, Sort: *sort, DryRun: *dryRun}
|
||||
opts := banksync.SyncOpts{Days: *days, Sort: *sort, DryRun: *dryRun, PrintFioTable: *printFioTable}
|
||||
if *fromStr != "" && *toStr != "" {
|
||||
opts.From, err = time.Parse("2006-01-02", *fromStr)
|
||||
if err != nil {
|
||||
|
||||
@@ -5,7 +5,9 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -25,6 +27,9 @@ func (c *apiClient) FetchTransactions(ctx context.Context, from, to time.Time) (
|
||||
const layout = "2006-01-02"
|
||||
url := fmt.Sprintf("https://fioapi.fio.cz/v1/rest/periods/%s/%s/%s/transactions.json",
|
||||
c.token, from.Format(layout), to.Format(layout))
|
||||
slog.Debug("fio api: GET",
|
||||
"url", strings.Replace(url, c.token, "****", 1),
|
||||
"from", from.Format(layout), "to", to.Format(layout))
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
@@ -35,6 +40,7 @@ func (c *apiClient) FetchTransactions(ctx context.Context, from, to time.Time) (
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
slog.Debug("fio api: response", "status", resp.StatusCode)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("fio api: HTTP %d", resp.StatusCode)
|
||||
}
|
||||
@@ -42,7 +48,9 @@ func (c *apiClient) FetchTransactions(ctx context.Context, from, to time.Time) (
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parseAPIResponse(body)
|
||||
txns, err := parseAPIResponse(body)
|
||||
slog.Debug("fio api: parsed", "body_bytes", len(body), "parsed_count", len(txns))
|
||||
return txns, err
|
||||
}
|
||||
|
||||
// fioAPIResponse is the top-level envelope from the Fio JSON API.
|
||||
|
||||
@@ -4,6 +4,7 @@ package fio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
@@ -36,7 +37,9 @@ func New(token, accountNum string, hc httpDoer) Client {
|
||||
hc = http.DefaultClient
|
||||
}
|
||||
if token != "" {
|
||||
slog.Debug("fio: client selected", "type", "api")
|
||||
return &apiClient{token: token, hc: hc}
|
||||
}
|
||||
slog.Debug("fio: client selected", "type", "transparent", "account_num", accountNum)
|
||||
return &transparentClient{accountNum: accountNum, hc: hc}
|
||||
}
|
||||
|
||||
@@ -97,6 +97,9 @@ func TestParseCzechDate(t *testing.T) {
|
||||
{"10/04/2026", "2026-04-10"},
|
||||
{"7.5.2026", "2026-05-07"}, // non-padded — real Fio transparent page format
|
||||
{"3.12.2025", "2025-12-03"}, // non-padded single-digit day, double-digit month
|
||||
{"07.05.26", "2026-05-07"}, // padded 2-digit year — current Fio transparent page format
|
||||
{"7.5.26", "2026-05-07"}, // non-padded 2-digit year
|
||||
{"07/05/26", "2026-05-07"}, // slash variant
|
||||
{"", ""},
|
||||
{"invalid", ""},
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -28,6 +29,10 @@ func (c *transparentClient) FetchTransactions(ctx context.Context, from, to time
|
||||
from.Format("2.1.2006"),
|
||||
to.Format("2.1.2006"),
|
||||
)
|
||||
slog.Debug("fio transparent: GET",
|
||||
"url", url,
|
||||
"from", from.Format("2006-01-02"), "to", to.Format("2006-01-02"))
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -37,6 +42,7 @@ func (c *transparentClient) FetchTransactions(ctx context.Context, from, to time
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
slog.Debug("fio transparent: response", "status", resp.StatusCode)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("fio transparent: HTTP %d", resp.StatusCode)
|
||||
}
|
||||
@@ -44,6 +50,7 @@ func (c *transparentClient) FetchTransactions(ctx context.Context, from, to time
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slog.Debug("fio transparent: body read", "body_bytes", len(body))
|
||||
return parseTransparentHTML(body)
|
||||
}
|
||||
|
||||
@@ -63,6 +70,7 @@ func parseTransparentHTML(body []byte) ([]Transaction, error) {
|
||||
rows := extractSecondTableRows(body)
|
||||
|
||||
var txns []Transaction
|
||||
var droppedBadDate, droppedNonpositive int
|
||||
for _, row := range rows {
|
||||
col := func(i int) string {
|
||||
if i < len(row) {
|
||||
@@ -72,7 +80,12 @@ func parseTransparentHTML(body []byte) ([]Transaction, error) {
|
||||
}
|
||||
dateStr := parseCzechDate(col(tColDate))
|
||||
amount := parseCzechAmount(col(tColAmount))
|
||||
if dateStr == "" || amount <= 0 {
|
||||
if dateStr == "" {
|
||||
droppedBadDate++
|
||||
continue
|
||||
}
|
||||
if amount <= 0 {
|
||||
droppedNonpositive++
|
||||
continue
|
||||
}
|
||||
txns = append(txns, Transaction{
|
||||
@@ -86,6 +99,11 @@ func parseTransparentHTML(body []byte) ([]Transaction, error) {
|
||||
BankID: "", // not available on HTML path
|
||||
})
|
||||
}
|
||||
slog.Debug("fio transparent: parsed",
|
||||
"raw_rows", len(rows),
|
||||
"kept", len(txns),
|
||||
"dropped_bad_date", droppedBadDate,
|
||||
"dropped_nonpositive_amount", droppedNonpositive)
|
||||
return txns, nil
|
||||
}
|
||||
|
||||
@@ -191,7 +209,10 @@ func hasClass(t ghtml.Token, cls string) bool {
|
||||
// Returns "" on parse error.
|
||||
func parseCzechDate(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
for _, layout := range []string{"2.1.2006", "02.01.2006", "2/1/2006", "02/01/2006"} {
|
||||
for _, layout := range []string{
|
||||
"2.1.2006", "02.01.2006", "2/1/2006", "02/01/2006",
|
||||
"2.1.06", "02.01.06", "2/1/06", "02/01/06",
|
||||
} {
|
||||
if t, err := time.Parse(layout, s); err == nil {
|
||||
return t.Format("2006-01-02")
|
||||
}
|
||||
|
||||
32
go/internal/services/banksync/fio_table.go
Normal file
32
go/internal/services/banksync/fio_table.go
Normal file
@@ -0,0 +1,32 @@
|
||||
package banksync
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"text/tabwriter"
|
||||
|
||||
"fuj-management/go/internal/io/fio"
|
||||
)
|
||||
|
||||
func printFioTable(w io.Writer, txns []fio.Transaction, syncIDs []string, existing map[string]bool) {
|
||||
tw := tabwriter.NewWriter(w, 0, 0, 2, ' ', 0)
|
||||
fmt.Fprintln(tw, "DATE\tAMOUNT\tSENDER\tVS\tMESSAGE\tBANKID\tSTATUS")
|
||||
for i, tx := range txns {
|
||||
status := "NEW"
|
||||
if existing[syncIDs[i]] {
|
||||
status = "DUP"
|
||||
}
|
||||
fmt.Fprintf(tw, "%s\t%.2f\t%s\t%s\t%s\t%s\t%s\n",
|
||||
tx.Date, tx.Amount, tx.Sender, tx.VS,
|
||||
truncRunes(tx.Message, 40), tx.BankID, status)
|
||||
}
|
||||
_ = tw.Flush()
|
||||
}
|
||||
|
||||
func truncRunes(s string, n int) string {
|
||||
rs := []rune(s)
|
||||
if len(rs) <= n {
|
||||
return s
|
||||
}
|
||||
return string(rs[:n-1]) + "…"
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"fuj-management/go/internal/domain/synch"
|
||||
"fuj-management/go/internal/io/fio"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
@@ -27,10 +28,11 @@ type sheetsWriter interface {
|
||||
|
||||
// SyncOpts controls the date window and sort behaviour.
|
||||
type SyncOpts struct {
|
||||
Days int // look-back window when From/To are zero
|
||||
From, To time.Time // explicit window (overrides Days)
|
||||
Sort bool // sort the sheet by Date after appending
|
||||
DryRun bool // print planned writes without modifying the sheet
|
||||
Days int // look-back window when From/To are zero
|
||||
From, To time.Time // explicit window (overrides Days)
|
||||
Sort bool // sort the sheet by Date after appending
|
||||
DryRun bool // print planned writes without modifying the sheet
|
||||
PrintFioTable bool // with DryRun: print every fetched Fio txn with NEW/DUP status
|
||||
}
|
||||
|
||||
// SyncToSheets fetches Fio transactions and appends new ones to the payments sheet.
|
||||
@@ -92,14 +94,14 @@ func SyncToSheets(
|
||||
from.Format("2006-01-02"), to.Format("2006-01-02"), len(txns))
|
||||
}
|
||||
|
||||
// 4. Append new rows.
|
||||
var newRows [][]any
|
||||
for _, tx := range txns {
|
||||
// 4a. Compute Sync IDs for every fetched txn (shared by table-print and row-build).
|
||||
syncIDs := make([]string, len(txns))
|
||||
for i, tx := range txns {
|
||||
currency := tx.Currency
|
||||
if currency == "" {
|
||||
currency = "CZK"
|
||||
}
|
||||
id := synch.GenerateSyncID(synch.Transaction{
|
||||
syncIDs[i] = synch.GenerateSyncID(synch.Transaction{
|
||||
Date: tx.Date,
|
||||
Amount: tx.Amount,
|
||||
Currency: currency,
|
||||
@@ -108,13 +110,23 @@ func SyncToSheets(
|
||||
Message: tx.Message,
|
||||
BankID: tx.BankID,
|
||||
})
|
||||
if existingIDs[id] {
|
||||
}
|
||||
|
||||
// 4b. Optional debug table (dry-run only; suppress when nothing was fetched).
|
||||
if opts.DryRun && opts.PrintFioTable && len(txns) > 0 {
|
||||
printFioTable(os.Stdout, txns, syncIDs, existingIDs)
|
||||
}
|
||||
|
||||
// 4c. Build new rows.
|
||||
var newRows [][]any
|
||||
for i, tx := range txns {
|
||||
if existingIDs[syncIDs[i]] {
|
||||
continue
|
||||
}
|
||||
newRows = append(newRows, []any{
|
||||
tx.Date, tx.Amount,
|
||||
"", "", "", "", // manual fix, Person, Purpose, Inferred Amount
|
||||
tx.Sender, tx.VS, tx.Message, tx.BankID, id,
|
||||
tx.Sender, tx.VS, tx.Message, tx.BankID, syncIDs[i],
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
445
scripts/views.py
Normal file
445
scripts/views.py
Normal file
@@ -0,0 +1,445 @@
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime
|
||||
|
||||
from attendance import ADULT_MERGED_MONTHS, JUNIOR_MERGED_MONTHS
|
||||
from match_payments import canonical_member_key
|
||||
|
||||
|
||||
def get_month_labels(sorted_months, merged_months):
|
||||
labels = {}
|
||||
for m in sorted_months:
|
||||
dt = datetime.strptime(m, "%Y-%m")
|
||||
merged_in = sorted([k for k, v in merged_months.items() if v == m])
|
||||
if merged_in:
|
||||
all_dts = [datetime.strptime(x, "%Y-%m") for x in sorted(merged_in + [m])]
|
||||
years = {d.year for d in all_dts}
|
||||
if len(years) > 1:
|
||||
parts = [d.strftime("%b %Y") for d in all_dts]
|
||||
labels[m] = "+".join(parts)
|
||||
else:
|
||||
parts = [d.strftime("%b") for d in all_dts]
|
||||
labels[m] = f"{'+'.join(parts)} {dt.strftime('%Y')}"
|
||||
else:
|
||||
labels[m] = dt.strftime("%b %Y")
|
||||
return labels
|
||||
|
||||
|
||||
def group_payments_by_person(transactions, member_names=None):
|
||||
canonical_by_key = (
|
||||
{canonical_member_key(n): n for n in member_names} if member_names else {}
|
||||
)
|
||||
grouped = {}
|
||||
for tx in transactions:
|
||||
person = str(tx.get("person", "")).strip()
|
||||
if not person:
|
||||
continue
|
||||
for p in person.split(","):
|
||||
p = re.sub(r"\[\?\]\s*", "", p).strip()
|
||||
if not p:
|
||||
continue
|
||||
key = canonical_by_key.get(canonical_member_key(p), p)
|
||||
grouped.setdefault(key, []).append(tx)
|
||||
for rows in grouped.values():
|
||||
rows.sort(key=lambda t: str(t.get("date", "")), reverse=True)
|
||||
return grouped
|
||||
|
||||
|
||||
def adapt_junior_members(junior_members):
|
||||
"""Convert 4-tuple junior fee data to (fee, total_count) for reconcile."""
|
||||
adapted = []
|
||||
for name, tier, fees_dict in junior_members:
|
||||
adapted_fees = {}
|
||||
for m, fee_data in fees_dict.items():
|
||||
if len(fee_data) == 4:
|
||||
fee, total_count, _, _ = fee_data
|
||||
adapted_fees[m] = (fee, total_count)
|
||||
else:
|
||||
fee, count = fee_data
|
||||
adapted_fees[m] = (fee, count)
|
||||
adapted.append((name, tier, adapted_fees))
|
||||
return adapted
|
||||
|
||||
|
||||
def build_adults_view_model(
|
||||
members,
|
||||
sorted_months,
|
||||
result,
|
||||
transactions,
|
||||
current_month,
|
||||
*,
|
||||
attendance_url,
|
||||
payments_url,
|
||||
bank_account,
|
||||
):
|
||||
month_labels = get_month_labels(sorted_months, ADULT_MERGED_MONTHS)
|
||||
adult_names = sorted([name for name, tier, _ in members if tier == "A"])
|
||||
|
||||
monthly_totals = {m: {"expected": 0, "paid": 0} for m in sorted_months}
|
||||
formatted_results = []
|
||||
for name in adult_names:
|
||||
data = result["members"][name]
|
||||
row = {
|
||||
"name": name,
|
||||
"months": [],
|
||||
"balance": data["total_balance"],
|
||||
"unpaid_periods": "",
|
||||
"raw_unpaid_periods": "",
|
||||
}
|
||||
unpaid_months = []
|
||||
raw_unpaid_months = []
|
||||
for m in sorted_months:
|
||||
mdata = data["months"].get(m, {
|
||||
"expected": 0, "original_expected": 0,
|
||||
"attendance_count": 0, "paid": 0, "exception": None,
|
||||
})
|
||||
expected = mdata.get("expected", 0)
|
||||
original_expected = mdata.get("original_expected", 0)
|
||||
count = mdata.get("attendance_count", 0)
|
||||
paid = int(mdata.get("paid", 0))
|
||||
exception_info = mdata.get("exception", None)
|
||||
|
||||
monthly_totals[m]["expected"] += expected
|
||||
monthly_totals[m]["paid"] += paid
|
||||
|
||||
override_amount = exception_info["amount"] if exception_info else None
|
||||
|
||||
if override_amount is not None and override_amount != original_expected:
|
||||
is_overridden = True
|
||||
fee_display = (
|
||||
f"{override_amount} ({original_expected}) CZK ({count})"
|
||||
if count > 0
|
||||
else f"{override_amount} ({original_expected}) CZK"
|
||||
)
|
||||
else:
|
||||
is_overridden = False
|
||||
fee_display = (
|
||||
f"{expected} CZK ({count})" if count > 0 else f"{expected} CZK"
|
||||
)
|
||||
|
||||
status = "empty"
|
||||
cell_text = "-"
|
||||
amount_to_pay = 0
|
||||
|
||||
if expected > 0:
|
||||
amount_to_pay = max(0, expected - paid)
|
||||
if paid >= expected:
|
||||
status = "ok"
|
||||
cell_text = f"{paid}/{fee_display}"
|
||||
elif paid > 0:
|
||||
status = "partial"
|
||||
cell_text = f"{paid}/{fee_display}"
|
||||
if m < current_month:
|
||||
unpaid_months.append(month_labels[m])
|
||||
raw_unpaid_months.append(
|
||||
datetime.strptime(m, "%Y-%m").strftime("%m/%Y")
|
||||
)
|
||||
else:
|
||||
status = "unpaid"
|
||||
cell_text = f"0/{fee_display}"
|
||||
if m < current_month:
|
||||
unpaid_months.append(month_labels[m])
|
||||
raw_unpaid_months.append(
|
||||
datetime.strptime(m, "%Y-%m").strftime("%m/%Y")
|
||||
)
|
||||
elif paid > 0:
|
||||
status = "surplus"
|
||||
cell_text = f"PAID {paid}"
|
||||
else:
|
||||
cell_text = "-"
|
||||
amount_to_pay = 0
|
||||
|
||||
if expected > 0 or paid > 0:
|
||||
tooltip = f"Received: {paid}, Expected: {expected}"
|
||||
else:
|
||||
tooltip = ""
|
||||
|
||||
row["months"].append({
|
||||
"text": cell_text,
|
||||
"overridden": is_overridden,
|
||||
"status": status,
|
||||
"amount": amount_to_pay,
|
||||
"month": month_labels[m],
|
||||
"raw_month": m,
|
||||
"tooltip": tooltip,
|
||||
})
|
||||
|
||||
settled_balance = 0
|
||||
for m, mdata in data["months"].items():
|
||||
if m >= current_month:
|
||||
continue
|
||||
exp = mdata.get("expected", 0)
|
||||
if isinstance(exp, int):
|
||||
settled_balance += int(mdata.get("paid", 0)) - exp
|
||||
|
||||
payable_amount = max(0, -settled_balance)
|
||||
row["unpaid_periods"] = ", ".join(unpaid_months)
|
||||
row["raw_unpaid_periods"] = "+".join(raw_unpaid_months)
|
||||
row["balance"] = settled_balance
|
||||
row["payable_amount"] = payable_amount
|
||||
formatted_results.append(row)
|
||||
|
||||
formatted_totals = []
|
||||
for m in sorted_months:
|
||||
t = monthly_totals[m]
|
||||
status = "empty"
|
||||
if t["expected"] > 0 or t["paid"] > 0:
|
||||
if t["paid"] == t["expected"]:
|
||||
status = "ok"
|
||||
elif t["paid"] < t["expected"]:
|
||||
status = "unpaid"
|
||||
else:
|
||||
status = "surplus"
|
||||
formatted_totals.append({
|
||||
"text": f"{t['paid']} / {t['expected']} CZK",
|
||||
"status": status,
|
||||
})
|
||||
|
||||
def _settled_balance(name):
|
||||
data = result["members"][name]
|
||||
total = 0
|
||||
for m, mdata in data["months"].items():
|
||||
if m >= current_month:
|
||||
continue
|
||||
exp = mdata.get("expected", 0)
|
||||
if isinstance(exp, int):
|
||||
total += int(mdata.get("paid", 0)) - exp
|
||||
return total
|
||||
|
||||
credits = sorted(
|
||||
[{"name": n, "amount": _settled_balance(n)} for n in adult_names if _settled_balance(n) > 0],
|
||||
key=lambda x: x["name"],
|
||||
)
|
||||
debts = sorted(
|
||||
[{"name": n, "amount": abs(_settled_balance(n))} for n in adult_names if _settled_balance(n) < 0],
|
||||
key=lambda x: x["name"],
|
||||
)
|
||||
|
||||
raw_payments_by_person = group_payments_by_person(
|
||||
transactions, [name for name, _, _ in members]
|
||||
)
|
||||
|
||||
return dict(
|
||||
months=[month_labels[m] for m in sorted_months],
|
||||
raw_months=sorted_months,
|
||||
results=formatted_results,
|
||||
totals=formatted_totals,
|
||||
member_data=json.dumps(result["members"]),
|
||||
month_labels_json=json.dumps(month_labels),
|
||||
raw_payments_json=json.dumps(raw_payments_by_person),
|
||||
credits=credits,
|
||||
debts=debts,
|
||||
unmatched=result["unmatched"],
|
||||
attendance_url=attendance_url,
|
||||
payments_url=payments_url,
|
||||
bank_account=bank_account,
|
||||
current_month=current_month,
|
||||
)
|
||||
|
||||
|
||||
def build_juniors_view_model(
|
||||
junior_members,
|
||||
adapted_members,
|
||||
sorted_months,
|
||||
result,
|
||||
transactions,
|
||||
current_month,
|
||||
*,
|
||||
attendance_url,
|
||||
payments_url,
|
||||
bank_account,
|
||||
):
|
||||
month_labels = get_month_labels(sorted_months, JUNIOR_MERGED_MONTHS)
|
||||
junior_names = sorted([name for name, tier, _ in adapted_members])
|
||||
junior_members_dict = {name: fees_dict for name, _, fees_dict in junior_members}
|
||||
|
||||
monthly_totals = {m: {"expected": 0, "paid": 0} for m in sorted_months}
|
||||
formatted_results = []
|
||||
for name in junior_names:
|
||||
data = result["members"][name]
|
||||
row = {
|
||||
"name": name,
|
||||
"months": [],
|
||||
"balance": data["total_balance"],
|
||||
"unpaid_periods": "",
|
||||
"raw_unpaid_periods": "",
|
||||
}
|
||||
unpaid_months = []
|
||||
raw_unpaid_months = []
|
||||
for m in sorted_months:
|
||||
mdata = data["months"].get(m, {
|
||||
"expected": 0, "original_expected": 0,
|
||||
"attendance_count": 0, "paid": 0, "exception": None,
|
||||
})
|
||||
expected = mdata.get("expected", 0)
|
||||
original_expected = mdata.get("original_expected", 0)
|
||||
count = mdata.get("attendance_count", 0)
|
||||
paid = int(mdata.get("paid", 0))
|
||||
exception_info = mdata.get("exception", None)
|
||||
|
||||
if expected != "?" and isinstance(expected, int):
|
||||
monthly_totals[m]["expected"] += expected
|
||||
monthly_totals[m]["paid"] += paid
|
||||
|
||||
orig_fee_data = junior_members_dict.get(name, {}).get(m)
|
||||
adult_count = 0
|
||||
junior_count = 0
|
||||
if orig_fee_data and len(orig_fee_data) == 4:
|
||||
_, _, adult_count, junior_count = orig_fee_data
|
||||
|
||||
breakdown = ""
|
||||
if adult_count > 0 and junior_count > 0:
|
||||
breakdown = f":{junior_count}J,{adult_count}A"
|
||||
elif junior_count > 0:
|
||||
breakdown = f":{junior_count}J"
|
||||
elif adult_count > 0:
|
||||
breakdown = f":{adult_count}A"
|
||||
|
||||
count_str = f" ({count}{breakdown})" if count > 0 else ""
|
||||
|
||||
override_amount = exception_info["amount"] if exception_info else None
|
||||
|
||||
if override_amount is not None and override_amount != original_expected:
|
||||
is_overridden = True
|
||||
fee_display = f"{override_amount} ({original_expected}) CZK{count_str}"
|
||||
else:
|
||||
is_overridden = False
|
||||
fee_display = f"{expected} CZK{count_str}"
|
||||
|
||||
status = "empty"
|
||||
cell_text = "-"
|
||||
amount_to_pay = 0
|
||||
|
||||
if expected == "?" or (isinstance(expected, int) and expected > 0):
|
||||
if expected == "?":
|
||||
status = "empty"
|
||||
cell_text = f"?{count_str}"
|
||||
elif paid >= expected:
|
||||
status = "ok"
|
||||
cell_text = f"{paid}/{fee_display}"
|
||||
elif paid > 0:
|
||||
status = "partial"
|
||||
cell_text = f"{paid}/{fee_display}"
|
||||
amount_to_pay = expected - paid
|
||||
if m < current_month:
|
||||
unpaid_months.append(month_labels[m])
|
||||
raw_unpaid_months.append(
|
||||
datetime.strptime(m, "%Y-%m").strftime("%m/%Y")
|
||||
)
|
||||
else:
|
||||
status = "unpaid"
|
||||
cell_text = f"0/{fee_display}"
|
||||
amount_to_pay = expected
|
||||
if m < current_month:
|
||||
unpaid_months.append(month_labels[m])
|
||||
raw_unpaid_months.append(
|
||||
datetime.strptime(m, "%Y-%m").strftime("%m/%Y")
|
||||
)
|
||||
elif paid > 0:
|
||||
status = "surplus"
|
||||
cell_text = f"PAID {paid}"
|
||||
|
||||
if (isinstance(expected, int) and expected > 0) or paid > 0:
|
||||
tooltip = f"Received: {paid}, Expected: {expected}"
|
||||
else:
|
||||
tooltip = ""
|
||||
|
||||
row["months"].append({
|
||||
"text": cell_text,
|
||||
"overridden": is_overridden,
|
||||
"status": status,
|
||||
"amount": amount_to_pay,
|
||||
"month": month_labels[m],
|
||||
"raw_month": m,
|
||||
"tooltip": tooltip,
|
||||
})
|
||||
|
||||
settled_balance = 0
|
||||
for m, mdata in data["months"].items():
|
||||
if m >= current_month:
|
||||
continue
|
||||
exp = mdata.get("expected", 0)
|
||||
if isinstance(exp, int):
|
||||
settled_balance += int(mdata.get("paid", 0)) - exp
|
||||
|
||||
payable_amount = max(0, -settled_balance)
|
||||
row["unpaid_periods"] = ", ".join(unpaid_months)
|
||||
row["raw_unpaid_periods"] = "+".join(raw_unpaid_months)
|
||||
row["balance"] = settled_balance
|
||||
row["payable_amount"] = payable_amount
|
||||
formatted_results.append(row)
|
||||
|
||||
formatted_totals = []
|
||||
for m in sorted_months:
|
||||
t = monthly_totals[m]
|
||||
status = "empty"
|
||||
if t["expected"] > 0 or t["paid"] > 0:
|
||||
if t["paid"] == t["expected"]:
|
||||
status = "ok"
|
||||
elif t["paid"] < t["expected"]:
|
||||
status = "unpaid"
|
||||
else:
|
||||
status = "surplus"
|
||||
formatted_totals.append({
|
||||
"text": f"{t['paid']} / {t['expected']} CZK",
|
||||
"status": status,
|
||||
})
|
||||
|
||||
junior_all_names = [name for name, _, _ in adapted_members]
|
||||
|
||||
def _junior_settled_balance(name):
|
||||
data = result["members"][name]
|
||||
total = 0
|
||||
for m, mdata in data["months"].items():
|
||||
if m >= current_month:
|
||||
continue
|
||||
exp = mdata.get("expected", 0)
|
||||
if isinstance(exp, int):
|
||||
total += int(mdata.get("paid", 0)) - exp
|
||||
return total
|
||||
|
||||
credits = sorted(
|
||||
[{"name": n, "amount": _junior_settled_balance(n)} for n in junior_all_names if _junior_settled_balance(n) > 0],
|
||||
key=lambda x: x["name"],
|
||||
)
|
||||
debts = sorted(
|
||||
[{"name": n, "amount": abs(_junior_settled_balance(n))} for n in junior_all_names if _junior_settled_balance(n) < 0],
|
||||
key=lambda x: x["name"],
|
||||
)
|
||||
|
||||
raw_payments_by_person = group_payments_by_person(
|
||||
transactions, [name for name, _, _ in adapted_members]
|
||||
)
|
||||
|
||||
return dict(
|
||||
months=[month_labels[m] for m in sorted_months],
|
||||
raw_months=sorted_months,
|
||||
results=formatted_results,
|
||||
totals=formatted_totals,
|
||||
member_data=json.dumps(result["members"]),
|
||||
month_labels_json=json.dumps(month_labels),
|
||||
raw_payments_json=json.dumps(raw_payments_by_person),
|
||||
credits=credits,
|
||||
debts=debts,
|
||||
unmatched=result["unmatched"],
|
||||
attendance_url=attendance_url,
|
||||
payments_url=payments_url,
|
||||
bank_account=bank_account,
|
||||
current_month=current_month,
|
||||
)
|
||||
|
||||
|
||||
def build_payments_view_model(transactions, member_names, *, attendance_url, payments_url):
|
||||
grouped = group_payments_by_person(transactions, member_names)
|
||||
for tx in transactions:
|
||||
if not str(tx.get("person", "")).strip():
|
||||
grouped.setdefault("Unmatched / Unknown", []).append(tx)
|
||||
for rows in grouped.values():
|
||||
rows.sort(key=lambda t: str(t.get("date", "")), reverse=True)
|
||||
sorted_people = sorted(grouped.keys())
|
||||
return dict(
|
||||
grouped_payments=grouped,
|
||||
sorted_people=sorted_people,
|
||||
attendance_url=attendance_url,
|
||||
payments_url=payments_url,
|
||||
)
|
||||
Reference in New Issue
Block a user