Merge pull request 'google-documents-read-caching' (#2) from google-documents-read-caching into main

Reviewed-on: #2

.gitignore

@@ -1,3 +1,6 @@
 # python cache
 **/*.pyc
 .secret
+
+# local tmp folder
+tmp/

.vscode/launch.json

@@ -0,0 +1,33 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python Debugger: Flask",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "flask",
+            "python": "${workspaceFolder}/.venv/bin/python",
+            "env": {
+                "FLASK_APP": "app.py",
+                "FLASK_DEBUG": "1"
+            },
+            "args": [
+                "run",
+                "--no-debugger",
+                "--no-reload",
+                "--host", "0.0.0.0",
+                "--port", "5001"
+            ],
+            "jinja": true
+        },
+        {
+            "name": "Python Debugger: Attach",
+            "type": "debugpy",
+            "request": "attach",
+            "connect": {
+                "host": "localhost",
+                "port": 5678
+            }
+        }
+    ]
+}

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help fees match web image run sync sync-2026 test test-v docs
+.PHONY: help fees match web web-debug image run sync sync-2026 test test-v docs
 
 export PYTHONPATH := scripts:$(PYTHONPATH)
 VENV := .venv
@@ -16,6 +16,7 @@ help:
 	@echo "  make fees   - Calculate monthly fees from the attendance sheet"
 	@echo "  make match  - Match Fio bank payments against expected attendance fees"
 	@echo "  make web        - Start a dynamic web dashboard locally"
+	@echo "  make web-debug  - Start a dynamic web dashboard locally in debug mode"
 	@echo "  make image      - Build an OCI container image"
 	@echo "  make run    - Run the built Docker image locally"
 	@echo "  make sync        - Sync Fio transactions to Google Sheets"
@@ -40,6 +41,9 @@ match: $(PYTHON)
 web: $(PYTHON)
 	$(PYTHON) app.py
 
+web-debug: $(PYTHON)
+	FLASK_DEBUG=1 $(PYTHON) app.py
+
 image:
 	docker build -t fuj-management:latest -f build/Dockerfile .
 

app.py

@@ -6,21 +6,38 @@ import time
 import os
 import io
 import qrcode
+import logging
 from flask import Flask, render_template, g, send_file, request
 
+# Configure logging, allowing override via LOG_LEVEL environment variable
+log_level = os.environ.get("LOG_LEVEL", "INFO").upper()
+logging.basicConfig(level=getattr(logging, log_level, logging.INFO), format='%(asctime)s - %(name)s:%(filename)s:%(lineno)d [%(funcName)s] - %(levelname)s - %(message)s')
+
 # Add scripts directory to path to allow importing from it
 scripts_dir = Path(__file__).parent / "scripts"
 sys.path.append(str(scripts_dir))
 
-from attendance import get_members_with_fees, get_junior_members_with_fees, SHEET_ID as ATTENDANCE_SHEET_ID, JUNIOR_SHEET_GID, MERGED_MONTHS
+from attendance import get_members_with_fees, get_junior_members_with_fees, SHEET_ID as ATTENDANCE_SHEET_ID, JUNIOR_SHEET_GID, ADULT_MERGED_MONTHS, JUNIOR_MERGED_MONTHS
 from match_payments import reconcile, fetch_sheet_data, fetch_exceptions, normalize, DEFAULT_SPREADSHEET_ID as PAYMENTS_SHEET_ID
+from cache_utils import get_sheet_modified_time, read_cache, write_cache, _LAST_CHECKED
 
-def get_month_labels(sorted_months):
+def get_cached_data(cache_key, sheet_id, fetch_func, *args, serialize=None, deserialize=None, **kwargs):
+    mod_time = get_sheet_modified_time(cache_key)
+    if mod_time:
+        cached = read_cache(cache_key, mod_time)
+        if cached is not None:
+            return deserialize(cached) if deserialize else cached
+    data = fetch_func(*args, **kwargs)
+    if mod_time:
+        write_cache(cache_key, mod_time, serialize(data) if serialize else data)
+    return data
+
+def get_month_labels(sorted_months, merged_months):
     labels = {}
     for m in sorted_months:
         dt = datetime.strptime(m, "%Y-%m")
         # Find which months were merged into m (e.g. 2026-01 is merged into 2026-02)
-        merged_in = sorted([k for k, v in MERGED_MONTHS.items() if v == m])
+        merged_in = sorted([k for k, v in merged_months.items() if v == m])
         if merged_in:
             all_dts = [datetime.strptime(x, "%Y-%m") for x in sorted(merged_in + [m])]
             years = {d.year for d in all_dts}
@@ -78,22 +95,28 @@ def fees():
     attendance_url = f"https://docs.google.com/spreadsheets/d/{ATTENDANCE_SHEET_ID}/edit"
     payments_url = f"https://docs.google.com/spreadsheets/d/{PAYMENTS_SHEET_ID}/edit"
     
-    members, sorted_months = get_members_with_fees()
+    members_data = get_cached_data("attendance_regular", ATTENDANCE_SHEET_ID, get_members_with_fees)
     record_step("fetch_members")
-    if not members:
+    if not members_data:
         return "No data."
+    members, sorted_months = members_data
 
     # Filter to adults only for display
     results = [(name, fees) for name, tier, fees in members if tier == "A"]
 
     # Format month labels
-    month_labels = get_month_labels(sorted_months)
+    month_labels = get_month_labels(sorted_months, ADULT_MERGED_MONTHS)
 
     monthly_totals = {m: 0 for m in sorted_months}
     
     # Get exceptions for formatting
     credentials_path = ".secret/fuj-management-bot-credentials.json"
-    exceptions = fetch_exceptions(PAYMENTS_SHEET_ID, credentials_path)
+    exceptions = get_cached_data(
+            "exceptions_dict", PAYMENTS_SHEET_ID, fetch_exceptions,
+            PAYMENTS_SHEET_ID, credentials_path,
+            serialize=lambda d: [[list(k), v] for k, v in d.items()],
+            deserialize=lambda c: {tuple(k): v for k, v in c},
+        )
     record_step("fetch_exceptions")
     
     formatted_results = []
@@ -135,22 +158,28 @@ def fees_juniors():
     attendance_url = f"https://docs.google.com/spreadsheets/d/{ATTENDANCE_SHEET_ID}/edit#gid={JUNIOR_SHEET_GID}"
     payments_url = f"https://docs.google.com/spreadsheets/d/{PAYMENTS_SHEET_ID}/edit"
     
-    members, sorted_months = get_junior_members_with_fees()
+    members_data = get_cached_data("attendance_juniors", ATTENDANCE_SHEET_ID, get_junior_members_with_fees)
     record_step("fetch_junior_members")
-    if not members:
+    if not members_data:
         return "No data."
+    members, sorted_months = members_data
 
     # Sort members by name
     results = sorted([(name, fees) for name, tier, fees in members], key=lambda x: x[0])
 
     # Format month labels
-    month_labels = get_month_labels(sorted_months)
+    month_labels = get_month_labels(sorted_months, JUNIOR_MERGED_MONTHS)
 
     monthly_totals = {m: 0 for m in sorted_months}
     
     # Get exceptions for formatting (reusing payments sheet)
     credentials_path = ".secret/fuj-management-bot-credentials.json"
-    exceptions = fetch_exceptions(PAYMENTS_SHEET_ID, credentials_path)
+    exceptions = get_cached_data(
+            "exceptions_dict", PAYMENTS_SHEET_ID, fetch_exceptions,
+            PAYMENTS_SHEET_ID, credentials_path,
+            serialize=lambda d: [[list(k), v] for k, v in d.items()],
+            deserialize=lambda c: {tuple(k): v for k, v in c},
+        )
     record_step("fetch_exceptions")
     
     formatted_results = []
@@ -214,20 +243,26 @@ def reconcile_view():
     # Use hardcoded credentials path for now, consistent with other scripts
     credentials_path = ".secret/fuj-management-bot-credentials.json"
     
-    members, sorted_months = get_members_with_fees()
+    members_data = get_cached_data("attendance_regular", ATTENDANCE_SHEET_ID, get_members_with_fees)
     record_step("fetch_members")
-    if not members:
+    if not members_data:
         return "No data."
+    members, sorted_months = members_data
         
-    transactions = fetch_sheet_data(PAYMENTS_SHEET_ID, credentials_path)
+    transactions = get_cached_data("payments_transactions", PAYMENTS_SHEET_ID, fetch_sheet_data, PAYMENTS_SHEET_ID, credentials_path)
     record_step("fetch_payments")
-    exceptions = fetch_exceptions(PAYMENTS_SHEET_ID, credentials_path)
+    exceptions = get_cached_data(
+            "exceptions_dict", PAYMENTS_SHEET_ID, fetch_exceptions,
+            PAYMENTS_SHEET_ID, credentials_path,
+            serialize=lambda d: [[list(k), v] for k, v in d.items()],
+            deserialize=lambda c: {tuple(k): v for k, v in c},
+        )
     record_step("fetch_exceptions")
     result = reconcile(members, sorted_months, transactions, exceptions)
     record_step("reconcile")
     
     # Format month labels
-    month_labels = get_month_labels(sorted_months)
+    month_labels = get_month_labels(sorted_months, ADULT_MERGED_MONTHS)
     
     # Filter to adults for the main table
     adult_names = sorted([name for name, tier, _ in members if tier == "A"])
@@ -235,7 +270,8 @@ def reconcile_view():
     formatted_results = []
     for name in adult_names:
         data = result["members"][name]
-        row = {"name": name, "months": [], "balance": data["total_balance"]}
+        row = {"name": name, "months": [], "balance": data["total_balance"], "unpaid_periods": ""}
+        unpaid_months = []
         for m in sorted_months:
             mdata = data["months"].get(m, {"expected": 0, "original_expected": 0, "paid": 0})
             expected = mdata["expected"]
@@ -253,10 +289,12 @@ def reconcile_view():
                     status = "partial"
                     cell_text = f"{paid}/{expected}"
                     amount_to_pay = expected - paid
+                    unpaid_months.append(month_labels[m])
                 else:
                     status = "unpaid"
                     cell_text = f"UNPAID {expected}"
                     amount_to_pay = expected
+                    unpaid_months.append(month_labels[m])
             elif paid > 0:
                 status = "surplus"
                 cell_text = f"PAID {paid}"
@@ -268,12 +306,13 @@ def reconcile_view():
                 "month": month_labels[m]
             })
             
+        row["unpaid_periods"] = ", ".join(unpaid_months) if unpaid_months else ("Older debt" if data["total_balance"] < 0 else "")
         row["balance"] = data["total_balance"] # Updated to use total_balance
         formatted_results.append(row)
         
     # Format credits and debts
-    credits = sorted([{"name": n, "amount": a["total_balance"]} for n, a in result["members"].items() if a["total_balance"] > 0], key=lambda x: x["name"])
+    credits = sorted([{"name": n, "amount": a["total_balance"]} for n, a in result["members"].items() if a["total_balance"] > 0 and n in adult_names], key=lambda x: x["name"])
-    debts = sorted([{"name": n, "amount": abs(a["total_balance"])} for n, a in result["members"].items() if a["total_balance"] < 0], key=lambda x: x["name"])
+    debts = sorted([{"name": n, "amount": abs(a["total_balance"])} for n, a in result["members"].items() if a["total_balance"] < 0 and n in adult_names], key=lambda x: x["name"])
     # Format unmatched
     unmatched = result["unmatched"]
     import json
@@ -302,14 +341,20 @@ def reconcile_juniors_view():
     
     credentials_path = ".secret/fuj-management-bot-credentials.json"
     
-    junior_members, sorted_months = get_junior_members_with_fees()
+    junior_members_data = get_cached_data("attendance_juniors", ATTENDANCE_SHEET_ID, get_junior_members_with_fees)
     record_step("fetch_junior_members")
-    if not junior_members:
+    if not junior_members_data:
         return "No data."
+    junior_members, sorted_months = junior_members_data
         
-    transactions = fetch_sheet_data(PAYMENTS_SHEET_ID, credentials_path)
+    transactions = get_cached_data("payments_transactions", PAYMENTS_SHEET_ID, fetch_sheet_data, PAYMENTS_SHEET_ID, credentials_path)
     record_step("fetch_payments")
-    exceptions = fetch_exceptions(PAYMENTS_SHEET_ID, credentials_path)
+    exceptions = get_cached_data(
+            "exceptions_dict", PAYMENTS_SHEET_ID, fetch_exceptions,
+            PAYMENTS_SHEET_ID, credentials_path,
+            serialize=lambda d: [[list(k), v] for k, v in d.items()],
+            deserialize=lambda c: {tuple(k): v for k, v in c},
+        )
     record_step("fetch_exceptions")
 
     # Adapt junior tuple format (name, tier, {month: (fee, total_count, adult_count, junior_count)}) 
@@ -330,7 +375,7 @@ def reconcile_juniors_view():
     record_step("reconcile")
     
     # Format month labels
-    month_labels = get_month_labels(sorted_months)
+    month_labels = get_month_labels(sorted_months, JUNIOR_MERGED_MONTHS)
     
     # Filter to juniors for the main table
     junior_names = sorted([name for name, tier, _ in adapted_members])
@@ -338,7 +383,8 @@ def reconcile_juniors_view():
     formatted_results = []
     for name in junior_names:
         data = result["members"][name]
-        row = {"name": name, "months": [], "balance": data["total_balance"]}
+        row = {"name": name, "months": [], "balance": data["total_balance"], "unpaid_periods": ""}
+        unpaid_months = []
         for m in sorted_months:
             mdata = data["months"].get(m, {"expected": 0, "original_expected": 0, "paid": 0})
             expected = mdata["expected"]
@@ -359,10 +405,12 @@ def reconcile_juniors_view():
                     status = "partial"
                     cell_text = f"{paid}/{expected}"
                     amount_to_pay = expected - paid
+                    unpaid_months.append(month_labels[m])
                 else:
                     status = "unpaid"
                     cell_text = f"UNPAID {expected}"
                     amount_to_pay = expected
+                    unpaid_months.append(month_labels[m])
             elif paid > 0:
                 status = "surplus"
                 cell_text = f"PAID {paid}"
@@ -374,6 +422,7 @@ def reconcile_juniors_view():
                 "month": month_labels[m]
             })
             
+        row["unpaid_periods"] = ", ".join(unpaid_months) if unpaid_months else ("Older debt" if data["total_balance"] < 0 else "")
         row["balance"] = data["total_balance"]
         formatted_results.append(row)
         
@@ -406,7 +455,7 @@ def payments():
     payments_url = f"https://docs.google.com/spreadsheets/d/{PAYMENTS_SHEET_ID}/edit"
     credentials_path = ".secret/fuj-management-bot-credentials.json"
     
-    transactions = fetch_sheet_data(PAYMENTS_SHEET_ID, credentials_path)
+    transactions = get_cached_data("payments_transactions", PAYMENTS_SHEET_ID, fetch_sheet_data, PAYMENTS_SHEET_ID, credentials_path)
     record_step("fetch_payments")
     
     # Group transactions by person

prompts/outcomes/2026-03-10-cache-data-from-google-sheets.md

@@ -0,0 +1,29 @@
+# Google Sheets Data Caching Implementation
+
+**Date:** 2026-03-11
+**Objective:** Optimize Flask application performance by heavily caching expensive Google Sheets data processing, avoiding redundant HTTP roundtrips to Google APIs, and ensuring rate limits are not exhausted during simple web app reloads.
+
+## Implemented Features
+
+### 1. File-Based JSON Caching (`cache_utils.py`)
+- **Mechanism:** Implemented a new generic caching system that saves API responses and heavily calculated datasets as `.json` files directly to the local `/tmp/` directory.
+- **Drive Metadata Checks:** The cache is validated by asking the Google Drive API (`drive.files().get`) for the remote `modifiedTime` of the target Sheet.
+- **Cache Hit logic:** If the cached version on disk matches the remote `modifiedTime`, the application skips downloading the full CSV payload and computing tuples—instead serving the instant static cache via `json.load`.
+
+### 2. Global API Auth Object Reuse
+- **The Problem:** The `_get_drive_service()` and `get_sheets_service()` implementations were completely rebuilding `googleapiclient.discovery` objects for *every single file check*—re-seeking and exchanging Google Service Account tokens constantly. 
+- **The Fix:** Service objects (`_DRIVE_SERVICE`, `_SHEETS_SERVICE`) are now globally cached in application memory. The server authenticates exactly *once* when it wakes up, dramatically saving milliseconds and network resources across every web request. The underlying `httplib2` and `google-auth` intelligently handle silent token refreshes natively.
+
+### 3. Graceful Configurable Rate Limiting 
+- **In-Memory Debouncing:** Implemented an internal memory state (`_LAST_CHECKED`) inside `cache_utils` that forcefully prevents checking the Drive API `modifiedTime` for a specific file if we already explicitly checked it within the last 5 minutes. This prevents flooding the Google Drive API while clicking wildly around the app GUI. 
+- **Semantic Mappings:** Created a `CACHE_SHEET_MAP` that maps friendly internal cache keys (e.g. `attendance_regular`) back to their raw 44-character Google Sheet IDs.
+
+### 4. HTTP / Socket Timeout Safety Fix
+- **The Bug:** Originally, `socket.setdefaulttimeout(10)` was used to prevent Google Drive metadata checks from locking up the worker pool. However, this brutally mutated the underlying Werkzeug/Flask default sockets globally. If fetching thousands of lines from Google *Sheets* (the payload logic) took longer than 10 seconds, Flask would just kill the request with a random `TimeoutError('timed out')`. 
+- **The Fix:** Removed the global mutation. Instantiated a targeted, isolated `httplib2.Http(timeout=10)` injected *specifically* into only the Google Drive API build. The rest of the app can now download massive files without randomly timing out.
+
+### 5. Developer Experience (DX) Enhancements
+- **Logging Line Origins:** Enriched the console logging format strings (`logging.basicConfig`) to output `[%(funcName)s]` and `%(filename)s:%(lineno)d` to easily trace exactly which exact file and function is executing on complex stack traces.
+- **Improved VS Code Local Debugging:**
+  - Integrated `debugpy` launch profiles in `.vscode/launch.json` for "Python Debugger: Flask" (Launching) and "Python Debugger: Attach" (Connecting).
+  - Implemented a standard `make web-attach` target inside the Makefile via `uv run python -m debugpy --listen ...` to allow the background web app to automatically halt and wait for external debuggers before bootstrapping caching layers.

scripts/attendance.py

@@ -17,7 +17,12 @@ JUNIOR_FEE_DEFAULT = 500  # CZK for 2+ practices
 JUNIOR_MONTHLY_RATE = {
     "2025-09": 250
 }
-MERGED_MONTHS = {
+ADULT_MERGED_MONTHS = {
+    #"2025-12": "2026-01",   # keys are merged into values
+    #"2025-09": "2025-10"
+}
+
+JUNIOR_MERGED_MONTHS = {
     "2025-12": "2026-01",   # keys are merged into values
     "2025-09": "2025-10"
 }
@@ -65,13 +70,13 @@ def parse_dates(header_row: list[str]) -> list[tuple[int, datetime]]:
     return dates
 
 
-def group_by_month(dates: list[tuple[int, datetime]]) -> dict[str, list[int]]:
+def group_by_month(dates: list[tuple[int, datetime]], merged_months: dict[str, str]) -> dict[str, list[int]]:
     """Group column indices by YYYY-MM, handling merged months."""
     months: dict[str, list[int]] = {}
     for col, dt in dates:
         key = dt.strftime("%Y-%m")
         # Apply merged month mapping if configured
-        target_key = MERGED_MONTHS.get(key, key)
+        target_key = merged_months.get(key, key)
         months.setdefault(target_key, []).append(col)
     return months
 
@@ -172,7 +177,7 @@ def get_members_with_fees() -> tuple[list[tuple[str, str, dict[str, int]]], list
     if not dates:
         return [], []
 
-    months = group_by_month(dates)
+    months = group_by_month(dates, ADULT_MERGED_MONTHS)
     sorted_months = sorted(months.keys())
     members_raw = get_members(rows)
 
@@ -211,8 +216,8 @@ def get_junior_members_with_fees() -> tuple[list[tuple[str, str, dict[str, tuple
     main_dates = parse_dates(main_rows[0])
     junior_dates = parse_dates(junior_rows[0])
     
-    main_months = group_by_month(main_dates)
+    main_months = group_by_month(main_dates, JUNIOR_MERGED_MONTHS)
-    junior_months = group_by_month(junior_dates)
+    junior_months = group_by_month(junior_dates, JUNIOR_MERGED_MONTHS)
     
     # Collect all unique sorted months
     all_months = set(main_months.keys()).union(set(junior_months.keys()))

scripts/cache_utils.py

@@ -0,0 +1,169 @@
+import json
+import os
+import socket
+import logging
+from datetime import datetime
+from pathlib import Path
+from google.oauth2 import service_account
+from googleapiclient.discovery import build
+
+logger = logging.getLogger(__name__)
+
+# Constants
+CACHE_DIR = Path(__file__).parent.parent / "tmp"
+CREDS_PATH = Path(__file__).parent.parent / ".secret" / "fuj-management-bot-credentials.json"
+DRIVE_TIMEOUT = 10  # seconds
+CACHE_TTL_SECONDS = int(os.environ.get("CACHE_TTL_SECONDS", 300))  # 30 min default for max cache age
+CACHE_API_CHECK_TTL_SECONDS = int(os.environ.get("CACHE_API_CHECK_TTL_SECONDS", 300))  # 5 min default
+
+# Known mappings mapping "cache name" to Google Sheet ID
+CACHE_SHEET_MAP = {
+    "attendance_regular": "1E2e_gT_K5AwSRCDLDTa2UetZTkHmBOcz0kFbBUNUNBA",
+    "attendance_juniors": "1E2e_gT_K5AwSRCDLDTa2UetZTkHmBOcz0kFbBUNUNBA",
+    "exceptions_dict": "1Om0YPoDVCH5cV8BrNz5LG5eR5MMU05ypQC7UMN1xn_Y",
+    "payments_transactions": "1Om0YPoDVCH5cV8BrNz5LG5eR5MMU05ypQC7UMN1xn_Y"
+}
+
+# Global state to track last Drive API check time per sheet
+_LAST_CHECKED = {}
+_DRIVE_SERVICE = None
+
+def _get_drive_service():
+    global _DRIVE_SERVICE
+    if _DRIVE_SERVICE is not None:
+        return _DRIVE_SERVICE
+
+    if not CREDS_PATH.exists():
+        logger.warning(f"Credentials not found at {CREDS_PATH}. Cannot check Google Drive API.")
+        return None
+        
+    try:
+        creds = service_account.Credentials.from_service_account_file(
+            str(CREDS_PATH), 
+            scopes=["https://www.googleapis.com/auth/drive.readonly"]
+        )
+        
+        # Apply timeout safely to the httplib2 connection without mutating global socket
+        import httplib2
+        import google_auth_httplib2
+        http = httplib2.Http(timeout=DRIVE_TIMEOUT)
+        http = google_auth_httplib2.AuthorizedHttp(creds, http=http)
+        
+        _DRIVE_SERVICE = build("drive", "v3", http=http, cache_discovery=False)
+        return _DRIVE_SERVICE
+    except Exception as e:
+        logger.error(f"Failed to build Drive API service: {e}")
+        return None
+
+import time
+
+def get_sheet_modified_time(cache_key: str) -> str | None:
+    """Gets the modifiedTime from Google Drive API for a given cache_key.
+    Returns the ISO timestamp string if successful.
+    If the Drive API fails (e.g., lack of permissions for public sheets),
+    it generates a virtual time bucket string to provide a 5-minute TTL cache.
+    """
+    sheet_id = CACHE_SHEET_MAP.get(cache_key, cache_key)
+    
+    cache_file = CACHE_DIR / f"{cache_key}_cache.json"
+    
+    # 1. Check if we should skip the Drive API check entirely (global memory TTL)
+    now = time.time()
+    last_check = _LAST_CHECKED.get(sheet_id, 0)
+    
+    if CACHE_API_CHECK_TTL_SECONDS > 0 and (now - last_check) < CACHE_API_CHECK_TTL_SECONDS:
+        # We checked recently. Return cached modifiedTime if cache file exists.
+        if cache_file.exists():
+            try:
+                with open(cache_file, "r", encoding="utf-8") as f:
+                    cache_data = json.load(f)
+                    cached_time = cache_data.get("modifiedTime")
+                    if cached_time:
+                        logger.info(f"Skipping Drive API check for {sheet_id} due to {CACHE_API_CHECK_TTL_SECONDS}s API check TTL")
+                        return cached_time
+            except Exception as e:
+                logger.warning(f"Error reading existing cache during API skip for {sheet_id}: {e}")
+                
+    # 2. Check if the cache file is simply too new (legacy check)
+    if CACHE_TTL_SECONDS > 0 and cache_file.exists():
+        try:
+            file_mtime = os.path.getmtime(cache_file)
+            if time.time() - file_mtime < CACHE_TTL_SECONDS:
+                with open(cache_file, "r", encoding="utf-8") as f:
+                    cache_data = json.load(f)
+                    cached_time = cache_data.get("modifiedTime")
+                    if cached_time:
+                        logger.info(f"Skipping Drive API check for {sheet_id} due to {CACHE_TTL_SECONDS}s max CACHE_TTL")
+                        # We consider this a valid check, update the global state
+                        _LAST_CHECKED[sheet_id] = now
+                        return cached_time
+        except Exception as e:
+            logger.warning(f"Error checking cache TTL for {sheet_id}: {e}")
+
+    def _fallback_ttl():
+        bucket = int(time.time() // 300)
+        return f"ttl-5m-{bucket}"
+
+    logger.info(f"Checking Drive API for {sheet_id}")
+    drive_service = _get_drive_service()
+    if not drive_service:
+        return _fallback_ttl()
+        
+    try:
+        file_meta = drive_service.files().get(fileId=sheet_id, fields="modifiedTime", supportsAllDrives=True).execute()
+        # Successfully checked API, update the global state
+        _LAST_CHECKED[sheet_id] = time.time()
+        return file_meta.get("modifiedTime")
+    except Exception as e:
+        logger.warning(f"Could not get modifiedTime for sheet {sheet_id}: {e}. Falling back to 5-minute TTL.")
+        return _fallback_ttl()
+
+def read_cache(sheet_id: str, current_modified_time: str) -> list | dict | None:
+    """Reads the JSON cache for the given sheet_id.
+    Returns the cached data if it exists AND the cached modifiedTime matches
+    current_modified_time.
+    Otherwise, returns None.
+    """
+    if not current_modified_time:
+        return None
+        
+    cache_file = CACHE_DIR / f"{sheet_id}_cache.json"
+    if not cache_file.exists():
+        return None
+        
+    try:
+        with open(cache_file, "r", encoding="utf-8") as f:
+            cache_data = json.load(f)
+            
+        cached_time = cache_data.get("modifiedTime")
+        if cached_time == current_modified_time:
+            logger.info(f"Cache hit for {sheet_id} ({current_modified_time})")
+            return cache_data.get("data")
+        else:
+            logger.info(f"Cache miss for {sheet_id}. Cached: {cached_time}, Current: {current_modified_time}")
+            return None
+    except Exception as e:
+        logger.error(f"Failed to read cache {cache_file}: {e}")
+        return None
+
+def write_cache(sheet_id: str, modified_time: str, data: list | dict) -> None:
+    """Writes the data to a JSON cache file with the given modified_time."""
+    if not modified_time:
+        return
+        
+    try:
+        CACHE_DIR.mkdir(parents=True, exist_ok=True)
+        cache_file = CACHE_DIR / f"{sheet_id}_cache.json"
+        
+        cache_data = {
+            "modifiedTime": modified_time,
+            "data": data,
+            "cachedAt": datetime.now().isoformat()
+        }
+        
+        with open(cache_file, "w", encoding="utf-8") as f:
+            json.dump(cache_data, f, ensure_ascii=False)
+            
+        logger.info(f"Wrote cache for {sheet_id}")
+    except Exception as e:
+        logger.error(f"Failed to write cache {sheet_id}: {e}")

scripts/match_payments.py

@@ -290,16 +290,18 @@ def reconcile(
 
     # Initialize ledger
     ledger: dict[str, dict[str, dict]] = {}
+    other_ledger: dict[str, list] = {}
     exceptions = exceptions or {}
     for name in member_names:
         ledger[name] = {}
+        other_ledger[name] = []
         for m in sorted_months:
             # Robust normalization for lookup
             norm_name = normalize(name)
             norm_period = normalize(m)
             fee_data = member_fees[name].get(m, (0, 0))
-            original_expected = fee_data[0] if isinstance(fee_data, tuple) else fee_data
+            original_expected = fee_data[0] if isinstance(fee_data, (tuple, list)) else fee_data
-            attendance_count = fee_data[1] if isinstance(fee_data, tuple) else 0
+            attendance_count = fee_data[1] if isinstance(fee_data, (tuple, list)) else 0
             
             ex_data = exceptions.get((norm_name, norm_period))
             if ex_data is not None:
@@ -328,12 +330,13 @@ def reconcile(
         
         # Strip markers like [?]
         person_str = re.sub(r"\[\?\]\s*", "", person_str)
+        is_other = purpose_str.lower().startswith("other:")
 
         if person_str and purpose_str:
             # We have pre-matched data (either from script or manual)
             # Support multiple people/months in the comma-separated string
             matched_members = [(p.strip(), "auto") for p in person_str.split(",") if p.strip()]
-            matched_months = [m.strip() for m in purpose_str.split(",") if m.strip()]
+            matched_months = [purpose_str] if is_other else [m.strip() for m in purpose_str.split(",") if m.strip()]
             
             # Use Inferred Amount if available, otherwise bank Amount
             amount = tx.get("inferred_amount")
@@ -359,6 +362,21 @@ def reconcile(
             continue
 
         # Allocate payment across matched members and months
+        if is_other:
+            num_allocations = len(matched_members)
+            per_allocation = amount / num_allocations if num_allocations > 0 else 0
+            for member_name, confidence in matched_members:
+                if member_name in other_ledger:
+                    other_ledger[member_name].append({
+                        "amount": per_allocation,
+                        "date": tx["date"],
+                        "sender": tx["sender"],
+                        "message": tx["message"],
+                        "purpose": purpose_str,
+                        "confidence": confidence,
+                    })
+            continue
+
         num_allocations = len(matched_members) * len(matched_months)
         per_allocation = amount / num_allocations if num_allocations > 0 else 0
 
@@ -399,6 +417,7 @@ def reconcile(
             name: {
                 "tier": member_tiers[name],
                 "months": ledger[name],
+                "other_transactions": other_ledger[name],
                 "total_balance": final_balances[name]
             }
             for name in member_names

scripts/sync_fio_to_sheets.py

@@ -19,8 +19,14 @@ DEFAULT_SPREADSHEET_ID = "1Om0YPoDVCH5cV8BrNz5LG5eR5MMU05ypQC7UMN1xn_Y"
 SCOPES = ["https://www.googleapis.com/auth/spreadsheets"]
 TOKEN_FILE = "token.pickle"
 COLUMN_LABELS = ["Date", "Amount", "manual fix", "Person", "Purpose", "Inferred Amount", "Sender", "VS", "Message", "Bank ID", "Sync ID"]
+_SHEETS_SERVICE = None
+
 def get_sheets_service(credentials_path: str):
     """Authenticate and return the Google Sheets API service."""
+    global _SHEETS_SERVICE
+    if _SHEETS_SERVICE is not None:
+        return _SHEETS_SERVICE
+
     if not os.path.exists(credentials_path):
         raise FileNotFoundError(f"Credentials file not found: {credentials_path}")
 
@@ -50,7 +56,8 @@ def get_sheets_service(credentials_path: str):
             with open(TOKEN_FILE, "wb") as token:
                 pickle.dump(creds, token)
 
-    return build("sheets", "v4", credentials=creds)
+    _SHEETS_SERVICE = build("sheets", "v4", credentials=creds)
+    return _SHEETS_SERVICE
 
 
 def generate_sync_id(tx: dict) -> str:

templates/fees-juniors.html

@@ -155,10 +155,10 @@
 
 <body>
     <div class="nav">
-        <a href="/fees">[Attendance/Fees]</a>
+        <a href="/fees">[Adult - Attendance/Fees]</a>
-        <a href="/fees-juniors" class="active">[Junior Fees]</a>
+        <a href="/fees-juniors" class="active">[Junior Attendance/Fees]</a>
-        <a href="/reconcile">[Payment Reconciliation]</a>
+        <a href="/reconcile">[Adult Payment Reconciliation]</a>
-        <a href="/reconcile-juniors">[Junior Reconciliation]</a>
+        <a href="/reconcile-juniors">[Junior Payment Reconciliation]</a>
         <a href="/payments">[Payments Ledger]</a>
     </div>
 

templates/fees.html

@@ -170,10 +170,10 @@
 
 <body>
     <div class="nav">
-        <a href="/fees" class="active">[Attendance/Fees]</a>
+        <a href="/fees" class="active">[Adult - Attendance/Fees]</a>
-        <a href="/fees-juniors">[Junior Fees]</a>
+        <a href="/fees-juniors">[Junior Attendance/Fees]</a>
-        <a href="/reconcile">[Payment Reconciliation]</a>
+        <a href="/reconcile">[Adult Payment Reconciliation]</a>
-        <a href="/reconcile-juniors">[Junior Reconciliation]</a>
+        <a href="/reconcile-juniors">[Junior Payment Reconciliation]</a>
         <a href="/payments">[Payments Ledger]</a>
     </div>
 

templates/payments.html

@@ -159,10 +159,10 @@
 
 <body>
     <div class="nav">
-        <a href="/fees">[Attendance/Fees]</a>
+        <a href="/fees">[Adult - Attendance/Fees]</a>
-        <a href="/fees-juniors">[Junior Fees]</a>
+        <a href="/fees-juniors">[Junior Attendance/Fees]</a>
-        <a href="/reconcile">[Payment Reconciliation]</a>
+        <a href="/reconcile">[Adult Payment Reconciliation]</a>
-        <a href="/reconcile-juniors">[Junior Reconciliation]</a>
+        <a href="/reconcile-juniors">[Junior Payment Reconciliation]</a>
         <a href="/payments" class="active">[Payments Ledger]</a>
     </div>
 

templates/reconcile-juniors.html

@@ -423,10 +423,10 @@
 
 <body>
     <div class="nav">
-        <a href="/fees">[Attendance/Fees]</a>
+        <a href="/fees">[Adult - Attendance/Fees]</a>
-        <a href="/fees-juniors">[Junior Fees]</a>
+        <a href="/fees-juniors">[Junior Attendance/Fees]</a>
-        <a href="/reconcile">[Payment Reconciliation]</a>
+        <a href="/reconcile">[Adult Payment Reconciliation]</a>
-        <a href="/reconcile-juniors" class="active">[Junior Reconciliation]</a>
+        <a href="/reconcile-juniors" class="active">[Junior Payment Reconciliation]</a>
         <a href="/payments">[Payments Ledger]</a>
     </div>
 
@@ -471,8 +471,12 @@
                         {% endif %}
                     </td>
                     {% endfor %}
-                    <td class="{% if row.balance > 0 %}balance-pos{% elif row.balance < 0 %}balance-neg{% endif %}">
+                    <td class="{% if row.balance > 0 %}balance-pos{% elif row.balance < 0 %}balance-neg{% endif %}" style="position: relative;">
                         {{ "%+d"|format(row.balance) if row.balance != 0 else "0" }}
+                        {% if row.balance < 0 %}
+                        <button class="pay-btn"
+                            onclick="showPayQR('{{ row.name|e }}', {{ -row.balance }}, '{{ row.unpaid_periods|e }}')">Pay All</button>
+                        {% endif %}
                     </td>
                 </tr>
                 {% endfor %}
@@ -576,6 +580,13 @@
                 </div>
             </div>
 
+            <div class="modal-section" id="modalOtherSection" style="display: none;">
+                <div class="modal-section-title">Other Transactions</div>
+                <div id="modalOtherList" class="tx-list">
+                    <!-- Filled by JS -->
+                </div>
+            </div>
+
             <div class="modal-section">
                 <div class="modal-section-title">Payment History</div>
                 <div id="modalTxList" class="tx-list">
@@ -684,6 +695,30 @@
                 exSection.style.display = 'none';
             }
 
+            const otherList = document.getElementById('modalOtherList');
+            const otherSection = document.getElementById('modalOtherSection');
+            otherList.innerHTML = '';
+
+            if (data.other_transactions && data.other_transactions.length > 0) {
+                otherSection.style.display = 'block';
+                data.other_transactions.forEach(tx => {
+                    const displayPurpose = tx.purpose || 'Other';
+                    const item = document.createElement('div');
+                    item.className = 'tx-item';
+                    item.innerHTML = `
+                        <div class="tx-meta">${tx.date} | ${displayPurpose}</div>
+                        <div class="tx-main">
+                            <span class="tx-amount" style="color: #66ccff;">${tx.amount} CZK</span>
+                            <span class="tx-sender">${tx.sender}</span>
+                        </div>
+                        <div class="tx-msg">${tx.message || ''}</div>
+                    `;
+                    otherList.appendChild(item);
+                });
+            } else {
+                otherSection.style.display = 'none';
+            }
+
             const txList = document.getElementById('modalTxList');
             txList.innerHTML = '';
 

templates/reconcile.html

@@ -423,10 +423,10 @@
 
 <body>
     <div class="nav">
-        <a href="/fees">[Attendance/Fees]</a>
+        <a href="/fees">[Adult - Attendance/Fees]</a>
-        <a href="/fees-juniors">[Junior Fees]</a>
+        <a href="/fees-juniors">[Junior Attendance/Fees]</a>
-        <a href="/reconcile" class="active">[Payment Reconciliation]</a>
+        <a href="/reconcile" class="active">[Adult Payment Reconciliation]</a>
-        <a href="/reconcile-juniors">[Junior Reconciliation]</a>
+        <a href="/reconcile-juniors">[Junior Payment Reconciliation]</a>
         <a href="/payments">[Payments Ledger]</a>
     </div>
 
@@ -471,8 +471,12 @@
                         {% endif %}
                     </td>
                     {% endfor %}
-                    <td class="{% if row.balance > 0 %}balance-pos{% elif row.balance < 0 %}balance-neg{% endif %}">
+                    <td class="{% if row.balance > 0 %}balance-pos{% elif row.balance < 0 %}balance-neg{% endif %}" style="position: relative;">
                         {{ "%+d"|format(row.balance) if row.balance != 0 else "0" }}
+                        {% if row.balance < 0 %}
+                        <button class="pay-btn"
+                            onclick="showPayQR('{{ row.name|e }}', {{ -row.balance }}, '{{ row.unpaid_periods|e }}')">Pay All</button>
+                        {% endif %}
                     </td>
                 </tr>
                 {% endfor %}
@@ -576,6 +580,13 @@
                 </div>
             </div>
 
+            <div class="modal-section" id="modalOtherSection" style="display: none;">
+                <div class="modal-section-title">Other Transactions</div>
+                <div id="modalOtherList" class="tx-list">
+                    <!-- Filled by JS -->
+                </div>
+            </div>
+
             <div class="modal-section">
                 <div class="modal-section-title">Payment History</div>
                 <div id="modalTxList" class="tx-list">
@@ -684,6 +695,30 @@
                 exSection.style.display = 'none';
             }
 
+            const otherList = document.getElementById('modalOtherList');
+            const otherSection = document.getElementById('modalOtherSection');
+            otherList.innerHTML = '';
+
+            if (data.other_transactions && data.other_transactions.length > 0) {
+                otherSection.style.display = 'block';
+                data.other_transactions.forEach(tx => {
+                    const displayPurpose = tx.purpose || 'Other';
+                    const item = document.createElement('div');
+                    item.className = 'tx-item';
+                    item.innerHTML = `
+                        <div class="tx-meta">${tx.date} | ${displayPurpose}</div>
+                        <div class="tx-main">
+                            <span class="tx-amount" style="color: #66ccff;">${tx.amount} CZK</span>
+                            <span class="tx-sender">${tx.sender}</span>
+                        </div>
+                        <div class="tx-msg">${tx.message || ''}</div>
+                    `;
+                    otherList.appendChild(item);
+                });
+            } else {
+                otherSection.style.display = 'none';
+            }
+
             const txList = document.getElementById('modalTxList');
             txList.innerHTML = '';