feat: implement caching for google sheets data

- Add cache_utils.py with JSON caching for Google Sheets
- Authenticate and cache Drive/Sheets API services globally to reuse tokens
- Use CACHE_SHEET_MAP dict to resolve cache names securely to Sheet IDs
- Change app.py data fetching to skip downloads if modifiedTime matches cache
- Replace global socket timeout with httplib2 to fix Werkzeug timeouts
- Add VS Code attach debugpy configurations to launch.json and Makefile

app.py

@@ -6,14 +6,43 @@ import time
 import os
 import io
 import qrcode
+import logging
 from flask import Flask, render_template, g, send_file, request
 
+# Configure logging, allowing override via LOG_LEVEL environment variable
+log_level = os.environ.get("LOG_LEVEL", "INFO").upper()
+logging.basicConfig(level=getattr(logging, log_level, logging.INFO), format='%(asctime)s - %(name)s:%(filename)s:%(lineno)d [%(funcName)s] - %(levelname)s - %(message)s')
+
 # Add scripts directory to path to allow importing from it
 scripts_dir = Path(__file__).parent / "scripts"
 sys.path.append(str(scripts_dir))
 
 from attendance import get_members_with_fees, get_junior_members_with_fees, SHEET_ID as ATTENDANCE_SHEET_ID, JUNIOR_SHEET_GID, ADULT_MERGED_MONTHS, JUNIOR_MERGED_MONTHS
 from match_payments import reconcile, fetch_sheet_data, fetch_exceptions, normalize, DEFAULT_SPREADSHEET_ID as PAYMENTS_SHEET_ID
+from cache_utils import get_sheet_modified_time, read_cache, write_cache
+
+def get_cached_data(cache_key, sheet_id, fetch_func, *args, **kwargs):
+    mod_time = get_sheet_modified_time(cache_key)
+    if mod_time:
+        cached = read_cache(cache_key, mod_time)
+        if cached is not None:
+            return cached
+    data = fetch_func(*args, **kwargs)
+    if mod_time:
+        write_cache(cache_key, mod_time, data)
+    return data
+
+def get_cached_exceptions(sheet_id, creds_path):
+    cache_key = "exceptions_dict"
+    mod_time = get_sheet_modified_time(cache_key)
+    if mod_time:
+        cached = read_cache(cache_key, mod_time)
+        if cached is not None:
+            return {tuple(k): v for k, v in cached}
+    data = fetch_exceptions(sheet_id, creds_path)
+    if mod_time:
+        write_cache(cache_key, mod_time, [[list(k), v] for k, v in data.items()])
+    return data
 
 def get_month_labels(sorted_months, merged_months):
     labels = {}
@@ -78,10 +107,11 @@ def fees():
     attendance_url = f"https://docs.google.com/spreadsheets/d/{ATTENDANCE_SHEET_ID}/edit"
     payments_url = f"https://docs.google.com/spreadsheets/d/{PAYMENTS_SHEET_ID}/edit"
     
-    members, sorted_months = get_members_with_fees()
+    members_data = get_cached_data("attendance_regular", ATTENDANCE_SHEET_ID, get_members_with_fees)
     record_step("fetch_members")
-    if not members:
+    if not members_data:
         return "No data."
+    members, sorted_months = members_data
 
     # Filter to adults only for display
     results = [(name, fees) for name, tier, fees in members if tier == "A"]
@@ -93,7 +123,7 @@ def fees():
     
     # Get exceptions for formatting
     credentials_path = ".secret/fuj-management-bot-credentials.json"
-    exceptions = fetch_exceptions(PAYMENTS_SHEET_ID, credentials_path)
+    exceptions = get_cached_exceptions(PAYMENTS_SHEET_ID, credentials_path)
     record_step("fetch_exceptions")
     
     formatted_results = []
@@ -135,10 +165,11 @@ def fees_juniors():
     attendance_url = f"https://docs.google.com/spreadsheets/d/{ATTENDANCE_SHEET_ID}/edit#gid={JUNIOR_SHEET_GID}"
     payments_url = f"https://docs.google.com/spreadsheets/d/{PAYMENTS_SHEET_ID}/edit"
     
-    members, sorted_months = get_junior_members_with_fees()
+    members_data = get_cached_data("attendance_juniors", ATTENDANCE_SHEET_ID, get_junior_members_with_fees)
     record_step("fetch_junior_members")
-    if not members:
+    if not members_data:
         return "No data."
+    members, sorted_months = members_data
 
     # Sort members by name
     results = sorted([(name, fees) for name, tier, fees in members], key=lambda x: x[0])
@@ -150,7 +181,7 @@ def fees_juniors():
     
     # Get exceptions for formatting (reusing payments sheet)
     credentials_path = ".secret/fuj-management-bot-credentials.json"
-    exceptions = fetch_exceptions(PAYMENTS_SHEET_ID, credentials_path)
+    exceptions = get_cached_exceptions(PAYMENTS_SHEET_ID, credentials_path)
     record_step("fetch_exceptions")
     
     formatted_results = []
@@ -214,14 +245,15 @@ def reconcile_view():
     # Use hardcoded credentials path for now, consistent with other scripts
     credentials_path = ".secret/fuj-management-bot-credentials.json"
     
-    members, sorted_months = get_members_with_fees()
+    members_data = get_cached_data("attendance_regular", ATTENDANCE_SHEET_ID, get_members_with_fees)
     record_step("fetch_members")
-    if not members:
+    if not members_data:
         return "No data."
+    members, sorted_months = members_data
         
-    transactions = fetch_sheet_data(PAYMENTS_SHEET_ID, credentials_path)
+    transactions = get_cached_data("payments_transactions", PAYMENTS_SHEET_ID, fetch_sheet_data, PAYMENTS_SHEET_ID, credentials_path)
     record_step("fetch_payments")
-    exceptions = fetch_exceptions(PAYMENTS_SHEET_ID, credentials_path)
+    exceptions = get_cached_exceptions(PAYMENTS_SHEET_ID, credentials_path)
     record_step("fetch_exceptions")
     result = reconcile(members, sorted_months, transactions, exceptions)
     record_step("reconcile")
@@ -306,14 +338,15 @@ def reconcile_juniors_view():
     
     credentials_path = ".secret/fuj-management-bot-credentials.json"
     
-    junior_members, sorted_months = get_junior_members_with_fees()
+    junior_members_data = get_cached_data("attendance_juniors", ATTENDANCE_SHEET_ID, get_junior_members_with_fees)
     record_step("fetch_junior_members")
-    if not junior_members:
+    if not junior_members_data:
         return "No data."
+    junior_members, sorted_months = junior_members_data
         
-    transactions = fetch_sheet_data(PAYMENTS_SHEET_ID, credentials_path)
+    transactions = get_cached_data("payments_transactions", PAYMENTS_SHEET_ID, fetch_sheet_data, PAYMENTS_SHEET_ID, credentials_path)
     record_step("fetch_payments")
-    exceptions = fetch_exceptions(PAYMENTS_SHEET_ID, credentials_path)
+    exceptions = get_cached_exceptions(PAYMENTS_SHEET_ID, credentials_path)
     record_step("fetch_exceptions")
 
     # Adapt junior tuple format (name, tier, {month: (fee, total_count, adult_count, junior_count)}) 
@@ -414,7 +447,7 @@ def payments():
     payments_url = f"https://docs.google.com/spreadsheets/d/{PAYMENTS_SHEET_ID}/edit"
     credentials_path = ".secret/fuj-management-bot-credentials.json"
     
-    transactions = fetch_sheet_data(PAYMENTS_SHEET_ID, credentials_path)
+    transactions = get_cached_data("payments_transactions", PAYMENTS_SHEET_ID, fetch_sheet_data, PAYMENTS_SHEET_ID, credentials_path)
     record_step("fetch_payments")
     
     # Group transactions by person