Jan Novak
56c21bcf03
All checks were successful
Deploy to K8s / deploy (push) Successful in 8s
parseDates was using "02.01.2006" / "01/02/2006" which require zero-padded fields. The Czech attendance sheet headers contain dates like "1.6.2026", "23.3.2026", "6.4.2026" — Go silently dropped those columns while Python's strptime accepted them. Effect was a missing 2026-06 month on /api/juniors plus undercounted attendance in any month with single-digit columns; surfaced via make parity. Use the unpadded reference forms "2.1.2006" / "1/2/2006" instead — Go's time.Parse accepts both padded and unpadded inputs against them. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
508 lines
15 KiB
Go
508 lines
15 KiB
Go
package membership
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"fuj-management/go/internal/config"
|
|
"fuj-management/go/internal/domain/czech"
|
|
"fuj-management/go/internal/domain/fees"
|
|
"fuj-management/go/internal/domain/matching"
|
|
"fuj-management/go/internal/domain/reconcile"
|
|
"fuj-management/go/internal/io/attendance"
|
|
"fuj-management/go/internal/io/cache"
|
|
"fuj-management/go/internal/io/drive"
|
|
"fuj-management/go/internal/io/sheets"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Attendance CSV column indices (mirrors COL_* in scripts/attendance.py)
|
|
const (
|
|
colName = 0
|
|
colTier = 1
|
|
firstDateCol = 3
|
|
)
|
|
|
|
// AdultMergedMonths mirrors ADULT_MERGED_MONTHS in scripts/attendance.py.
|
|
// Source month → target month (source attendance accumulated into target).
|
|
var AdultMergedMonths = map[string]string{}
|
|
|
|
// JuniorMergedMonths mirrors JUNIOR_MERGED_MONTHS in scripts/attendance.py.
|
|
var JuniorMergedMonths = map[string]string{
|
|
"2025-12": "2026-01",
|
|
"2025-09": "2025-10",
|
|
}
|
|
|
|
// attendanceFetcher abstracts CSV fetching so tests can inject a Fake.
|
|
type attendanceFetcher interface {
|
|
FetchAdults(ctx context.Context) ([][]string, error)
|
|
FetchJuniors(ctx context.Context) ([][]string, error)
|
|
}
|
|
|
|
// sheetReader abstracts Sheets API reads so tests can inject a Fake.
|
|
type sheetReader interface {
|
|
GetValues(ctx context.Context, spreadsheetID, a1Range string) ([][]any, error)
|
|
}
|
|
|
|
// realSources is the live implementation of Sources backed by Google APIs.
|
|
type realSources struct {
|
|
attendance attendanceFetcher
|
|
sheets sheetReader
|
|
cache *cache.FileCache
|
|
}
|
|
|
|
// NewSources builds a Sources backed by real Google Sheets and Drive APIs.
|
|
// Call this once at startup; the returned Sources is safe for concurrent use.
|
|
func NewSources(ctx context.Context, cfg config.Config) (Sources, error) {
|
|
driveCli, err := drive.New(ctx, cfg.CredentialsPath, cfg.DriveTimeout)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("drive client: %w", err)
|
|
}
|
|
sheetsCli, err := sheets.New(ctx, cfg.CredentialsPath, cfg.DriveTimeout)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("sheets client: %w", err)
|
|
}
|
|
attendanceCli := attendance.New(nil, config.AttendanceSheetID, config.AttendanceAdultSheetGID, config.JuniorSheetGID)
|
|
fc := cache.New(driveCli, cfg.CacheDir, config.CacheSheetMap, cfg.CacheTTL, cfg.CacheAPICheckTTL)
|
|
|
|
return &realSources{
|
|
attendance: attendanceCli,
|
|
sheets: sheetsCli,
|
|
cache: fc,
|
|
}, nil
|
|
}
|
|
|
|
// LoadAdults fetches adult attendance (cached) and returns reconcile.Members for all tiers.
|
|
func (s *realSources) LoadAdults(ctx context.Context) ([]reconcile.Member, []string, error) {
|
|
rows, err := cache.Get(ctx, s.cache, "attendance_regular", s.attendance.FetchAdults)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("LoadAdults: %w", err)
|
|
}
|
|
return parseAdultRows(rows)
|
|
}
|
|
|
|
// LoadJuniors fetches junior attendance (cached) and returns reconcile.Members for juniors.
|
|
func (s *realSources) LoadJuniors(ctx context.Context) ([]reconcile.Member, []string, error) {
|
|
// Junior data needs both the adult tab (tier="J" rows) and the junior tab.
|
|
adultRows, err := cache.Get(ctx, s.cache, "attendance_regular", s.attendance.FetchAdults)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("LoadJuniors (adult tab): %w", err)
|
|
}
|
|
juniorRows, err := cache.Get(ctx, s.cache, "attendance_juniors", s.attendance.FetchJuniors)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("LoadJuniors (junior tab): %w", err)
|
|
}
|
|
return parseJuniorRows(adultRows, juniorRows)
|
|
}
|
|
|
|
// LoadTransactions fetches payment rows from the payments sheet (cached).
|
|
func (s *realSources) LoadTransactions(ctx context.Context) ([]reconcile.Transaction, error) {
|
|
rows, err := cache.Get(ctx, s.cache, "payments_transactions",
|
|
func(ctx context.Context) ([][]any, error) {
|
|
return s.sheets.GetValues(ctx, config.PaymentsSheetID, "A1:Z")
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("LoadTransactions: %w", err)
|
|
}
|
|
return parseTransactionRows(rows)
|
|
}
|
|
|
|
// LoadExceptions fetches the exceptions tab (cached).
|
|
func (s *realSources) LoadExceptions(ctx context.Context) (map[reconcile.ExceptionKey]reconcile.Exception, error) {
|
|
rows, err := cache.Get(ctx, s.cache, "exceptions_dict",
|
|
func(ctx context.Context) ([][]any, error) {
|
|
return s.sheets.GetValues(ctx, config.PaymentsSheetID, "'exceptions'!A2:D")
|
|
})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("LoadExceptions: %w", err)
|
|
}
|
|
return parseExceptionRows(rows), nil
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Attendance CSV parsing (ports scripts/attendance.py)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// parseDates returns (columnIndex, YYYY-MM) pairs for all date columns.
|
|
// Ports scripts/attendance.py parse_dates + strftime("%Y-%m").
|
|
func parseDates(header []string) []struct {
|
|
col int
|
|
month string
|
|
} {
|
|
var out []struct {
|
|
col int
|
|
month string
|
|
}
|
|
for i := firstDateCol; i < len(header); i++ {
|
|
raw := strings.TrimSpace(header[i])
|
|
if raw == "" {
|
|
continue
|
|
}
|
|
var dt time.Time
|
|
var err error
|
|
// Use the unpadded reference forms ("2.1" and "1/2"): Go's time.Parse
|
|
// accepts both single-digit and zero-padded inputs against them, so
|
|
// "1.6.2026", "01.06.2026", "23.3.2026" all parse. Czech sheet authors
|
|
// drop the leading zero on dates ≤ 9 — Python's strptime is lenient
|
|
// the same way; the previous "02.01.2006" form silently dropped those
|
|
// columns and undercounted attendance.
|
|
for _, fmt_ := range []string{"2.1.2006", "1/2/2006"} {
|
|
dt, err = time.Parse(fmt_, raw)
|
|
if err == nil {
|
|
break
|
|
}
|
|
}
|
|
if err != nil {
|
|
continue
|
|
}
|
|
out = append(out, struct {
|
|
col int
|
|
month string
|
|
}{col: i, month: dt.Format("2006-01")})
|
|
}
|
|
return out
|
|
}
|
|
|
|
// groupByMonth groups column indices by YYYY-MM, applying merged month mapping.
|
|
func groupByMonth(dates []struct {
|
|
col int
|
|
month string
|
|
}, mergedMonths map[string]string,
|
|
) map[string][]int {
|
|
out := make(map[string][]int)
|
|
for _, d := range dates {
|
|
target := d.month
|
|
if v, ok := mergedMonths[d.month]; ok {
|
|
target = v
|
|
}
|
|
out[target] = append(out[target], d.col)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// countTrue counts how many cells in the given columns have the value "TRUE" (case-insensitive).
|
|
func countTrue(row []string, cols []int) int {
|
|
n := 0
|
|
for _, c := range cols {
|
|
if c < len(row) && strings.EqualFold(strings.TrimSpace(row[c]), "true") {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
// parseAdultRows converts raw CSV rows to []reconcile.Member.
|
|
// Includes all tiers; fee is 0 for non-A tiers (reconcile filters downstream).
|
|
// Ports scripts/attendance.py get_members_with_fees.
|
|
func parseAdultRows(rows [][]string) ([]reconcile.Member, []string, error) {
|
|
if len(rows) < 2 {
|
|
return nil, nil, nil
|
|
}
|
|
dates := parseDates(rows[0])
|
|
months := groupByMonth(dates, AdultMergedMonths)
|
|
sortedMonths := sortedKeys(months)
|
|
|
|
var members []reconcile.Member
|
|
for _, row := range rows[1:] {
|
|
if len(row) == 0 {
|
|
continue
|
|
}
|
|
first := strings.TrimSpace(row[colName])
|
|
if strings.Contains(strings.ToLower(first), "# last line") {
|
|
break
|
|
}
|
|
if strings.HasPrefix(first, "#") || first == "" {
|
|
continue
|
|
}
|
|
if strings.ToLower(first) == "jméno" || strings.ToLower(first) == "name" || strings.ToLower(first) == "jmeno" {
|
|
continue
|
|
}
|
|
tier := ""
|
|
if len(row) > colTier {
|
|
tier = strings.ToUpper(strings.TrimSpace(row[colTier]))
|
|
}
|
|
|
|
feeMap := make(map[string]reconcile.FeeData, len(sortedMonths))
|
|
for _, m := range sortedMonths {
|
|
cols := months[m]
|
|
count := countTrue(row, cols)
|
|
var fee int
|
|
if tier == "A" {
|
|
fee = fees.CalculateFee(count, m)
|
|
}
|
|
feeMap[m] = reconcile.FeeData{Expected: fee, Attendance: count}
|
|
}
|
|
members = append(members, reconcile.Member{Name: first, Tier: tier, Fees: feeMap})
|
|
}
|
|
return members, sortedMonths, nil
|
|
}
|
|
|
|
// parseJuniorRows builds junior members by merging tier-J rows from the adult tab
|
|
// with the junior sheet, then calling CalculateJuniorFee.
|
|
// Ports scripts/attendance.py get_junior_members_with_fees.
|
|
func parseJuniorRows(adultRows, juniorRows [][]string) ([]reconcile.Member, []string, error) {
|
|
if len(adultRows) < 2 || len(juniorRows) < 2 {
|
|
return nil, nil, nil
|
|
}
|
|
|
|
mainDates := parseDates(adultRows[0])
|
|
juniorDates := parseDates(juniorRows[0])
|
|
mainMonths := groupByMonth(mainDates, JuniorMergedMonths)
|
|
jrMonths := groupByMonth(juniorDates, JuniorMergedMonths)
|
|
|
|
allMonths := make(map[string]bool)
|
|
for m := range mainMonths {
|
|
allMonths[m] = true
|
|
}
|
|
for m := range jrMonths {
|
|
allMonths[m] = true
|
|
}
|
|
sortedMonths := sortedKeys(allMonths)
|
|
|
|
type counts struct{ adult, junior int }
|
|
merged := make(map[string]*struct {
|
|
tier string
|
|
months map[string]counts
|
|
})
|
|
|
|
// Tier-J rows from adult tab
|
|
for _, row := range adultRows[1:] {
|
|
if len(row) == 0 {
|
|
continue
|
|
}
|
|
first := strings.TrimSpace(row[colName])
|
|
if strings.Contains(strings.ToLower(first), "# last line") {
|
|
break
|
|
}
|
|
if strings.HasPrefix(first, "#") || first == "" {
|
|
continue
|
|
}
|
|
tier := ""
|
|
if len(row) > colTier {
|
|
tier = strings.ToUpper(strings.TrimSpace(row[colTier]))
|
|
}
|
|
if tier != "J" {
|
|
continue
|
|
}
|
|
if _, ok := merged[first]; !ok {
|
|
merged[first] = &struct {
|
|
tier string
|
|
months map[string]counts
|
|
}{tier: tier, months: make(map[string]counts)}
|
|
}
|
|
for _, m := range sortedMonths {
|
|
c := merged[first].months[m]
|
|
c.adult += countTrue(row, mainMonths[m])
|
|
merged[first].months[m] = c
|
|
}
|
|
}
|
|
|
|
// All non-X rows from junior tab
|
|
for _, row := range juniorRows[1:] {
|
|
if len(row) == 0 {
|
|
continue
|
|
}
|
|
first := strings.TrimSpace(row[colName])
|
|
fl := strings.ToLower(first)
|
|
if strings.Contains(fl, "# treneri") || strings.Contains(fl, "# trenéři") {
|
|
break
|
|
}
|
|
if strings.HasPrefix(first, "#") || first == "" {
|
|
continue
|
|
}
|
|
tier := ""
|
|
if len(row) > colTier {
|
|
tier = strings.ToUpper(strings.TrimSpace(row[colTier]))
|
|
}
|
|
if tier == "X" {
|
|
continue
|
|
}
|
|
if _, ok := merged[first]; !ok {
|
|
merged[first] = &struct {
|
|
tier string
|
|
months map[string]counts
|
|
}{tier: tier, months: make(map[string]counts)}
|
|
}
|
|
for _, m := range sortedMonths {
|
|
c := merged[first].months[m]
|
|
c.junior += countTrue(row, jrMonths[m])
|
|
merged[first].months[m] = c
|
|
}
|
|
}
|
|
|
|
var members []reconcile.Member
|
|
for name, data := range merged {
|
|
feeMap := make(map[string]reconcile.FeeData, len(sortedMonths))
|
|
for _, m := range sortedMonths {
|
|
c := data.months[m]
|
|
total := c.adult + c.junior
|
|
exp := fees.CalculateJuniorFee(total, m)
|
|
fee := 0
|
|
if !exp.Unknown {
|
|
fee = exp.Value
|
|
}
|
|
feeMap[m] = reconcile.FeeData{
|
|
Expected: fee,
|
|
IsUnknown: exp.Unknown,
|
|
Attendance: total,
|
|
JuniorAttendance: c.junior,
|
|
AdultAttendance: c.adult,
|
|
}
|
|
}
|
|
members = append(members, reconcile.Member{Name: name, Tier: data.tier, Fees: feeMap})
|
|
}
|
|
return members, sortedMonths, nil
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Payments sheet row parsing (ports scripts/match_payments.py fetch_sheet_data)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func parseTransactionRows(rows [][]any) ([]reconcile.Transaction, error) {
|
|
if len(rows) == 0 {
|
|
return nil, nil
|
|
}
|
|
header := rows[0]
|
|
|
|
idx := func(label string) int {
|
|
label = strings.ToLower(strings.TrimSpace(label))
|
|
for i, h := range header {
|
|
if strings.ToLower(strings.TrimSpace(fmt.Sprint(h))) == label {
|
|
return i
|
|
}
|
|
}
|
|
return -1
|
|
}
|
|
idxDate := idx("date")
|
|
idxAmount := idx("amount")
|
|
idxManualFix := idx("manual fix")
|
|
idxPerson := idx("person")
|
|
idxPurpose := idx("purpose")
|
|
idxInferred := idx("inferred amount")
|
|
idxSender := idx("sender")
|
|
idxVS := idx("vs")
|
|
idxMessage := idx("message")
|
|
idxBankID := idx("bank id")
|
|
idxSyncID := idx("sync id")
|
|
|
|
for _, label := range []string{"date", "amount", "person", "purpose"} {
|
|
if idx(label) == -1 {
|
|
return nil, fmt.Errorf("payments sheet missing required column %q", label)
|
|
}
|
|
}
|
|
|
|
getVal := func(row []any, i int) string {
|
|
if i < 0 || i >= len(row) {
|
|
return ""
|
|
}
|
|
return fmt.Sprint(row[i])
|
|
}
|
|
|
|
// getRaw returns row[i] without stringifying — needed for FormatDate to
|
|
// dispatch on the underlying numeric type (Sheets returns serial-day
|
|
// numbers as float64). Stringifying first defeats that dispatch.
|
|
getRaw := func(row []any, i int) any {
|
|
if i < 0 || i >= len(row) {
|
|
return nil
|
|
}
|
|
return row[i]
|
|
}
|
|
|
|
var txns []reconcile.Transaction
|
|
for _, row := range rows[1:] {
|
|
dateStr := matching.FormatDate(getRaw(row, idxDate))
|
|
amountRaw := row[idxAmount]
|
|
if idxAmount < 0 || idxAmount >= len(row) {
|
|
amountRaw = ""
|
|
}
|
|
amount := parseFloat(amountRaw)
|
|
|
|
var inferredAmount *float64
|
|
if iv := getVal(row, idxInferred); iv != "" && iv != "<nil>" {
|
|
if f := parseFloat(iv); f != 0 {
|
|
inferredAmount = &f
|
|
}
|
|
}
|
|
|
|
txns = append(txns, reconcile.Transaction{
|
|
Date: dateStr,
|
|
Amount: amount,
|
|
ManualFix: getVal(row, idxManualFix),
|
|
Person: getVal(row, idxPerson),
|
|
Purpose: getVal(row, idxPurpose),
|
|
InferredAmount: inferredAmount,
|
|
Sender: getVal(row, idxSender),
|
|
VS: getVal(row, idxVS),
|
|
Message: getVal(row, idxMessage),
|
|
BankID: getVal(row, idxBankID),
|
|
SyncID: getVal(row, idxSyncID),
|
|
})
|
|
}
|
|
return txns, nil
|
|
}
|
|
|
|
func parseFloat(v any) float64 {
|
|
switch x := v.(type) {
|
|
case float64:
|
|
return x
|
|
case float32:
|
|
return float64(x)
|
|
case int:
|
|
return float64(x)
|
|
case int64:
|
|
return float64(x)
|
|
case string:
|
|
f, _ := strconv.ParseFloat(strings.TrimSpace(x), 64)
|
|
return f
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Exceptions tab parsing (ports scripts/match_payments.py fetch_exceptions)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func parseExceptionRows(rows [][]any) map[reconcile.ExceptionKey]reconcile.Exception {
|
|
out := make(map[reconcile.ExceptionKey]reconcile.Exception)
|
|
for _, row := range rows {
|
|
if len(row) < 3 {
|
|
continue
|
|
}
|
|
name := strings.TrimSpace(fmt.Sprint(row[0]))
|
|
if strings.ToLower(name) == "name" || strings.HasPrefix(strings.ToLower(name), "name") {
|
|
continue
|
|
}
|
|
period := strings.TrimSpace(fmt.Sprint(row[1]))
|
|
amountStr := fmt.Sprint(row[2])
|
|
amount, err := strconv.Atoi(strings.TrimSpace(amountStr))
|
|
if err != nil {
|
|
continue
|
|
}
|
|
note := ""
|
|
if len(row) > 3 {
|
|
note = strings.TrimSpace(fmt.Sprint(row[3]))
|
|
}
|
|
key := reconcile.ExceptionKey{
|
|
Name: czech.Normalize(name),
|
|
Period: czech.Normalize(period),
|
|
}
|
|
out[key] = reconcile.Exception{Amount: amount, Note: note}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Helpers
|
|
// ---------------------------------------------------------------------------
|
|
|
|
func sortedKeys[V any](m map[string]V) []string {
|
|
keys := make([]string, 0, len(m))
|
|
for k := range m {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
return keys
|
|
}
|