feat(go): IO layer behind interfaces (M4)
All checks were successful
Deploy to K8s / deploy (push) Successful in 11s
All checks were successful
Deploy to K8s / deploy (push) Successful in 11s
- io/attendance: CSV-over-public-URL client + Fake for adult/junior tabs - io/drive: Drive v3 modifiedTime client + Fake - io/sheets: Sheets v4 client (GetValues/AppendValues/BatchUpdateValues/ WriteHeader/SortByDateColumn) + Fake with call-capture - io/cache: Drive-modifiedTime-gated FileCache; two TTL knobs; atomic writes; generic Get[T]; Python-compatible JSON format; Flush() - io/fio: Client interface backed by Fio REST API (apiClient) and HTML scraper (transparentClient); Fake; testdata fixtures - membership/sources: NewSources wires attendance CSV + Sheets + cache into LoadAdults/LoadJuniors/LoadTransactions/LoadExceptions; Czech month parsing + merged-month maps - banksync: SyncToSheets (SHA-256 dedup, optional sort) and InferPayments ([?] review prefix, dry-run) — tested with fakes - cmd/fuj: sync and infer subcommands wired; fees and reconcile use real NewSources; go.mod gains google.golang.org/api + x/net - gofumpt extra-rules applied across all packages; lint clean Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
477
go/internal/services/membership/sources.go
Normal file
477
go/internal/services/membership/sources.go
Normal file
@@ -0,0 +1,477 @@
|
||||
package membership
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"fuj-management/go/internal/config"
|
||||
"fuj-management/go/internal/domain/czech"
|
||||
"fuj-management/go/internal/domain/fees"
|
||||
"fuj-management/go/internal/domain/matching"
|
||||
"fuj-management/go/internal/domain/reconcile"
|
||||
"fuj-management/go/internal/io/attendance"
|
||||
"fuj-management/go/internal/io/cache"
|
||||
"fuj-management/go/internal/io/drive"
|
||||
"fuj-management/go/internal/io/sheets"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Attendance CSV column indices (mirrors COL_* in scripts/attendance.py)
|
||||
const (
|
||||
colName = 0
|
||||
colTier = 1
|
||||
firstDateCol = 3
|
||||
)
|
||||
|
||||
// adultMergedMonths mirrors ADULT_MERGED_MONTHS in scripts/attendance.py.
|
||||
// Source month → target month (source attendance accumulated into target).
|
||||
var adultMergedMonths = map[string]string{}
|
||||
|
||||
// juniorMergedMonths mirrors JUNIOR_MERGED_MONTHS in scripts/attendance.py.
|
||||
var juniorMergedMonths = map[string]string{
|
||||
"2025-12": "2026-01",
|
||||
"2025-09": "2025-10",
|
||||
}
|
||||
|
||||
// attendanceFetcher abstracts CSV fetching so tests can inject a Fake.
|
||||
type attendanceFetcher interface {
|
||||
FetchAdults(ctx context.Context) ([][]string, error)
|
||||
FetchJuniors(ctx context.Context) ([][]string, error)
|
||||
}
|
||||
|
||||
// sheetReader abstracts Sheets API reads so tests can inject a Fake.
|
||||
type sheetReader interface {
|
||||
GetValues(ctx context.Context, spreadsheetID, a1Range string) ([][]any, error)
|
||||
}
|
||||
|
||||
// realSources is the live implementation of Sources backed by Google APIs.
|
||||
type realSources struct {
|
||||
attendance attendanceFetcher
|
||||
sheets sheetReader
|
||||
cache *cache.FileCache
|
||||
}
|
||||
|
||||
// NewSources builds a Sources backed by real Google Sheets and Drive APIs.
|
||||
// Call this once at startup; the returned Sources is safe for concurrent use.
|
||||
func NewSources(ctx context.Context, cfg config.Config) (Sources, error) {
|
||||
driveCli, err := drive.New(ctx, cfg.CredentialsPath, cfg.DriveTimeout)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("drive client: %w", err)
|
||||
}
|
||||
sheetsCli, err := sheets.New(ctx, cfg.CredentialsPath, cfg.DriveTimeout)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sheets client: %w", err)
|
||||
}
|
||||
attendanceCli := attendance.New(nil, config.AttendanceSheetID, config.AttendanceAdultSheetGID, config.JuniorSheetGID)
|
||||
fc := cache.New(driveCli, cfg.CacheDir, config.CacheSheetMap, cfg.CacheTTL, cfg.CacheAPICheckTTL)
|
||||
|
||||
return &realSources{
|
||||
attendance: attendanceCli,
|
||||
sheets: sheetsCli,
|
||||
cache: fc,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// LoadAdults fetches adult attendance (cached) and returns reconcile.Members for all tiers.
|
||||
func (s *realSources) LoadAdults(ctx context.Context) ([]reconcile.Member, []string, error) {
|
||||
rows, err := cache.Get(ctx, s.cache, "attendance_regular", s.attendance.FetchAdults)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("LoadAdults: %w", err)
|
||||
}
|
||||
return parseAdultRows(rows)
|
||||
}
|
||||
|
||||
// LoadJuniors fetches junior attendance (cached) and returns reconcile.Members for juniors.
|
||||
func (s *realSources) LoadJuniors(ctx context.Context) ([]reconcile.Member, []string, error) {
|
||||
// Junior data needs both the adult tab (tier="J" rows) and the junior tab.
|
||||
adultRows, err := cache.Get(ctx, s.cache, "attendance_regular", s.attendance.FetchAdults)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("LoadJuniors (adult tab): %w", err)
|
||||
}
|
||||
juniorRows, err := cache.Get(ctx, s.cache, "attendance_juniors", s.attendance.FetchJuniors)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("LoadJuniors (junior tab): %w", err)
|
||||
}
|
||||
return parseJuniorRows(adultRows, juniorRows)
|
||||
}
|
||||
|
||||
// LoadTransactions fetches payment rows from the payments sheet (cached).
|
||||
func (s *realSources) LoadTransactions(ctx context.Context) ([]reconcile.Transaction, error) {
|
||||
rows, err := cache.Get(ctx, s.cache, "payments_transactions",
|
||||
func(ctx context.Context) ([][]any, error) {
|
||||
return s.sheets.GetValues(ctx, config.PaymentsSheetID, "A1:Z")
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("LoadTransactions: %w", err)
|
||||
}
|
||||
return parseTransactionRows(rows)
|
||||
}
|
||||
|
||||
// LoadExceptions fetches the exceptions tab (cached).
|
||||
func (s *realSources) LoadExceptions(ctx context.Context) (map[reconcile.ExceptionKey]reconcile.Exception, error) {
|
||||
rows, err := cache.Get(ctx, s.cache, "exceptions_dict",
|
||||
func(ctx context.Context) ([][]any, error) {
|
||||
return s.sheets.GetValues(ctx, config.PaymentsSheetID, "'exceptions'!A2:D")
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("LoadExceptions: %w", err)
|
||||
}
|
||||
return parseExceptionRows(rows), nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Attendance CSV parsing (ports scripts/attendance.py)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// parseDates returns (columnIndex, YYYY-MM) pairs for all date columns.
|
||||
// Ports scripts/attendance.py parse_dates + strftime("%Y-%m").
|
||||
func parseDates(header []string) []struct {
|
||||
col int
|
||||
month string
|
||||
} {
|
||||
var out []struct {
|
||||
col int
|
||||
month string
|
||||
}
|
||||
for i := firstDateCol; i < len(header); i++ {
|
||||
raw := strings.TrimSpace(header[i])
|
||||
if raw == "" {
|
||||
continue
|
||||
}
|
||||
var dt time.Time
|
||||
var err error
|
||||
for _, fmt_ := range []string{"02.01.2006", "01/02/2006"} {
|
||||
dt, err = time.Parse(fmt_, raw)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
out = append(out, struct {
|
||||
col int
|
||||
month string
|
||||
}{col: i, month: dt.Format("2006-01")})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// groupByMonth groups column indices by YYYY-MM, applying merged month mapping.
|
||||
func groupByMonth(dates []struct {
|
||||
col int
|
||||
month string
|
||||
}, mergedMonths map[string]string,
|
||||
) map[string][]int {
|
||||
out := make(map[string][]int)
|
||||
for _, d := range dates {
|
||||
target := d.month
|
||||
if v, ok := mergedMonths[d.month]; ok {
|
||||
target = v
|
||||
}
|
||||
out[target] = append(out[target], d.col)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// countTrue counts how many cells in the given columns have the value "TRUE" (case-insensitive).
|
||||
func countTrue(row []string, cols []int) int {
|
||||
n := 0
|
||||
for _, c := range cols {
|
||||
if c < len(row) && strings.EqualFold(strings.TrimSpace(row[c]), "true") {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// parseAdultRows converts raw CSV rows to []reconcile.Member.
|
||||
// Includes all tiers; fee is 0 for non-A tiers (reconcile filters downstream).
|
||||
// Ports scripts/attendance.py get_members_with_fees.
|
||||
func parseAdultRows(rows [][]string) ([]reconcile.Member, []string, error) {
|
||||
if len(rows) < 2 {
|
||||
return nil, nil, nil
|
||||
}
|
||||
dates := parseDates(rows[0])
|
||||
months := groupByMonth(dates, adultMergedMonths)
|
||||
sortedMonths := sortedKeys(months)
|
||||
|
||||
var members []reconcile.Member
|
||||
for _, row := range rows[1:] {
|
||||
if len(row) == 0 {
|
||||
continue
|
||||
}
|
||||
first := strings.TrimSpace(row[colName])
|
||||
if strings.Contains(strings.ToLower(first), "# last line") {
|
||||
break
|
||||
}
|
||||
if strings.HasPrefix(first, "#") || first == "" {
|
||||
continue
|
||||
}
|
||||
if strings.ToLower(first) == "jméno" || strings.ToLower(first) == "name" || strings.ToLower(first) == "jmeno" {
|
||||
continue
|
||||
}
|
||||
tier := ""
|
||||
if len(row) > colTier {
|
||||
tier = strings.ToUpper(strings.TrimSpace(row[colTier]))
|
||||
}
|
||||
|
||||
feeMap := make(map[string]reconcile.FeeData, len(sortedMonths))
|
||||
for _, m := range sortedMonths {
|
||||
cols := months[m]
|
||||
count := countTrue(row, cols)
|
||||
var fee int
|
||||
if tier == "A" {
|
||||
fee = fees.CalculateFee(count, m)
|
||||
}
|
||||
feeMap[m] = reconcile.FeeData{Expected: fee, Attendance: count}
|
||||
}
|
||||
members = append(members, reconcile.Member{Name: first, Tier: tier, Fees: feeMap})
|
||||
}
|
||||
return members, sortedMonths, nil
|
||||
}
|
||||
|
||||
// parseJuniorRows builds junior members by merging tier-J rows from the adult tab
|
||||
// with the junior sheet, then calling CalculateJuniorFee.
|
||||
// Ports scripts/attendance.py get_junior_members_with_fees.
|
||||
func parseJuniorRows(adultRows, juniorRows [][]string) ([]reconcile.Member, []string, error) {
|
||||
if len(adultRows) < 2 || len(juniorRows) < 2 {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
mainDates := parseDates(adultRows[0])
|
||||
juniorDates := parseDates(juniorRows[0])
|
||||
mainMonths := groupByMonth(mainDates, juniorMergedMonths)
|
||||
jrMonths := groupByMonth(juniorDates, juniorMergedMonths)
|
||||
|
||||
allMonths := make(map[string]bool)
|
||||
for m := range mainMonths {
|
||||
allMonths[m] = true
|
||||
}
|
||||
for m := range jrMonths {
|
||||
allMonths[m] = true
|
||||
}
|
||||
sortedMonths := sortedKeys(allMonths)
|
||||
|
||||
type counts struct{ adult, junior int }
|
||||
merged := make(map[string]*struct {
|
||||
tier string
|
||||
months map[string]counts
|
||||
})
|
||||
|
||||
// Tier-J rows from adult tab
|
||||
for _, row := range adultRows[1:] {
|
||||
if len(row) == 0 {
|
||||
continue
|
||||
}
|
||||
first := strings.TrimSpace(row[colName])
|
||||
if strings.Contains(strings.ToLower(first), "# last line") {
|
||||
break
|
||||
}
|
||||
if strings.HasPrefix(first, "#") || first == "" {
|
||||
continue
|
||||
}
|
||||
tier := ""
|
||||
if len(row) > colTier {
|
||||
tier = strings.ToUpper(strings.TrimSpace(row[colTier]))
|
||||
}
|
||||
if tier != "J" {
|
||||
continue
|
||||
}
|
||||
if _, ok := merged[first]; !ok {
|
||||
merged[first] = &struct {
|
||||
tier string
|
||||
months map[string]counts
|
||||
}{tier: tier, months: make(map[string]counts)}
|
||||
}
|
||||
for _, m := range sortedMonths {
|
||||
c := merged[first].months[m]
|
||||
c.adult += countTrue(row, mainMonths[m])
|
||||
merged[first].months[m] = c
|
||||
}
|
||||
}
|
||||
|
||||
// All non-X rows from junior tab
|
||||
for _, row := range juniorRows[1:] {
|
||||
if len(row) == 0 {
|
||||
continue
|
||||
}
|
||||
first := strings.TrimSpace(row[colName])
|
||||
fl := strings.ToLower(first)
|
||||
if strings.Contains(fl, "# treneri") || strings.Contains(fl, "# trenéři") {
|
||||
break
|
||||
}
|
||||
if strings.HasPrefix(first, "#") || first == "" {
|
||||
continue
|
||||
}
|
||||
tier := ""
|
||||
if len(row) > colTier {
|
||||
tier = strings.ToUpper(strings.TrimSpace(row[colTier]))
|
||||
}
|
||||
if tier == "X" {
|
||||
continue
|
||||
}
|
||||
if _, ok := merged[first]; !ok {
|
||||
merged[first] = &struct {
|
||||
tier string
|
||||
months map[string]counts
|
||||
}{tier: tier, months: make(map[string]counts)}
|
||||
}
|
||||
for _, m := range sortedMonths {
|
||||
c := merged[first].months[m]
|
||||
c.junior += countTrue(row, jrMonths[m])
|
||||
merged[first].months[m] = c
|
||||
}
|
||||
}
|
||||
|
||||
var members []reconcile.Member
|
||||
for name, data := range merged {
|
||||
feeMap := make(map[string]reconcile.FeeData, len(sortedMonths))
|
||||
for _, m := range sortedMonths {
|
||||
c := data.months[m]
|
||||
total := c.adult + c.junior
|
||||
exp := fees.CalculateJuniorFee(total, m)
|
||||
fee := 0
|
||||
if !exp.Unknown {
|
||||
fee = exp.Value
|
||||
}
|
||||
feeMap[m] = reconcile.FeeData{Expected: fee, Attendance: total}
|
||||
}
|
||||
members = append(members, reconcile.Member{Name: name, Tier: data.tier, Fees: feeMap})
|
||||
}
|
||||
return members, sortedMonths, nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Payments sheet row parsing (ports scripts/match_payments.py fetch_sheet_data)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func parseTransactionRows(rows [][]any) ([]reconcile.Transaction, error) {
|
||||
if len(rows) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
header := rows[0]
|
||||
|
||||
idx := func(label string) int {
|
||||
label = strings.ToLower(strings.TrimSpace(label))
|
||||
for i, h := range header {
|
||||
if strings.ToLower(strings.TrimSpace(fmt.Sprint(h))) == label {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
idxDate := idx("date")
|
||||
idxAmount := idx("amount")
|
||||
idxPerson := idx("person")
|
||||
idxPurpose := idx("purpose")
|
||||
idxInferred := idx("inferred amount")
|
||||
idxSender := idx("sender")
|
||||
idxMessage := idx("message")
|
||||
|
||||
for _, label := range []string{"date", "amount", "person", "purpose"} {
|
||||
if idx(label) == -1 {
|
||||
return nil, fmt.Errorf("payments sheet missing required column %q", label)
|
||||
}
|
||||
}
|
||||
|
||||
getVal := func(row []any, i int) string {
|
||||
if i < 0 || i >= len(row) {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprint(row[i])
|
||||
}
|
||||
|
||||
var txns []reconcile.Transaction
|
||||
for _, row := range rows[1:] {
|
||||
dateStr := matching.FormatDate(getVal(row, idxDate))
|
||||
amountRaw := row[idxAmount]
|
||||
if idxAmount < 0 || idxAmount >= len(row) {
|
||||
amountRaw = ""
|
||||
}
|
||||
amount := parseFloat(amountRaw)
|
||||
|
||||
var inferredAmount *float64
|
||||
if iv := getVal(row, idxInferred); iv != "" && iv != "<nil>" {
|
||||
if f := parseFloat(iv); f != 0 {
|
||||
inferredAmount = &f
|
||||
}
|
||||
}
|
||||
|
||||
txns = append(txns, reconcile.Transaction{
|
||||
Date: dateStr,
|
||||
Amount: amount,
|
||||
Person: getVal(row, idxPerson),
|
||||
Purpose: getVal(row, idxPurpose),
|
||||
InferredAmount: inferredAmount,
|
||||
Sender: getVal(row, idxSender),
|
||||
Message: getVal(row, idxMessage),
|
||||
})
|
||||
}
|
||||
return txns, nil
|
||||
}
|
||||
|
||||
func parseFloat(v any) float64 {
|
||||
switch x := v.(type) {
|
||||
case float64:
|
||||
return x
|
||||
case float32:
|
||||
return float64(x)
|
||||
case int:
|
||||
return float64(x)
|
||||
case int64:
|
||||
return float64(x)
|
||||
case string:
|
||||
f, _ := strconv.ParseFloat(strings.TrimSpace(x), 64)
|
||||
return f
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Exceptions tab parsing (ports scripts/match_payments.py fetch_exceptions)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func parseExceptionRows(rows [][]any) map[reconcile.ExceptionKey]reconcile.Exception {
|
||||
out := make(map[reconcile.ExceptionKey]reconcile.Exception)
|
||||
for _, row := range rows {
|
||||
if len(row) < 3 {
|
||||
continue
|
||||
}
|
||||
name := strings.TrimSpace(fmt.Sprint(row[0]))
|
||||
if strings.ToLower(name) == "name" || strings.HasPrefix(strings.ToLower(name), "name") {
|
||||
continue
|
||||
}
|
||||
period := strings.TrimSpace(fmt.Sprint(row[1]))
|
||||
amountStr := fmt.Sprint(row[2])
|
||||
amount, err := strconv.Atoi(strings.TrimSpace(amountStr))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
note := ""
|
||||
if len(row) > 3 {
|
||||
note = strings.TrimSpace(fmt.Sprint(row[3]))
|
||||
}
|
||||
key := reconcile.ExceptionKey{
|
||||
Name: czech.Normalize(name),
|
||||
Period: czech.Normalize(period),
|
||||
}
|
||||
out[key] = reconcile.Exception{Amount: amount, Note: note}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func sortedKeys[V any](m map[string]V) []string {
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
Reference in New Issue
Block a user