feat: implement automated payment inference and sync to Google Sheets
This commit is contained in:
Jan Novak
210
scripts/sync_fio_to_sheets.py
Normal file
210
scripts/sync_fio_to_sheets.py
Normal file
@@ -0,0 +1,210 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sync Fio bank transactions to a Google Sheet intermediary ledger."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import pickle
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from google.auth.transport.requests import Request
|
||||
from google_auth_oauthlib.flow import InstalledAppFlow
|
||||
from google.oauth2 import service_account
|
||||
from googleapiclient.discovery import build
|
||||
|
||||
from fio_utils import fetch_transactions
|
||||
|
||||
# Configuration
|
||||
DEFAULT_SPREADSHEET_ID = "1Om0YPoDVCH5cV8BrNz5LG5eR5MMU05ypQC7UMN1xn_Y"
|
||||
SCOPES = ["https://www.googleapis.com/auth/spreadsheets"]
|
||||
TOKEN_FILE = "token.pickle"
|
||||
COLUMN_LABELS = ["Date", "Amount", "manual fix", "Person", "Purpose", "Inferred Amount", "Sender", "VS", "Message", "Bank ID", "Sync ID"]
|
||||
def get_sheets_service(credentials_path: str):
|
||||
"""Authenticate and return the Google Sheets API service."""
|
||||
if not os.path.exists(credentials_path):
|
||||
raise FileNotFoundError(f"Credentials file not found: {credentials_path}")
|
||||
|
||||
# Check if it's a service account
|
||||
import json
|
||||
with open(credentials_path, "r") as f:
|
||||
creds_data = json.load(f)
|
||||
|
||||
if creds_data.get("type") == "service_account":
|
||||
creds = service_account.Credentials.from_service_account_file(
|
||||
credentials_path, scopes=SCOPES
|
||||
)
|
||||
else:
|
||||
# Fallback to OAuth2 flow
|
||||
creds = None
|
||||
if os.path.exists(TOKEN_FILE):
|
||||
with open(TOKEN_FILE, "rb") as token:
|
||||
creds = pickle.load(token)
|
||||
|
||||
if not creds or not creds.valid:
|
||||
if creds and creds.expired and creds.refresh_token:
|
||||
creds.refresh(Request())
|
||||
else:
|
||||
flow = InstalledAppFlow.from_client_secrets_file(credentials_path, SCOPES)
|
||||
creds = flow.run_local_server(port=0)
|
||||
|
||||
with open(TOKEN_FILE, "wb") as token:
|
||||
pickle.dump(creds, token)
|
||||
|
||||
return build("sheets", "v4", credentials=creds)
|
||||
|
||||
|
||||
def generate_sync_id(tx: dict) -> str:
|
||||
"""Generate a unique SHA-256 hash for a transaction.
|
||||
|
||||
Hash components: date|amount|currency|sender|vs|message|bank_id
|
||||
"""
|
||||
components = [
|
||||
str(tx.get("date", "")),
|
||||
str(tx.get("amount", "")),
|
||||
str(tx.get("currency", "CZK")),
|
||||
str(tx.get("sender", "")),
|
||||
str(tx.get("vs", "")),
|
||||
str(tx.get("message", "")),
|
||||
str(tx.get("bank_id", "")),
|
||||
]
|
||||
raw_str = "|".join(components).lower()
|
||||
return hashlib.sha256(raw_str.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def sort_sheet_by_date(service, spreadsheet_id):
|
||||
"""Sort the sheet by the Date column (Column B)."""
|
||||
# Get the sheet ID (gid) of the first sheet
|
||||
spreadsheet = service.spreadsheets().get(spreadsheetId=spreadsheet_id).execute()
|
||||
sheet_id = spreadsheet['sheets'][0]['properties']['sheetId']
|
||||
|
||||
requests = [{
|
||||
"sortRange": {
|
||||
"range": {
|
||||
"sheetId": sheet_id,
|
||||
"startRowIndex": 1, # Skip header
|
||||
"endRowIndex": 10000
|
||||
},
|
||||
"sortSpecs": [{
|
||||
"dimensionIndex": 0, # Column A (Date)
|
||||
"sortOrder": "ASCENDING"
|
||||
}]
|
||||
}
|
||||
}]
|
||||
|
||||
service.spreadsheets().batchUpdate(
|
||||
spreadsheetId=spreadsheet_id,
|
||||
body={"requests": requests}
|
||||
).execute()
|
||||
print("Sheet sorted by date.")
|
||||
|
||||
|
||||
def sync_to_sheets(spreadsheet_id: str, credentials_path: str, days: int = None, date_from_str: str = None, date_to_str: str = None, sort_by_date: bool = False):
|
||||
print(f"Connecting to Google Sheets using {credentials_path}...")
|
||||
service = get_sheets_service(credentials_path)
|
||||
sheet = service.spreadsheets()
|
||||
|
||||
# 1. Fetch existing IDs from Column G (last column in A-G range)
|
||||
print(f"Reading existing sync IDs from sheet...")
|
||||
try:
|
||||
result = sheet.values().get(
|
||||
spreadsheetId=spreadsheet_id,
|
||||
range="A1:K" # Include header and all columns to check Sync ID
|
||||
).execute()
|
||||
values = result.get("values", [])
|
||||
|
||||
# Check and insert labels if missing
|
||||
if not values or values[0] != COLUMN_LABELS:
|
||||
print("Inserting column labels...")
|
||||
sheet.values().update(
|
||||
spreadsheetId=spreadsheet_id,
|
||||
range="A1",
|
||||
valueInputOption="USER_ENTERED",
|
||||
body={"values": [COLUMN_LABELS]}
|
||||
).execute()
|
||||
existing_ids = set()
|
||||
else:
|
||||
# Sync ID is now the last column (index 10)
|
||||
existing_ids = {row[10] for row in values[1:] if len(row) > 10}
|
||||
except Exception as e:
|
||||
print(f"Error reading sheet (maybe empty?): {e}")
|
||||
existing_ids = set()
|
||||
|
||||
# 2. Fetch Fio transactions
|
||||
if date_from_str and date_to_str:
|
||||
df_str = date_from_str
|
||||
dt_str = date_to_str
|
||||
else:
|
||||
now = datetime.now()
|
||||
date_to = now
|
||||
date_from = now - timedelta(days=days or 30)
|
||||
df_str = date_from.strftime("%Y-%m-%d")
|
||||
dt_str = date_to.strftime("%Y-%m-%d")
|
||||
|
||||
print(f"Fetching Fio transactions from {df_str} to {dt_str}...")
|
||||
transactions = fetch_transactions(df_str, dt_str)
|
||||
print(f"Found {len(transactions)} transactions.")
|
||||
|
||||
# 3. Filter for new transactions
|
||||
new_rows = []
|
||||
for tx in transactions:
|
||||
sync_id = generate_sync_id(tx)
|
||||
if sync_id not in existing_ids:
|
||||
# Schema: Date | Amount | Manual | Person | Purpose | Inferred Amount | Sender | VS | Message | Bank ID | Sync ID
|
||||
new_rows.append([
|
||||
tx.get("date", ""),
|
||||
tx.get("amount", ""),
|
||||
"", # Manual
|
||||
"", # Person
|
||||
"", # Purpose
|
||||
"", # Inferred Amount
|
||||
tx.get("sender", ""),
|
||||
tx.get("vs", ""),
|
||||
tx.get("message", ""),
|
||||
tx.get("bank_id", ""),
|
||||
sync_id,
|
||||
])
|
||||
|
||||
if not new_rows:
|
||||
print("No new transactions to sync.")
|
||||
return
|
||||
|
||||
# 4. Append to sheet
|
||||
print(f"Appending {len(new_rows)} new transactions to the sheet...")
|
||||
body = {"values": new_rows}
|
||||
sheet.values().append(
|
||||
spreadsheetId=spreadsheet_id,
|
||||
range="A2", # Appends to the end of the sheet
|
||||
valueInputOption="USER_ENTERED",
|
||||
body=body
|
||||
).execute()
|
||||
print("Sync completed successfully.")
|
||||
|
||||
if sort_by_date:
|
||||
sort_sheet_by_date(service, spreadsheet_id)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Sync Fio transactions to Google Sheets.")
|
||||
parser.add_argument("--days", type=int, default=30, help="Days to look back (default: 30)")
|
||||
parser.add_argument("--sheet-id", default=DEFAULT_SPREADSHEET_ID, help="Google Sheet ID")
|
||||
parser.add_argument("--credentials", default="credentials.json", help="Path to Google API credentials JSON")
|
||||
parser.add_argument("--from", dest="date_from", help="Start date YYYY-MM-DD")
|
||||
parser.add_argument("--to", dest="date_to", help="End date YYYY-MM-DD")
|
||||
parser.add_argument("--sort-by-date", action="store_true", help="Sort the sheet by date after sync")
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
sync_to_sheets(
|
||||
spreadsheet_id=args.sheet_id,
|
||||
credentials_path=args.credentials,
|
||||
days=args.days,
|
||||
date_from_str=args.date_from,
|
||||
date_to_str=args.date_to,
|
||||
sort_by_date=args.sort_by_date
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"Sync failed: {e}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user