ryan/FargoImport
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first commit

Browse Source
This commit is contained in:
Ryan Kegel
2026-03-22 23:30:38 -04:00
commit 78a6d73c11
19 changed files with 2732 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

658
main.py Normal file
View File

@@ -0,0 +1,658 @@
import argparse
import json
from datetime import date, datetime, timedelta, timezone
from pathlib import Path
from typing import Any
import ynab
from wf_credit_extractor import WellsFargoPayloadExtractor as CreditExtractor
from wf_debit_extractor import WellsFargoPayloadExtractor as DebitExtractor
from ynab_client import YNABClient, YNABTransaction
# Maps the extractor name in ynab.yml to the correct extractor class
EXTRACTOR_MAP = {
"wf_credit_extractor": CreditExtractor,
"wf_debit_extractor": DebitExtractor,
}
# Maps CLI flag name to the extractor name in ynab.yml
FLAG_TO_EXTRACTOR = {
"credit": "wf_credit_extractor",
"debit": "wf_debit_extractor",
}
# Maps extractor name to the balance type to display
EXTRACTOR_BALANCE_TYPE = {
"wf_credit_extractor": "OUTSTANDING",
"wf_debit_extractor": "AVAILABLE",
}
def _default_output_path(input_path: Path) -> Path:
"""Derive a JSON output path from the input file path."""
return input_path.with_suffix(".json")
def _extract_raw_transactions(extracted_objects: list[Any]) -> list[dict]:
"""Pull the raw transaction list out of extracted WF payload objects.
Supports multiple HAR structures:
- /transactions/transactionData/transactions (posted.dat / checking.dat style)
- /accountModel/applicationData/transactionData/transactions (credit fetch style)
Each transaction is tagged with a "status" field: "POSTED" or "PENDING".
TEMP_AUTH transactionType in requestedCriteria marks pending transactions.
"""
transactions: list[dict] = []
for obj in extracted_objects:
_collect_from_block(obj, transactions)
return transactions
def _collect_from_block(obj: dict, out: list[dict]) -> None:
"""Recurse into a single extracted object to find all transactionData blocks."""
# Path 1: /transactions/transactionData (old credit / debit style, and TEMP_AUTH)
txn_wrapper = obj.get("transactions", {})
if isinstance(txn_wrapper, dict):
td = txn_wrapper.get("transactionData")
if isinstance(td, dict):
_append_transactions(td, out)
# Path 2: /accountModel/applicationData/transactionData (new credit fetch style)
acct_td = (
obj.get("accountModel", {})
.get("applicationData", {})
.get("transactionData")
)
if isinstance(acct_td, dict):
_append_transactions(acct_td, out)
def _append_transactions(td: dict, out: list[dict]) -> None:
"""Extract transactions from a transactionData block and tag POSTED/PENDING."""
txn_type = (
td.get("requestedCriteria", {}).get("transactionType", "")
or td.get("type", "")
)
status = "PENDING" if txn_type == "TEMP_AUTH" else "POSTED"
txn_list = td.get("transactions", [])
if isinstance(txn_list, list):
for txn in txn_list:
txn_copy = dict(txn)
txn_copy["status"] = status
if "transactionDescription" in txn_copy:
txn_copy["transactionDescription"] = " ".join(txn_copy["transactionDescription"].split())
out.append(txn_copy)
def _wf_date_to_iso(timestamp_ms: int) -> str:
"""Convert WF millisecond timestamp to YYYY-MM-DD string."""
return datetime.fromtimestamp(timestamp_ms / 1000.0, tz=timezone.utc).strftime(
"%Y-%m-%d"
)
def _wf_amount_to_cents(amount: float) -> int:
"""Convert WF dollar amount to integer cents."""
return round(amount * 100)
def _ynab_amount_to_cents(amount_milli: int) -> int:
"""Convert YNAB milliCurrency to integer cents (divide by 10)."""
return round(amount_milli / 10)
def _extract_account_balance(
extracted_objects: list[dict],
balance_type: str,
) -> float | None:
"""Extract the dollar balance from extracted WF objects.
Searches each object's accountModel.applicationData.account.balance[]
for an entry whose 'type' matches *balance_type* and returns its 'amount'.
Returns None if not found.
"""
for obj in extracted_objects:
balances = (
obj.get("accountModel", {})
.get("applicationData", {})
.get("account", {})
.get("balance", [])
)
for bal in balances:
if bal.get("type") == balance_type:
return bal.get("amount")
return None
def _run_extraction(
label: str,
input_file: str,
extractor: CreditExtractor | DebitExtractor,
) -> tuple[list[dict], list[dict]]:
"""Run extractor, save JSON output, and return (raw_txns, extracted_objects)."""
input_path = Path(input_file)
extracted = extractor.extract_from_file(input_path)
output_path = _default_output_path(input_path)
with output_path.open("w", encoding="utf-8") as f:
json.dump(extracted, f, indent=2)
print(f"[{label}] Extracted {len(extracted)} object(s) to {output_path}")
raw_txns = _extract_raw_transactions(extracted)
print(f"[{label}] Found {len(raw_txns)} raw transaction(s)")
if extractor.errors:
print(f"[{label}] Errors:")
for error in extractor.errors:
print(f" - {error}")
return raw_txns, extracted
def _run_ynab_transactions(days: int) -> None:
"""Fetch and display YNAB transactions for the past N days, excluding reconciled."""
client = YNABClient("ynab.yml")
transactions = client.get_transactions_for_days(
days=days,
exclude_cleared_statuses=["reconciled"],
)
print(
f"[ynab] Found {len(transactions)} transaction(s) "
f"for the past {days} day(s), excluding reconciled"
)
print(json.dumps(transactions, indent=2))
def _filter_wf_from_date(wf_transactions: list[dict], start_iso: str) -> list[dict]:
"""Keep only WF transactions whose transactionDate is on or after start_iso."""
return [
txn for txn in wf_transactions
if _wf_date_to_iso(txn["transactionDate"]) >= start_iso
]
def _find_new_transactions(
wf_transactions: list[dict],
ynab_transactions: list[dict],
) -> list[dict]:
"""
Return WF transactions that do NOT already exist in YNAB.
Match on (date, abs(amount_cents)):
- WF: transactionDate (ms) -> YYYY-MM-DD, transactionAmount * 100 -> cents
- YNAB: date (YYYY-MM-DD), abs(amount_milliCurrency / 10) -> cents
Uses a multiset so duplicate amounts on the same date are handled correctly.
"""
ynab_keys: dict[tuple[str, int], int] = {}
for txn in ynab_transactions:
key = (txn["date"], abs(_ynab_amount_to_cents(txn["amount_milliCurrency"])))
ynab_keys[key] = ynab_keys.get(key, 0) + 1
new_transactions: list[dict] = []
for txn in wf_transactions:
date_str = _wf_date_to_iso(txn["transactionDate"])
cents = abs(_wf_amount_to_cents(txn["transactionAmount"]))
key = (date_str, cents)
if ynab_keys.get(key, 0) > 0:
ynab_keys[key] -= 1
else:
new_transactions.append(txn)
return new_transactions
# ---------------------------------------------------------------------------
# YNAB Import helpers
# ---------------------------------------------------------------------------
def _wf_to_ynab_millis(wf_txn: dict) -> int:
"""Convert a WF transaction amount to YNAB milliCurrency.
Sign logic (unified for credit & debit cards):
- debitCreditType == "CREDIT" → inflow (positive)
- absent or "DEBIT" → outflow (negative)
"""
amount = wf_txn["transactionAmount"]
millis = round(amount * 1000)
if wf_txn.get("debitCreditType") == "CREDIT":
return millis # inflow
return -millis # outflow
def _build_import_ids(
amounts: list[int],
dates: list[str],
) -> list[str]:
"""Build YNAB import_id strings: ``YNAB:{amount}:{date}:{occurrence}``.
Occurrence starts at 1 and increments for duplicate (amount, date) pairs
within the batch.
"""
seen: dict[tuple[int, str], int] = {}
import_ids: list[str] = []
for amount, date_str in zip(amounts, dates):
key = (amount, date_str)
occurrence = seen.get(key, 0) + 1
seen[key] = occurrence
import_ids.append(f"YNAB:{amount}:{date_str}:{occurrence}")
return import_ids
def _convert_wf_to_ynab_transactions(
wf_transactions: list[dict],
account_id: str,
) -> list[dict]:
"""Convert raw WF transactions to YNAB-ready dicts.
Each dict has the keys expected by ``ynab.NewTransaction`` plus a
``_wf_source`` reference back to the original WF transaction.
"""
amounts: list[int] = []
dates: list[str] = []
for txn in wf_transactions:
amounts.append(_wf_to_ynab_millis(txn))
dates.append(_wf_date_to_iso(txn["transactionDate"]))
import_ids = _build_import_ids(amounts, dates)
results: list[dict] = []
for txn, amount, date_str, import_id in zip(
wf_transactions, amounts, dates, import_ids
):
status = txn.get("status", "POSTED")
cleared = "cleared" if status == "POSTED" else "uncleared"
payee = " ".join(txn.get("transactionDescription", "").split())
results.append({
"account_id": account_id,
"date": date_str,
"amount": amount,
"payee_name": payee[:200] if payee else None,
"cleared": cleared,
"flag_color": "blue",
"import_id": import_id,
"approved": False,
"_wf_source": txn,
})
return results
class _ImportClassification:
"""Holds the three buckets produced by ``_classify_for_import``."""
def __init__(self) -> None:
self.new: list[dict] = []
self.pending_to_posted: list[tuple[dict, YNABTransaction]] = []
self.duplicates: list[dict] = []
def _classify_for_import(
ynab_ready: list[dict],
ynab_transactions: list[YNABTransaction],
) -> _ImportClassification:
"""Classify incoming transactions into new / pending→posted / duplicate.
Matching rules (all use (date, abs(amount)) via multiset):
1. If a posted incoming txn matches an **uncleared** YNAB txn
→ pending→posted (update existing to cleared).
2. If a cleared YNAB txn already has the same import_id
→ duplicate (create with RED flag).
3. Otherwise → new.
"""
result = _ImportClassification()
# Build a set of existing cleared import_ids
cleared_import_ids: set[str] = set()
for t in ynab_transactions:
if t["cleared"] in ("cleared", "reconciled") and t.get("import_id"):
cleared_import_ids.add(t["import_id"])
# Build a multiset of uncleared YNAB txns keyed by (date, abs_amount)
# Each entry stores the list of YNAB transactions for that key.
uncleared_pool: dict[tuple[str, int], list[YNABTransaction]] = {}
for t in ynab_transactions:
if t["cleared"] == "uncleared":
key = (t["date"], abs(t["amount_milliCurrency"]))
uncleared_pool.setdefault(key, []).append(t)
for txn in ynab_ready:
import_id = txn["import_id"]
abs_amount = abs(txn["amount"])
txn_key = (txn["date"], abs_amount)
# 1. Pending→Posted: incoming is cleared, matches an uncleared YNAB txn
if txn["cleared"] == "cleared" and uncleared_pool.get(txn_key):
matched_ynab = uncleared_pool[txn_key].pop(0)
if not uncleared_pool[txn_key]:
del uncleared_pool[txn_key]
result.pending_to_posted.append((txn, matched_ynab))
# 2. Duplicate: import_id already exists on a cleared YNAB txn
elif import_id in cleared_import_ids:
txn["flag_color"] = "red"
result.duplicates.append(txn)
# 3. New
else:
result.new.append(txn)
return result
def _format_amount_dollars(millis: int) -> str:
"""Format milliCurrency as a dollar string with sign."""
dollars = millis / 1000.0
return f"${dollars:+,.2f}"
def _preview_import(classification: _ImportClassification) -> None:
"""Print a formatted preview of the import classification."""
col_date = 12
col_amount = 14
col_desc = 50
header = f" {'Date':<{col_date}} {'Amount':>{col_amount}} {'Description':<{col_desc}}"
separator = f" {'-' * col_date} {'-' * col_amount} {'-' * col_desc}"
# --- New ---
print(f"\n{'=' * 80}")
print(f" NEW TRANSACTIONS ({len(classification.new)}) — will be created with BLUE flag")
print(f"{'=' * 80}")
if classification.new:
print(header)
print(separator)
for txn in classification.new:
desc = (txn.get("payee_name") or "")[:col_desc]
print(f" {txn['date']:<{col_date}} {_format_amount_dollars(txn['amount']):>{col_amount}} {desc:<{col_desc}}")
else:
print(" (none)")
# --- Pending→Posted ---
print(f"\n{'=' * 80}")
print(f" PENDING → POSTED ({len(classification.pending_to_posted)}) — existing uncleared will be marked cleared")
print(f"{'=' * 80}")
if classification.pending_to_posted:
print(header)
print(separator)
for txn, matched in classification.pending_to_posted:
desc = (txn.get("payee_name") or "")[:col_desc]
print(f" {txn['date']:<{col_date}} {_format_amount_dollars(txn['amount']):>{col_amount}} {desc:<{col_desc}}")
else:
print(" (none)")
# --- Duplicates ---
print(f"\n{'=' * 80}")
print(f" POSSIBLE DUPLICATES ({len(classification.duplicates)}) — will be created with RED flag")
print(f"{'=' * 80}")
if classification.duplicates:
print(header)
print(separator)
for txn in classification.duplicates:
desc = (txn.get("payee_name") or "")[:col_desc]
print(f" {txn['date']:<{col_date}} {_format_amount_dollars(txn['amount']):>{col_amount}} {desc:<{col_desc}}")
else:
print(" (none)")
total = (
len(classification.new)
+ len(classification.pending_to_posted)
+ len(classification.duplicates)
)
print(f"\n Total: {total} transaction(s)")
print()
def _execute_import(
classification: _ImportClassification,
ynab_client: YNABClient,
) -> None:
"""Send classified transactions to YNAB."""
# --- Create new + duplicates ---
to_create = classification.new + classification.duplicates
if to_create:
new_txns = [
ynab.NewTransaction(
account_id=t["account_id"],
date=date.fromisoformat(t["date"]),
amount=t["amount"],
payee_name=t["payee_name"],
cleared=ynab.TransactionClearedStatus(t["cleared"]),
flag_color=ynab.TransactionFlagColor(t["flag_color"]),
import_id=t["import_id"],
approved=t.get("approved", False),
)
for t in to_create
]
resp = ynab_client.create_transactions(new_txns)
created_ids = resp.transaction_ids or []
dup_ids = resp.duplicate_import_ids or []
print(
f" [ynab] Created {len(created_ids)} transaction(s)"
+ (f", {len(dup_ids)} server-side duplicate(s) skipped" if dup_ids else "")
)
# --- Update pending→posted ---
if classification.pending_to_posted:
updated = 0
for _txn, matched_ynab in classification.pending_to_posted:
try:
ynab_client.update_transaction(
transaction_id=matched_ynab["id"],
cleared="cleared",
)
updated += 1
except Exception as e:
print(f" [ynab] Error updating {matched_ynab['id']}: {e}")
print(f" [ynab] Updated {updated} transaction(s) from uncleared → cleared")
def main() -> None:
parser = argparse.ArgumentParser(
description="Extract WF HAR files, compare with YNAB, find new transactions"
)
parser.add_argument(
"--credit",
default="",
help="Optional credit HAR input file",
)
parser.add_argument(
"--debit",
default="",
help="Optional debit HAR input file",
)
parser.add_argument(
"--lookback",
type=int,
default=3,
metavar="DAYS",
help="Days before last reconciled YNAB transaction to start comparison (default: 3)",
)
parser.add_argument(
"--ynab-transactions",
type=int,
default=None,
metavar="DAYS",
help="List YNAB transactions for the past DAYS (excluding reconciled)",
)
parser.add_argument(
"--ynab-import",
action="store_true",
default=False,
help="Import new WF transactions into YNAB (requires --credit and/or --debit)",
)
args = parser.parse_args()
# Load YNAB config to map extractors -> account IDs
ynab_client = YNABClient("ynab.yml")
account_configs = ynab_client.account_configs
# Build extractor_name -> [account_id, ...] mapping from ynab.yml
extractor_to_accounts: dict[str, list[str]] = {}
for acct in account_configs:
ext_name = acct.get("extractor", "")
if ext_name:
extractor_to_accounts.setdefault(ext_name, []).append(acct["id"])
# -- Extract WF transactions -------------------------------------------
extracted_by_extractor: dict[str, list[dict]] = {}
objects_by_extractor: dict[str, list[dict]] = {}
for flag, extractor_name in FLAG_TO_EXTRACTOR.items():
input_file = getattr(args, flag, "")
if not input_file:
continue
extractor_cls = EXTRACTOR_MAP[extractor_name]
raw_txns, extracted_objects = _run_extraction(flag, input_file, extractor_cls())
extracted_by_extractor[extractor_name] = raw_txns
objects_by_extractor[extractor_name] = extracted_objects
# -- Compare with YNAB -------------------------------------------------
if extracted_by_extractor:
# Gather all relevant account IDs across active extractors
all_account_ids = [
aid
for ext_name in extracted_by_extractor
for aid in extractor_to_accounts.get(ext_name, [])
]
# Find the last reconciled transaction date in YNAB
last_reconciled = ynab_client.get_last_reconciled_date(
account_ids=all_account_ids
)
if last_reconciled is None:
print("\n[ynab] No reconciled transactions found; defaulting to 30 day lookback")
start_date = (datetime.now() - timedelta(days=30)).date()
else:
start_date = last_reconciled - timedelta(days=args.lookback)
print(
f"\n[ynab] Last reconciled date: {last_reconciled}"
f" → start comparison from {start_date} "
f"({args.lookback} day lookback)"
)
start_iso = start_date.isoformat()
days_from_start = (datetime.now().date() - start_date).days + 1
# Fetch ALL YNAB transactions from start_date onward (including reconciled)
# so that reconciled matches also get consumed during comparison.
all_ynab_txns = ynab_client.get_transactions_for_days(
days=days_from_start,
)
print(f"[ynab] Retrieved {len(all_ynab_txns)} transaction(s) from {start_iso} onward")
# Compare per-extractor, scoped to the matching YNAB account IDs
for extractor_name, wf_txns in extracted_by_extractor.items():
account_ids = set(extractor_to_accounts.get(extractor_name, []))
scoped_ynab = [t for t in all_ynab_txns if t["account_id"] in account_ids]
# Filter WF transactions to the same start date
filtered_wf = _filter_wf_from_date(wf_txns, start_iso)
new_txns = _find_new_transactions(filtered_wf, scoped_ynab)
label = extractor_name.replace("wf_", "").replace("_extractor", "")
print(
f"\n[{label}] {len(new_txns)} new transaction(s) "
f"not yet in YNAB (of {len(filtered_wf)} extracted from {start_iso}, "
f"{len(scoped_ynab)} in YNAB)"
)
if new_txns:
print(json.dumps(new_txns, indent=2))
# -- YNAB Import -------------------------------------------------------
if args.ynab_import and extracted_by_extractor:
all_ynab_ready: list[dict] = []
balances: dict[str, float] = {} # label -> balance amount
for extractor_name, wf_txns in extracted_by_extractor.items():
account_ids_for_ext = extractor_to_accounts.get(extractor_name, [])
if not account_ids_for_ext:
continue
# Use the first account ID for this extractor
target_account_id = account_ids_for_ext[0]
scoped_ynab = [
t for t in all_ynab_txns if t["account_id"] in set(account_ids_for_ext)
]
# Filter WF to the comparison window, then find only new ones
filtered_wf = _filter_wf_from_date(wf_txns, start_iso)
new_wf = _find_new_transactions(filtered_wf, scoped_ynab)
# Convert the new WF transactions to YNAB-ready dicts
ynab_ready = _convert_wf_to_ynab_transactions(new_wf, target_account_id)
# Classify against existing YNAB transactions
classification = _classify_for_import(ynab_ready, scoped_ynab)
label = extractor_name.replace("wf_", "").replace("_extractor", "")
print(f"\n--- {label.upper()} IMPORT PREVIEW ---")
# Collect account balance for bottom-of-report summary
balance_type = EXTRACTOR_BALANCE_TYPE.get(extractor_name)
if balance_type:
balance = _extract_account_balance(
objects_by_extractor.get(extractor_name, []),
balance_type,
)
if balance is not None:
balances[label] = balance
_preview_import(classification)
all_ynab_ready.append({
"label": label,
"classification": classification,
})
# Display account balances at the bottom of the report
if balances:
parts = []
for lbl, amt in balances.items():
parts.append(f"{lbl.title()} ${amt:,.2f}")
print(f"{'=' * 80}")
print(" " + " ".join(parts))
print(f"{'=' * 80}\n")
# Check if there's anything to import
has_work = any(
entry["classification"].new
or entry["classification"].pending_to_posted
or entry["classification"].duplicates
for entry in all_ynab_ready
)
if has_work:
answer = input("Proceed with import? [y/N]: ").strip().lower()
if answer == "y":
for entry in all_ynab_ready:
label = entry["label"]
classification = entry["classification"]
print(f"\n[{label}] Importing...")
_execute_import(classification, ynab_client)
print("\n[ynab] Import complete.")
else:
print("\n[ynab] Import cancelled.")
else:
print("\n[ynab] Nothing to import.")
elif args.ynab_import and not extracted_by_extractor:
print("[ynab-import] No extracted transactions. Use --credit and/or --debit.")
# -- Standalone YNAB listing -------------------------------------------
if args.ynab_transactions is not None:
_run_ynab_transactions(args.ynab_transactions)
if not args.credit and not args.debit and args.ynab_transactions is None:
print("No input provided. Use --credit, --debit, --ynab-import, and/or --ynab-transactions.")
if __name__ == "__main__":
main()