add transaction CSV export and category suggestion features
This commit is contained in:
@@ -1,11 +1,14 @@
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import ynab
|
||||
|
||||
from category_suggester import CategoryHistory
|
||||
from wf_credit_extractor import WellsFargoPayloadExtractor as CreditExtractor
|
||||
from wf_debit_extractor import WellsFargoPayloadExtractor as DebitExtractor
|
||||
from ynab_client import YNABClient, YNABTransaction
|
||||
@@ -166,6 +169,167 @@ def _run_ynab_transactions(days: int) -> None:
|
||||
print(json.dumps(transactions, indent=2))
|
||||
|
||||
|
||||
def _parse_record_days(value: str) -> int:
|
||||
"""Validate the supported lookback for transaction CSV exports."""
|
||||
try:
|
||||
days = int(value)
|
||||
except ValueError as exc:
|
||||
raise argparse.ArgumentTypeError("days must be an integer from 1 to 240") from exc
|
||||
if not 1 <= days <= 240:
|
||||
raise argparse.ArgumentTypeError("days must be between 1 and 240")
|
||||
return days
|
||||
|
||||
|
||||
def _write_transactions_csv(
|
||||
transactions: list[YNABTransaction],
|
||||
output_path: Path,
|
||||
) -> int:
|
||||
"""Write YNAB transactions as one CSV row per category line item."""
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fieldnames = ["payee", "category", "inflow", "outflow"]
|
||||
row_count = 0
|
||||
|
||||
with output_path.open("w", encoding="utf-8", newline="") as csv_file:
|
||||
writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
|
||||
for transaction in transactions:
|
||||
line_items = transaction.get("subtransactions") or [transaction]
|
||||
for line_item in line_items:
|
||||
amount = Decimal(line_item["amount_milliCurrency"]) / Decimal(1000)
|
||||
if amount >= 0:
|
||||
inflow = f"{amount:.2f}"
|
||||
outflow = ""
|
||||
else:
|
||||
inflow = ""
|
||||
outflow = f"{abs(amount):.2f}"
|
||||
|
||||
writer.writerow({
|
||||
"payee": transaction.get("payee_name") or "",
|
||||
"category": line_item.get("category_name") or "",
|
||||
"inflow": inflow,
|
||||
"outflow": outflow,
|
||||
})
|
||||
row_count += 1
|
||||
|
||||
return row_count
|
||||
|
||||
|
||||
def _record_transactions_csv(
|
||||
ynab_client: YNABClient,
|
||||
days: int,
|
||||
output_path: Path,
|
||||
) -> None:
|
||||
"""Fetch credit and debit account transactions and save their CSV rows."""
|
||||
target_extractors = set(FLAG_TO_EXTRACTOR.values())
|
||||
target_names = {"Credit Card", "Debit Card", "Wells Fargo Bank"}
|
||||
target_account_ids = {
|
||||
account["id"]
|
||||
for account in ynab_client.account_configs
|
||||
if account.get("extractor") in target_extractors
|
||||
or account.get("name") in target_names
|
||||
}
|
||||
if not target_account_ids:
|
||||
raise ValueError(
|
||||
"No Credit Card or Wells Fargo Bank accounts found in ynab.yml"
|
||||
)
|
||||
|
||||
transactions = ynab_client.get_transactions_for_days(
|
||||
days=days,
|
||||
include_subtransactions=True,
|
||||
)
|
||||
selected_transactions = [
|
||||
transaction
|
||||
for transaction in transactions
|
||||
if transaction["account_id"] in target_account_ids
|
||||
]
|
||||
|
||||
history, memory_was_readable = _load_category_history(
|
||||
output_path,
|
||||
report_empty=False,
|
||||
)
|
||||
row_count = _write_transactions_csv(selected_transactions, output_path)
|
||||
memory_path = CategoryHistory.memory_path_for_csv(output_path)
|
||||
try:
|
||||
history.merge(CategoryHistory.from_csv(output_path))
|
||||
history.save(memory_path)
|
||||
action = "Updated" if memory_was_readable else "Rebuilt"
|
||||
print(f"[category] {action} reusable category patterns in {memory_path}")
|
||||
except (OSError, ValueError) as exc:
|
||||
print(f"[category] Could not update {memory_path}: {exc}")
|
||||
print(
|
||||
f"[ynab] Wrote {row_count} transaction row(s) from the past {days} day(s) "
|
||||
f"to {output_path}"
|
||||
)
|
||||
|
||||
|
||||
def _load_category_history(
|
||||
csv_path: Path,
|
||||
*,
|
||||
report_empty: bool = True,
|
||||
) -> tuple[CategoryHistory, bool]:
|
||||
"""Load persistent patterns and merge the current CSV without losing old data."""
|
||||
memory_path = CategoryHistory.memory_path_for_csv(csv_path)
|
||||
memory_exists = memory_path.exists()
|
||||
can_update_memory = True
|
||||
|
||||
if memory_exists:
|
||||
try:
|
||||
history = CategoryHistory.load(memory_path)
|
||||
except (
|
||||
OSError,
|
||||
ValueError,
|
||||
TypeError,
|
||||
KeyError,
|
||||
AttributeError,
|
||||
json.JSONDecodeError,
|
||||
) as exc:
|
||||
print(f"[category] Could not read {memory_path}: {exc}")
|
||||
history = CategoryHistory()
|
||||
can_update_memory = False
|
||||
else:
|
||||
history = CategoryHistory()
|
||||
|
||||
try:
|
||||
current_csv = CategoryHistory.from_csv(csv_path)
|
||||
added_patterns = history.merge(current_csv)
|
||||
if can_update_memory and (not memory_exists or added_patterns):
|
||||
history.save(memory_path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except (OSError, ValueError, csv.Error) as exc:
|
||||
print(f"[category] Could not read transaction history {csv_path}: {exc}")
|
||||
|
||||
if report_empty and not history.patterns:
|
||||
print(
|
||||
f"[category] No usable history found in {csv_path} or {memory_path}; "
|
||||
"new transactions will be imported without category suggestions."
|
||||
)
|
||||
return history, can_update_memory
|
||||
|
||||
|
||||
def _apply_category_suggestions(
|
||||
transactions: list[dict],
|
||||
history: CategoryHistory,
|
||||
category_ids_by_name: dict[str, list[str]],
|
||||
) -> None:
|
||||
"""Annotate import candidates and set category_id only for confident matches."""
|
||||
for transaction in transactions:
|
||||
suggestion = history.suggest(
|
||||
transaction.get("payee_name") or "",
|
||||
transaction["amount"],
|
||||
category_ids_by_name,
|
||||
)
|
||||
if suggestion is None:
|
||||
continue
|
||||
|
||||
transaction["_category_suggestion"] = suggestion.category_name
|
||||
transaction["_category_confidence"] = suggestion.confidence
|
||||
transaction["_category_applied"] = suggestion.applied
|
||||
if suggestion.category_id is not None:
|
||||
transaction["category_id"] = suggestion.category_id
|
||||
|
||||
|
||||
def _filter_wf_from_date(wf_transactions: list[dict], start_iso: str) -> list[dict]:
|
||||
"""Keep only WF transactions whose transactionDate is on or after start_iso."""
|
||||
return [
|
||||
@@ -353,46 +517,69 @@ def _preview_import(classification: _ImportClassification) -> None:
|
||||
"""Print a formatted preview of the import classification."""
|
||||
col_date = 12
|
||||
col_amount = 14
|
||||
col_desc = 50
|
||||
header = f" {'Date':<{col_date}} {'Amount':>{col_amount}} {'Description':<{col_desc}}"
|
||||
separator = f" {'-' * col_date} {'-' * col_amount} {'-' * col_desc}"
|
||||
col_desc = 34
|
||||
col_category = 28
|
||||
report_width = col_date + col_amount + col_desc + col_category + 10
|
||||
header = (
|
||||
f" {'Date':<{col_date}} {'Amount':>{col_amount}} "
|
||||
f"{'Description':<{col_desc}} {'Category suggestion':<{col_category}}"
|
||||
)
|
||||
separator = (
|
||||
f" {'-' * col_date} {'-' * col_amount} {'-' * col_desc} "
|
||||
f"{'-' * col_category}"
|
||||
)
|
||||
|
||||
def category_text(txn: dict) -> str:
|
||||
suggestion = txn.get("_category_suggestion")
|
||||
if not suggestion:
|
||||
return "(uncategorized)"
|
||||
confidence = txn.get("_category_confidence", 0.0)
|
||||
prefix = "" if txn.get("_category_applied") else "? "
|
||||
return f"{prefix}{suggestion} ({confidence:.0%})"
|
||||
|
||||
def row_text(txn: dict, category: str) -> str:
|
||||
desc = (txn.get("payee_name") or "")[:col_desc]
|
||||
category = category[:col_category]
|
||||
return (
|
||||
f" {txn['date']:<{col_date}} "
|
||||
f"{_format_amount_dollars(txn['amount']):>{col_amount}} "
|
||||
f"{desc:<{col_desc}} {category:<{col_category}}"
|
||||
)
|
||||
|
||||
# --- New ---
|
||||
print(f"\n{'=' * 80}")
|
||||
print(f"\n{'=' * report_width}")
|
||||
print(f" NEW TRANSACTIONS ({len(classification.new)}) — will be created with BLUE flag")
|
||||
print(f"{'=' * 80}")
|
||||
print(f"{'=' * report_width}")
|
||||
if classification.new:
|
||||
print(header)
|
||||
print(separator)
|
||||
for txn in classification.new:
|
||||
desc = (txn.get("payee_name") or "")[:col_desc]
|
||||
print(f" {txn['date']:<{col_date}} {_format_amount_dollars(txn['amount']):>{col_amount}} {desc:<{col_desc}}")
|
||||
print(row_text(txn, category_text(txn)))
|
||||
else:
|
||||
print(" (none)")
|
||||
|
||||
# --- Pending→Posted ---
|
||||
print(f"\n{'=' * 80}")
|
||||
print(f"\n{'=' * report_width}")
|
||||
print(f" PENDING → POSTED ({len(classification.pending_to_posted)}) — existing uncleared will be marked cleared")
|
||||
print(f"{'=' * 80}")
|
||||
print(f"{'=' * report_width}")
|
||||
if classification.pending_to_posted:
|
||||
print(header)
|
||||
print(separator)
|
||||
for txn, matched in classification.pending_to_posted:
|
||||
desc = (txn.get("payee_name") or "")[:col_desc]
|
||||
print(f" {txn['date']:<{col_date}} {_format_amount_dollars(txn['amount']):>{col_amount}} {desc:<{col_desc}}")
|
||||
existing_category = matched.get("category_name") or "(unchanged)"
|
||||
print(row_text(txn, existing_category))
|
||||
else:
|
||||
print(" (none)")
|
||||
|
||||
# --- Duplicates ---
|
||||
print(f"\n{'=' * 80}")
|
||||
print(f"\n{'=' * report_width}")
|
||||
print(f" POSSIBLE DUPLICATES ({len(classification.duplicates)}) — will be created with RED flag")
|
||||
print(f"{'=' * 80}")
|
||||
print(f"{'=' * report_width}")
|
||||
if classification.duplicates:
|
||||
print(header)
|
||||
print(separator)
|
||||
for txn in classification.duplicates:
|
||||
desc = (txn.get("payee_name") or "")[:col_desc]
|
||||
print(f" {txn['date']:<{col_date}} {_format_amount_dollars(txn['amount']):>{col_amount}} {desc:<{col_desc}}")
|
||||
print(row_text(txn, category_text(txn)))
|
||||
else:
|
||||
print(" (none)")
|
||||
|
||||
@@ -420,6 +607,7 @@ def _execute_import(
|
||||
date=date.fromisoformat(t["date"]),
|
||||
amount=t["amount"],
|
||||
payee_name=t["payee_name"],
|
||||
category_id=t.get("category_id"),
|
||||
cleared=ynab.TransactionClearedStatus(t["cleared"]),
|
||||
flag_color=ynab.TransactionFlagColor(t["flag_color"]),
|
||||
import_id=t["import_id"],
|
||||
@@ -478,6 +666,24 @@ def main() -> None:
|
||||
metavar="DAYS",
|
||||
help="List YNAB transactions for the past DAYS (excluding reconciled)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--record-transactions",
|
||||
"--record-trasactions",
|
||||
dest="record_transactions",
|
||||
type=_parse_record_days,
|
||||
nargs="?",
|
||||
const=60,
|
||||
default=None,
|
||||
metavar="DAYS",
|
||||
help="Write Credit Card and Wells Fargo Bank transactions to CSV (default: 60 days; max: 240)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--transactions-csv",
|
||||
type=Path,
|
||||
default=Path("transactions.csv"),
|
||||
metavar="PATH",
|
||||
help="CSV history/export path for category learning (default: transactions.csv)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ynab-import",
|
||||
action="store_true",
|
||||
@@ -488,6 +694,12 @@ def main() -> None:
|
||||
|
||||
# Load YNAB config to map extractors -> account IDs
|
||||
ynab_client = YNABClient("ynab.yml")
|
||||
if args.record_transactions is not None:
|
||||
_record_transactions_csv(
|
||||
ynab_client,
|
||||
args.record_transactions,
|
||||
args.transactions_csv,
|
||||
)
|
||||
account_configs = ynab_client.account_configs
|
||||
|
||||
# Build extractor_name -> [account_id, ...] mapping from ynab.yml
|
||||
@@ -568,6 +780,15 @@ def main() -> None:
|
||||
if args.ynab_import and extracted_by_extractor:
|
||||
all_ynab_ready: list[dict] = []
|
||||
balances: dict[str, float] = {} # label -> balance amount
|
||||
category_history, _ = _load_category_history(args.transactions_csv)
|
||||
try:
|
||||
category_ids_by_name = ynab_client.get_category_ids_by_name()
|
||||
except Exception as exc:
|
||||
print(
|
||||
f"[category] Could not load YNAB category IDs ({exc}); "
|
||||
"suggestions will not be assigned."
|
||||
)
|
||||
category_ids_by_name = {}
|
||||
|
||||
for extractor_name, wf_txns in extracted_by_extractor.items():
|
||||
account_ids_for_ext = extractor_to_accounts.get(extractor_name, [])
|
||||
@@ -596,6 +817,11 @@ def main() -> None:
|
||||
ynab_ready = _convert_wf_to_ynab_transactions(
|
||||
import_candidates, target_account_id,
|
||||
)
|
||||
_apply_category_suggestions(
|
||||
ynab_ready,
|
||||
category_history,
|
||||
category_ids_by_name,
|
||||
)
|
||||
|
||||
# Classify against existing YNAB transactions
|
||||
classification = _classify_for_import(ynab_ready, scoped_ynab)
|
||||
@@ -658,8 +884,16 @@ def main() -> None:
|
||||
if args.ynab_transactions is not None:
|
||||
_run_ynab_transactions(args.ynab_transactions)
|
||||
|
||||
if not args.credit and not args.debit and args.ynab_transactions is None:
|
||||
print("No input provided. Use --credit, --debit, --ynab-import, and/or --ynab-transactions.")
|
||||
if (
|
||||
not args.credit
|
||||
and not args.debit
|
||||
and args.ynab_transactions is None
|
||||
and args.record_transactions is None
|
||||
):
|
||||
print(
|
||||
"No input provided. Use --credit, --debit, --ynab-import, "
|
||||
"--ynab-transactions, and/or --record-transactions."
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user