feat: migrate backend persistence from TinyDB to MongoDB Atlas
- Implement lazy MongoDB client initialization in backend/db/mongo_client.py. - Create Gunicorn configuration to ensure MongoDB client is initialized per worker. - Refactor database access layer to support MongoDB with a new MongoLockedTable adapter. - Add migration script to transfer existing TinyDB data to MongoDB, preserving idempotency. - Update tracking event handling to ensure deterministic ordering with a monotonic sequence. - Modify tests to use mongomock for MongoDB integration and ensure existing tests pass. - Add integration test script to run tests against a local MongoDB Docker container. - Document environment variables and migration process in specs/feat-database-migration.md.
This commit is contained in:
@@ -0,0 +1,203 @@
|
||||
# python
|
||||
"""
|
||||
Migrate existing TinyDB JSON files into MongoDB.
|
||||
|
||||
Usage:
|
||||
cd backend
|
||||
python -m scripts.migrate_to_mongodb [--dry-run]
|
||||
|
||||
The script reads files from ``data/db/`` (or ``test_data/db/`` when
|
||||
``DB_ENV=test``), maps each record's ``id`` field to MongoDB's ``_id`` field,
|
||||
and inserts the records idempotently. TinyDB files are backed up to
|
||||
``<db_dir>/backups/<timestamp>/`` before the first migration run.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
|
||||
|
||||
from config.paths import get_database_dir
|
||||
from db.db import COLLECTION_INDEXES, ensure_mongodb_indexes
|
||||
from db.mongo_client import get_mongo_client, get_mongo_db_name
|
||||
|
||||
|
||||
# Map TinyDB JSON filenames to MongoDB collection names.
|
||||
COLLECTION_FILE_MAP = {
|
||||
'children.json': 'children',
|
||||
'tasks.json': 'tasks',
|
||||
'routines.json': 'routines',
|
||||
'routine_items.json': 'routine_items',
|
||||
'routine_schedules.json': 'routine_schedules',
|
||||
'routine_extensions.json': 'routine_extensions',
|
||||
'rewards.json': 'rewards',
|
||||
'images.json': 'images',
|
||||
'pending_rewards.json': 'pending_rewards',
|
||||
'pending_confirmations.json': 'pending_confirmations',
|
||||
'users.json': 'users',
|
||||
'tracking_events.json': 'tracking_events',
|
||||
'child_overrides.json': 'child_overrides',
|
||||
'chore_schedules.json': 'chore_schedules',
|
||||
'task_extensions.json': 'task_extensions',
|
||||
'refresh_tokens.json': 'refresh_tokens',
|
||||
'push_subscriptions.json': 'push_subscriptions',
|
||||
'digest_action_tokens.json': 'digest_action_tokens',
|
||||
}
|
||||
|
||||
|
||||
def _load_tinydb_records(path: str) -> list[dict]:
|
||||
"""Load all records from a TinyDB JSON file."""
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
default_table = data.get('_default', {})
|
||||
return list(default_table.values())
|
||||
|
||||
|
||||
def _doc_to_mongo(doc: dict) -> dict:
|
||||
"""Map the model ``id`` field to MongoDB's ``_id`` field.
|
||||
|
||||
The original ``id`` field is removed so documents do not store both
|
||||
``_id`` and ``id`` with identical values.
|
||||
"""
|
||||
mongo_doc = dict(doc)
|
||||
if 'id' in mongo_doc:
|
||||
mongo_doc['_id'] = mongo_doc.pop('id')
|
||||
return mongo_doc
|
||||
|
||||
|
||||
def migrate(dry_run: bool = False) -> dict:
|
||||
"""Migrate TinyDB files to MongoDB and return a per-collection summary."""
|
||||
db_dir = get_database_dir()
|
||||
if not os.path.isdir(db_dir):
|
||||
raise FileNotFoundError(f'Database directory does not exist: {db_dir}')
|
||||
|
||||
client = get_mongo_client()
|
||||
db_name = get_mongo_db_name()
|
||||
db = client[db_name]
|
||||
|
||||
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
|
||||
backup_dir = os.path.join(db_dir, 'backups', timestamp)
|
||||
|
||||
if not dry_run:
|
||||
os.makedirs(backup_dir, exist_ok=True)
|
||||
ensure_mongodb_indexes(client=client, db_name=db_name)
|
||||
|
||||
summary: dict[str, dict] = {}
|
||||
|
||||
for filename, collection_name in COLLECTION_FILE_MAP.items():
|
||||
path = os.path.join(db_dir, filename)
|
||||
if not os.path.exists(path):
|
||||
summary[collection_name] = {
|
||||
'source_file': filename,
|
||||
'total': 0,
|
||||
'migrated': 0,
|
||||
'skipped': 0,
|
||||
'status': 'missing',
|
||||
}
|
||||
continue
|
||||
|
||||
records = _load_tinydb_records(path)
|
||||
|
||||
if not dry_run:
|
||||
shutil.copy2(path, backup_dir)
|
||||
|
||||
collection = db[collection_name]
|
||||
to_insert: list[dict] = []
|
||||
skipped = 0
|
||||
|
||||
for record in records:
|
||||
doc_id = record.get('id')
|
||||
if not doc_id:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
if not dry_run:
|
||||
existing = collection.find_one({'_id': doc_id})
|
||||
if existing:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
to_insert.append(_doc_to_mongo(record))
|
||||
|
||||
if not dry_run and to_insert:
|
||||
try:
|
||||
collection.insert_many(to_insert, ordered=False)
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
print(
|
||||
f' Warning: error inserting into {collection_name}: {exc}',
|
||||
file=sys.stderr,
|
||||
)
|
||||
raise
|
||||
|
||||
summary[collection_name] = {
|
||||
'source_file': filename,
|
||||
'total': len(records),
|
||||
'migrated': len(to_insert),
|
||||
'skipped': skipped,
|
||||
'status': 'migrated' if not dry_run else 'dry-run',
|
||||
}
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
def _print_summary(summary: dict) -> None:
|
||||
"""Print a human-readable migration summary."""
|
||||
print('\nMigration Summary')
|
||||
print('-' * 70)
|
||||
print(f'{"Collection":<30}{"Total":>8}{"Migrated":>10}{"Skipped":>10}{"Status":>10}')
|
||||
print('-' * 70)
|
||||
total_records = 0
|
||||
total_migrated = 0
|
||||
total_skipped = 0
|
||||
for collection_name, info in summary.items():
|
||||
print(
|
||||
f'{collection_name:<30}'
|
||||
f'{info["total"]:>8}'
|
||||
f'{info["migrated"]:>10}'
|
||||
f'{info["skipped"]:>10}'
|
||||
f'{info["status"]:>10}'
|
||||
)
|
||||
total_records += info['total']
|
||||
total_migrated += info['migrated']
|
||||
total_skipped += info['skipped']
|
||||
print('-' * 70)
|
||||
print(
|
||||
f'{"TOTAL":<30}'
|
||||
f'{total_records:>8}'
|
||||
f'{total_migrated:>10}'
|
||||
f'{total_skipped:>10}'
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Migrate TinyDB JSON files to MongoDB.'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--dry-run',
|
||||
action='store_true',
|
||||
help='Analyze files and report counts without writing to MongoDB.',
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.environ.get('USE_MONGODB', 'false').lower() != 'true':
|
||||
print('Set USE_MONGODB=true to run the migration.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
if not os.environ.get('MONGO_URI'):
|
||||
print('MONGO_URI is required when USE_MONGODB=true.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
if args.dry_run:
|
||||
print('Dry run: no data will be written to MongoDB.')
|
||||
|
||||
summary = migrate(dry_run=args.dry_run)
|
||||
_print_summary(summary)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user