feat: migrate backend persistence from TinyDB to MongoDB Atlas
- Implement lazy MongoDB client initialization in backend/db/mongo_client.py. - Create Gunicorn configuration to ensure MongoDB client is initialized per worker. - Refactor database access layer to support MongoDB with a new MongoLockedTable adapter. - Add migration script to transfer existing TinyDB data to MongoDB, preserving idempotency. - Update tracking event handling to ensure deterministic ordering with a monotonic sequence. - Modify tests to use mongomock for MongoDB integration and ensure existing tests pass. - Add integration test script to run tests against a local MongoDB Docker container. - Document environment variables and migration process in specs/feat-database-migration.md.
This commit is contained in:
@@ -0,0 +1,203 @@
|
||||
# python
|
||||
"""
|
||||
Migrate existing TinyDB JSON files into MongoDB.
|
||||
|
||||
Usage:
|
||||
cd backend
|
||||
python -m scripts.migrate_to_mongodb [--dry-run]
|
||||
|
||||
The script reads files from ``data/db/`` (or ``test_data/db/`` when
|
||||
``DB_ENV=test``), maps each record's ``id`` field to MongoDB's ``_id`` field,
|
||||
and inserts the records idempotently. TinyDB files are backed up to
|
||||
``<db_dir>/backups/<timestamp>/`` before the first migration run.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
|
||||
|
||||
from config.paths import get_database_dir
|
||||
from db.db import COLLECTION_INDEXES, ensure_mongodb_indexes
|
||||
from db.mongo_client import get_mongo_client, get_mongo_db_name
|
||||
|
||||
|
||||
# Map TinyDB JSON filenames to MongoDB collection names.
|
||||
COLLECTION_FILE_MAP = {
|
||||
'children.json': 'children',
|
||||
'tasks.json': 'tasks',
|
||||
'routines.json': 'routines',
|
||||
'routine_items.json': 'routine_items',
|
||||
'routine_schedules.json': 'routine_schedules',
|
||||
'routine_extensions.json': 'routine_extensions',
|
||||
'rewards.json': 'rewards',
|
||||
'images.json': 'images',
|
||||
'pending_rewards.json': 'pending_rewards',
|
||||
'pending_confirmations.json': 'pending_confirmations',
|
||||
'users.json': 'users',
|
||||
'tracking_events.json': 'tracking_events',
|
||||
'child_overrides.json': 'child_overrides',
|
||||
'chore_schedules.json': 'chore_schedules',
|
||||
'task_extensions.json': 'task_extensions',
|
||||
'refresh_tokens.json': 'refresh_tokens',
|
||||
'push_subscriptions.json': 'push_subscriptions',
|
||||
'digest_action_tokens.json': 'digest_action_tokens',
|
||||
}
|
||||
|
||||
|
||||
def _load_tinydb_records(path: str) -> list[dict]:
|
||||
"""Load all records from a TinyDB JSON file."""
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
default_table = data.get('_default', {})
|
||||
return list(default_table.values())
|
||||
|
||||
|
||||
def _doc_to_mongo(doc: dict) -> dict:
|
||||
"""Map the model ``id`` field to MongoDB's ``_id`` field.
|
||||
|
||||
The original ``id`` field is removed so documents do not store both
|
||||
``_id`` and ``id`` with identical values.
|
||||
"""
|
||||
mongo_doc = dict(doc)
|
||||
if 'id' in mongo_doc:
|
||||
mongo_doc['_id'] = mongo_doc.pop('id')
|
||||
return mongo_doc
|
||||
|
||||
|
||||
def migrate(dry_run: bool = False) -> dict:
|
||||
"""Migrate TinyDB files to MongoDB and return a per-collection summary."""
|
||||
db_dir = get_database_dir()
|
||||
if not os.path.isdir(db_dir):
|
||||
raise FileNotFoundError(f'Database directory does not exist: {db_dir}')
|
||||
|
||||
client = get_mongo_client()
|
||||
db_name = get_mongo_db_name()
|
||||
db = client[db_name]
|
||||
|
||||
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
|
||||
backup_dir = os.path.join(db_dir, 'backups', timestamp)
|
||||
|
||||
if not dry_run:
|
||||
os.makedirs(backup_dir, exist_ok=True)
|
||||
ensure_mongodb_indexes(client=client, db_name=db_name)
|
||||
|
||||
summary: dict[str, dict] = {}
|
||||
|
||||
for filename, collection_name in COLLECTION_FILE_MAP.items():
|
||||
path = os.path.join(db_dir, filename)
|
||||
if not os.path.exists(path):
|
||||
summary[collection_name] = {
|
||||
'source_file': filename,
|
||||
'total': 0,
|
||||
'migrated': 0,
|
||||
'skipped': 0,
|
||||
'status': 'missing',
|
||||
}
|
||||
continue
|
||||
|
||||
records = _load_tinydb_records(path)
|
||||
|
||||
if not dry_run:
|
||||
shutil.copy2(path, backup_dir)
|
||||
|
||||
collection = db[collection_name]
|
||||
to_insert: list[dict] = []
|
||||
skipped = 0
|
||||
|
||||
for record in records:
|
||||
doc_id = record.get('id')
|
||||
if not doc_id:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
if not dry_run:
|
||||
existing = collection.find_one({'_id': doc_id})
|
||||
if existing:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
to_insert.append(_doc_to_mongo(record))
|
||||
|
||||
if not dry_run and to_insert:
|
||||
try:
|
||||
collection.insert_many(to_insert, ordered=False)
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
print(
|
||||
f' Warning: error inserting into {collection_name}: {exc}',
|
||||
file=sys.stderr,
|
||||
)
|
||||
raise
|
||||
|
||||
summary[collection_name] = {
|
||||
'source_file': filename,
|
||||
'total': len(records),
|
||||
'migrated': len(to_insert),
|
||||
'skipped': skipped,
|
||||
'status': 'migrated' if not dry_run else 'dry-run',
|
||||
}
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
def _print_summary(summary: dict) -> None:
|
||||
"""Print a human-readable migration summary."""
|
||||
print('\nMigration Summary')
|
||||
print('-' * 70)
|
||||
print(f'{"Collection":<30}{"Total":>8}{"Migrated":>10}{"Skipped":>10}{"Status":>10}')
|
||||
print('-' * 70)
|
||||
total_records = 0
|
||||
total_migrated = 0
|
||||
total_skipped = 0
|
||||
for collection_name, info in summary.items():
|
||||
print(
|
||||
f'{collection_name:<30}'
|
||||
f'{info["total"]:>8}'
|
||||
f'{info["migrated"]:>10}'
|
||||
f'{info["skipped"]:>10}'
|
||||
f'{info["status"]:>10}'
|
||||
)
|
||||
total_records += info['total']
|
||||
total_migrated += info['migrated']
|
||||
total_skipped += info['skipped']
|
||||
print('-' * 70)
|
||||
print(
|
||||
f'{"TOTAL":<30}'
|
||||
f'{total_records:>8}'
|
||||
f'{total_migrated:>10}'
|
||||
f'{total_skipped:>10}'
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Migrate TinyDB JSON files to MongoDB.'
|
||||
)
|
||||
parser.add_argument(
|
||||
'--dry-run',
|
||||
action='store_true',
|
||||
help='Analyze files and report counts without writing to MongoDB.',
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if os.environ.get('USE_MONGODB', 'false').lower() != 'true':
|
||||
print('Set USE_MONGODB=true to run the migration.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
if not os.environ.get('MONGO_URI'):
|
||||
print('MONGO_URI is required when USE_MONGODB=true.', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
if args.dry_run:
|
||||
print('Dry run: no data will be written to MongoDB.')
|
||||
|
||||
summary = migrate(dry_run=args.dry_run)
|
||||
_print_summary(summary)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,79 @@
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Run the MongoDB adapter integration tests against a local Docker MongoDB container.
|
||||
|
||||
.DESCRIPTION
|
||||
Starts a temporary MongoDB container, runs a targeted pytest suite with
|
||||
USE_MONGODB=true, then stops and removes the container.
|
||||
|
||||
.EXAMPLE
|
||||
cd backend
|
||||
.\scripts\run_integration_tests.ps1
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$ContainerName = 'chore-db-integration-test',
|
||||
[int]$HostPort = 27017,
|
||||
[string]$Image = 'mongo:8',
|
||||
[string]$DbName = 'chore_db_test',
|
||||
[string]$TestPath = 'tests/test_mongo_adapter.py'
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$mongoUri = "mongodb://localhost:${HostPort}/${DbName}"
|
||||
|
||||
function Test-ContainerRunning {
|
||||
$containers = docker ps --filter "name=$ContainerName" --format '{{.Names}}' 2>$null
|
||||
return $containers -contains $ContainerName
|
||||
}
|
||||
|
||||
function Wait-MongoReady {
|
||||
param([int]$TimeoutSeconds = 30)
|
||||
$start = Get-Date
|
||||
while (((Get-Date) - $start).TotalSeconds -lt $TimeoutSeconds) {
|
||||
try {
|
||||
$null = docker exec $ContainerName mongosh --eval 'db.adminCommand({ ping: 1 })' --quiet 2>$null
|
||||
if ($LASTEXITCODE -eq 0) {
|
||||
return
|
||||
}
|
||||
} catch {
|
||||
# Container or mongosh may not be ready yet.
|
||||
}
|
||||
Start-Sleep -Seconds 1
|
||||
}
|
||||
throw "MongoDB container did not become ready within ${TimeoutSeconds} seconds."
|
||||
}
|
||||
|
||||
# Clean up any leftover container from a previous aborted run.
|
||||
if (Test-ContainerRunning) {
|
||||
Write-Host "Removing existing container '$ContainerName'..."
|
||||
docker rm -f $ContainerName | Out-Null
|
||||
}
|
||||
|
||||
Write-Host "Starting MongoDB container '$ContainerName' on port $HostPort..."
|
||||
docker run -d `
|
||||
--name $ContainerName `
|
||||
-p "${HostPort}:27017" `
|
||||
$Image | Out-Null
|
||||
|
||||
try {
|
||||
Wait-MongoReady
|
||||
Write-Host "MongoDB is ready. Running integration tests..."
|
||||
|
||||
$env:USE_MONGODB = 'true'
|
||||
$env:MONGO_URI = $mongoUri
|
||||
$env:MONGO_DB_NAME = $DbName
|
||||
$env:DB_ENV = 'test'
|
||||
$env:DATA_ENV = 'test'
|
||||
|
||||
pytest $TestPath
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "Integration tests failed with exit code $LASTEXITCODE."
|
||||
}
|
||||
} finally {
|
||||
Write-Host "Stopping and removing container '$ContainerName'..."
|
||||
docker rm -f $ContainerName | Out-Null
|
||||
}
|
||||
|
||||
Write-Host "Integration tests complete."
|
||||
Reference in New Issue
Block a user