feat: migrate backend persistence from TinyDB to MongoDB Atlas

- Implement lazy MongoDB client initialization in backend/db/mongo_client.py. - Create Gunicorn configuration to ensure MongoDB client is initialized per worker. - Refactor database access layer to support MongoDB with a new MongoLockedTable adapter. - Add migration script to transfer existing TinyDB data to MongoDB, preserving idempotency. - Update tracking event handling to ensure deterministic ordering with a monotonic sequence. - Modify tests to use mongomock for MongoDB integration and ensure existing tests pass. - Add integration test script to run tests against a local MongoDB Docker container. - Document environment variables and migration process in specs/feat-database-migration.md.
2026-07-28 01:10:52 -04:00
parent c9c1fa18a2
commit da7ed41938
17 changed files with 1364 additions and 62 deletions
@@ -1,5 +1,7 @@
 import os
 os.environ['DB_ENV'] = 'test'
+os.environ['USE_MONGODB'] = 'true'
+os.environ['MONGO_URI'] = 'mongomock'
 os.environ.setdefault('SECRET_KEY', 'test-secret-key')
 os.environ.setdefault('REFRESH_TOKEN_EXPIRY_DAYS', '90')
 os.environ.setdefault('DIGEST_TOKEN_SECRET', 'test-digest-secret')
@@ -18,5 +20,11 @@ TEST_REFRESH_TOKEN_EXPIRY_DAYS = 90
@pytest.fixture(scope="session", autouse=True)
 def set_test_db_env():
    os.environ['DB_ENV'] = 'test'
+    os.environ['USE_MONGODB'] = 'true'
+    os.environ['MONGO_URI'] = 'mongomock'
    os.environ['SECRET_KEY'] = TEST_SECRET_KEY
-    os.environ['REFRESH_TOKEN_EXPIRY_DAYS'] = str(TEST_REFRESH_TOKEN_EXPIRY_DAYS)
+    os.environ['REFRESH_TOKEN_EXPIRY_DAYS'] = str(TEST_REFRESH_TOKEN_EXPIRY_DAYS)
+    # Ensure indexes are created once for the test session.  This is safe to
+    # call repeatedly because MongoDB treats index creation as idempotent.
+    from db.db import ensure_mongodb_indexes
+    ensure_mongodb_indexes()
@@ -0,0 +1,201 @@
+import os
+import pytest
+from tinydb import Query
+
+from db.db import (
+    MongoLockedTable,
+    _query_to_mongo_filter,
+    child_db,
+    task_db,
+    users_db,
+    refresh_tokens_db,
+)
+from db.mongo_client import get_mongo_client, get_mongo_db_name
+
+
+# All tests in this module require the mongomock-backed MongoDB adapter.
+pytestmark = pytest.mark.skipif(
+    os.environ.get('USE_MONGODB', 'false').lower() != 'true',
+    reason='MongoDB adapter tests require USE_MONGODB=true',
+)
+
+
+@pytest.fixture(autouse=True)
+def clean_mongo_collections():
+    """Truncate relevant collections before each test."""
+    child_db.truncate()
+    task_db.truncate()
+    users_db.truncate()
+    refresh_tokens_db.truncate()
+    yield
+
+
+class TestQueryTranslation:
+    """Unit tests for TinyDB Query -> MongoDB filter translation."""
+
+    def test_simple_equality(self):
+        q = Query()
+        assert _query_to_mongo_filter(q.id == 'abc') == {'_id': {'$eq': 'abc'}}
+
+    def test_field_other_than_id(self):
+        q = Query()
+        assert _query_to_mongo_filter(q.user_id == 'u1') == {
+            'user_id': {'$eq': 'u1'}
+        }
+
+    def test_and_query(self):
+        q = Query()
+        mongo_filter = _query_to_mongo_filter(
+            (q.id == 'abc') & (q.user_id == 'u1')
+        )
+        assert mongo_filter == {'_id': {'$eq': 'abc'}, 'user_id': {'$eq': 'u1'}}
+
+    def test_or_query(self):
+        q = Query()
+        mongo_filter = _query_to_mongo_filter(
+            (q.user_id == 'u1') | (q.user_id == None)  # noqa: E711
+        )
+        assert mongo_filter == {
+            '$or': [
+                {'user_id': {'$eq': 'u1'}},
+                {'user_id': {'$eq': None}},
+            ]
+        }
+
+    def test_and_with_nested_or(self):
+        q = Query()
+        mongo_filter = _query_to_mongo_filter(
+            (q.id == 'abc') & ((q.user_id == 'u1') | (q.user_id == None))  # noqa: E711
+        )
+        assert mongo_filter == {
+            '_id': {'$eq': 'abc'},
+            '$or': [
+                {'user_id': {'$eq': 'u1'}},
+                {'user_id': {'$eq': None}},
+            ],
+        }
+
+
+class TestMongoCrud:
+    """CRUD tests against the mongomock-backed MongoLockedTable."""
+
+    def test_insert_maps_id_to_underscore_id(self):
+        child_db.insert({'id': 'c1', 'name': 'Alice', 'age': 8})
+
+        raw = get_mongo_client()[get_mongo_db_name()]['children'].find_one(
+            {'_id': 'c1'}
+        )
+        assert raw is not None
+        assert raw['_id'] == 'c1'
+        assert 'id' not in raw
+        assert raw['name'] == 'Alice'
+
+    def test_get_returns_document_without_underscore_id(self):
+        child_db.insert({'id': 'c1', 'name': 'Alice', 'age': 8})
+
+        doc = child_db.get(Query().id == 'c1')
+        assert doc is not None
+        assert doc['id'] == 'c1'
+        assert doc['name'] == 'Alice'
+        assert '_id' not in doc
+
+    def test_get_none_when_missing(self):
+        assert child_db.get(Query().id == 'missing') is None
+
+    def test_search_with_query(self):
+        child_db.insert({'id': 'c1', 'name': 'Alice', 'user_id': 'u1'})
+        child_db.insert({'id': 'c2', 'name': 'Bob', 'user_id': 'u1'})
+        child_db.insert({'id': 'c3', 'name': 'Carol', 'user_id': 'u2'})
+
+        results = child_db.search(Query().user_id == 'u1')
+        assert len(results) == 2
+        assert {r['id'] for r in results} == {'c1', 'c2'}
+
+    def test_search_with_or(self):
+        task_db.insert({'id': 't1', 'name': 'Default', 'user_id': None})
+        task_db.insert({'id': 't2', 'name': 'User task', 'user_id': 'u1'})
+
+        q = Query()
+        results = task_db.search((q.user_id == 'u1') | (q.user_id == None))  # noqa: E711
+        assert len(results) == 2
+
+    def test_update_modifies_matching_documents(self):
+        child_db.insert({'id': 'c1', 'name': 'Alice', 'points': 0})
+        child_db.insert({'id': 'c2', 'name': 'Bob', 'points': 0})
+
+        modified = child_db.update({'points': 10}, Query().id == 'c1')
+        # TinyDB returns a list of updated document ids; the adapter mirrors that.
+        assert modified == ['c1']
+
+        doc = child_db.get(Query().id == 'c1')
+        assert doc['points'] == 10
+
+        other = child_db.get(Query().id == 'c2')
+        assert other['points'] == 0
+
+    def test_update_does_not_overwrite_id(self):
+        child_db.insert({'id': 'c1', 'name': 'Alice'})
+
+        child_db.update({'id': 'c2', 'name': 'Alice Smith'}, Query().id == 'c1')
+        # The id field must remain unchanged; update should have stripped id.
+        assert child_db.get(Query().id == 'c1')['name'] == 'Alice Smith'
+        assert child_db.get(Query().id == 'c2') is None
+
+    def test_remove_deletes_matching_documents(self):
+        child_db.insert({'id': 'c1', 'name': 'Alice'})
+        child_db.insert({'id': 'c2', 'name': 'Bob'})
+
+        deleted = child_db.remove(Query().id == 'c1')
+        # TinyDB returns a list of removed document ids; the adapter mirrors that.
+        assert deleted == ['c1']
+
+        assert child_db.get(Query().id == 'c1') is None
+        assert child_db.get(Query().id == 'c2') is not None
+
+    def test_all_returns_all_documents(self):
+        child_db.insert({'id': 'c1', 'name': 'Alice'})
+        child_db.insert({'id': 'c2', 'name': 'Bob'})
+
+        docs = child_db.all()
+        assert len(docs) == 2
+        assert all('_id' not in d for d in docs)
+
+    def test_truncate_removes_all_documents(self):
+        child_db.insert({'id': 'c1', 'name': 'Alice'})
+        child_db.truncate()
+        assert child_db.all() == []
+
+    def test_insert_multiple(self):
+        ids = child_db.insert_multiple([
+            {'id': 'c1', 'name': 'Alice'},
+            {'id': 'c2', 'name': 'Bob'},
+        ])
+        assert sorted(ids) == ['c1', 'c2']
+        assert len(child_db.all()) == 2
+
+    def test_unique_token_index(self):
+        refresh_tokens_db.insert({'id': 'r1', 'token': 'abc', 'user_id': 'u1'})
+        refresh_tokens_db.insert({'id': 'r2', 'token': 'def', 'user_id': 'u1'})
+
+        # mongomock does not enforce unique indexes by default, but we verify
+        # both records are readable.
+        assert refresh_tokens_db.get(Query().token == 'abc') is not None
+        assert refresh_tokens_db.get(Query().token == 'def') is not None
+
+    def test_user_id_secondary_index_is_created(self):
+        # Insert and query via the secondary index path used by the app.
+        users_db.insert({'id': 'u1', 'email': 'a@example.com', 'verified': True})
+        users_db.insert({'id': 'u2', 'email': 'b@example.com', 'verified': False})
+
+        found = users_db.search(Query().verified == True)  # noqa: E712
+        assert len(found) == 1
+        assert found[0]['id'] == 'u1'
+
+
+class TestAdapterApi:
+    """Tests that the adapter exposes the expected LockedTable-compatible API."""
+
+    def test_close_is_noop(self):
+        # Existing cleanup fixtures call ``*_db.close()``; ensure it does not
+        # raise for the MongoDB-backed adapter.
+        child_db.close()