Improve ingestion concurrency safety and batch transaction robustness

tests/test_ingestion_tasks.py

@@ -1,8 +1,11 @@
 import pytest
+from contextlib import contextmanager
 from celery.schedules import crontab
-from django.utils import timezone
+import psycopg
+from django.conf import settings
 
 from apps.ingestion.models import IngestionRun
+from apps.ingestion.services.runs import _build_ingestion_lock_key, release_ingestion_lock, try_acquire_ingestion_lock
 from apps.ingestion.tasks import scheduled_provider_sync, trigger_incremental_sync
 from config.celery import app as celery_app, build_periodic_schedule
 
@@ -44,23 +47,51 @@ def test_build_periodic_schedule_invalid_cron_disables_task_and_logs(settings, c
 
 
 @pytest.mark.django_db
-def test_trigger_incremental_sync_skips_overlapping_runs(settings):
+def test_trigger_incremental_sync_skips_when_advisory_lock_not_acquired(settings, monkeypatch):
     settings.INGESTION_PREVENT_OVERLAP = True
-    settings.INGESTION_OVERLAP_WINDOW_MINUTES = 180
 
-    IngestionRun.objects.create(
-        provider_namespace="mvp_demo",
-        job_type=IngestionRun.JobType.INCREMENTAL,
-        status=IngestionRun.RunStatus.RUNNING,
-        started_at=timezone.now(),
-    )
+    @contextmanager
+    def fake_lock(**kwargs):
+        yield False
 
+    monkeypatch.setattr("apps.ingestion.tasks.ingestion_advisory_lock", fake_lock)
     run_id = trigger_incremental_sync.apply(
         kwargs={"provider_namespace": "mvp_demo"},
     ).get()
     skipped_run = IngestionRun.objects.get(id=run_id)
     assert skipped_run.status == IngestionRun.RunStatus.CANCELED
-    assert "overlapping running job" in skipped_run.error_summary
+    assert "advisory lock" in skipped_run.error_summary
+
+
+@pytest.mark.django_db
+def test_advisory_lock_prevents_concurrent_acquisition():
+    provider_namespace = "mvp_demo"
+    job_type = IngestionRun.JobType.INCREMENTAL
+    lock_key = _build_ingestion_lock_key(provider_namespace=provider_namespace, job_type=job_type)
+
+    conninfo = (
+        f"dbname={settings.DATABASES['default']['NAME']} "
+        f"user={settings.DATABASES['default']['USER']} "
+        f"password={settings.DATABASES['default']['PASSWORD']} "
+        f"host={settings.DATABASES['default']['HOST']} "
+        f"port={settings.DATABASES['default']['PORT']}"
+    )
+    with psycopg.connect(conninfo) as external_conn:
+        with external_conn.cursor() as cursor:
+            cursor.execute("SELECT pg_advisory_lock(%s);", [lock_key])
+            acquired, _ = try_acquire_ingestion_lock(
+                provider_namespace=provider_namespace,
+                job_type=job_type,
+            )
+            assert acquired is False
+            cursor.execute("SELECT pg_advisory_unlock(%s);", [lock_key])
+
+    acquired, django_key = try_acquire_ingestion_lock(
+        provider_namespace=provider_namespace,
+        job_type=job_type,
+    )
+    assert acquired is True
+    release_ingestion_lock(lock_key=django_key)
 
 
 @pytest.mark.django_db