Wire Celery Beat periodic sync with ingestion run tracking
This commit is contained in:
@ -67,6 +67,10 @@ def test_incremental_sync_runs_successfully(settings):
|
||||
|
||||
assert run.status == IngestionRun.RunStatus.SUCCESS
|
||||
assert run.records_processed > 0
|
||||
assert run.started_at is not None
|
||||
assert run.finished_at is not None
|
||||
assert run.finished_at >= run.started_at
|
||||
assert run.error_summary == ""
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@ -80,6 +84,9 @@ def test_run_sync_handles_rate_limit(settings):
|
||||
run = IngestionRun.objects.order_by("-id").first()
|
||||
assert run is not None
|
||||
assert run.status == IngestionRun.RunStatus.FAILED
|
||||
assert run.started_at is not None
|
||||
assert run.finished_at is not None
|
||||
assert "Rate limit" in run.error_summary
|
||||
assert IngestionError.objects.filter(ingestion_run=run).exists()
|
||||
|
||||
os.environ.pop("PROVIDER_MVP_FORCE_RATE_LIMIT", None)
|
||||
|
||||
69
tests/test_ingestion_tasks.py
Normal file
69
tests/test_ingestion_tasks.py
Normal file
@ -0,0 +1,69 @@
|
||||
import pytest
|
||||
from celery.schedules import crontab
|
||||
from django.utils import timezone
|
||||
|
||||
from apps.ingestion.models import IngestionRun
|
||||
from apps.ingestion.tasks import scheduled_provider_sync, trigger_incremental_sync
|
||||
from config.celery import app as celery_app, build_periodic_schedule
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_periodic_task_registered():
|
||||
assert "apps.ingestion.tasks.scheduled_provider_sync" in celery_app.tasks
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_build_periodic_schedule_enabled(settings):
|
||||
settings.INGESTION_SCHEDULE_ENABLED = True
|
||||
settings.INGESTION_SCHEDULE_CRON = "15 * * * *"
|
||||
|
||||
schedule = build_periodic_schedule()
|
||||
assert "ingestion.scheduled_provider_sync" in schedule
|
||||
entry = schedule["ingestion.scheduled_provider_sync"]
|
||||
assert entry["task"] == "apps.ingestion.tasks.scheduled_provider_sync"
|
||||
assert isinstance(entry["schedule"], crontab)
|
||||
assert entry["schedule"]._orig_minute == "15"
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_build_periodic_schedule_disabled(settings):
|
||||
settings.INGESTION_SCHEDULE_ENABLED = False
|
||||
assert build_periodic_schedule() == {}
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_trigger_incremental_sync_skips_overlapping_runs(settings):
|
||||
settings.INGESTION_PREVENT_OVERLAP = True
|
||||
settings.INGESTION_OVERLAP_WINDOW_MINUTES = 180
|
||||
|
||||
IngestionRun.objects.create(
|
||||
provider_namespace="mvp_demo",
|
||||
job_type=IngestionRun.JobType.INCREMENTAL,
|
||||
status=IngestionRun.RunStatus.RUNNING,
|
||||
started_at=timezone.now(),
|
||||
)
|
||||
|
||||
run_id = trigger_incremental_sync.apply(
|
||||
kwargs={"provider_namespace": "mvp_demo"},
|
||||
).get()
|
||||
skipped_run = IngestionRun.objects.get(id=run_id)
|
||||
assert skipped_run.status == IngestionRun.RunStatus.CANCELED
|
||||
assert "overlapping running job" in skipped_run.error_summary
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_scheduled_provider_sync_uses_configured_job_type(settings, monkeypatch):
|
||||
settings.INGESTION_SCHEDULE_JOB_TYPE = IngestionRun.JobType.FULL_SYNC
|
||||
settings.INGESTION_SCHEDULE_PROVIDER_NAMESPACE = "mvp_demo"
|
||||
captured = {}
|
||||
|
||||
def fake_runner(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return 99
|
||||
|
||||
monkeypatch.setattr("apps.ingestion.tasks._run_sync_with_overlap_guard", fake_runner)
|
||||
|
||||
result = scheduled_provider_sync.apply().get()
|
||||
assert result == 99
|
||||
assert captured["provider_namespace"] == "mvp_demo"
|
||||
assert captured["job_type"] == IngestionRun.JobType.FULL_SYNC
|
||||
Reference in New Issue
Block a user