phase6: add provider abstraction, mvp adapter, and ingestion sync tasks

This commit is contained in:
Alfredo Di Stasio
2026-03-10 11:05:57 +01:00
parent f207ffbad8
commit ecd665e872
12 changed files with 1006 additions and 1 deletions

38
apps/ingestion/tasks.py Normal file
View File

@ -0,0 +1,38 @@
from celery import shared_task
from apps.ingestion.models import IngestionRun
from apps.ingestion.services.sync import run_sync_job
from apps.providers.exceptions import ProviderRateLimitError, ProviderTransientError
@shared_task(
bind=True,
autoretry_for=(ProviderRateLimitError, ProviderTransientError),
retry_backoff=True,
retry_jitter=True,
retry_kwargs={"max_retries": 5},
)
def trigger_full_sync(self, provider_namespace: str, triggered_by_id: int | None = None, context: dict | None = None):
return run_sync_job(
provider_namespace=provider_namespace,
job_type=IngestionRun.JobType.FULL_SYNC,
triggered_by_id=triggered_by_id,
context=context or {},
).id
@shared_task(
bind=True,
autoretry_for=(ProviderRateLimitError, ProviderTransientError),
retry_backoff=True,
retry_jitter=True,
retry_kwargs={"max_retries": 5},
)
def trigger_incremental_sync(self, provider_namespace: str, cursor: str | None = None, triggered_by_id: int | None = None, context: dict | None = None):
return run_sync_job(
provider_namespace=provider_namespace,
job_type=IngestionRun.JobType.INCREMENTAL,
triggered_by_id=triggered_by_id,
context=context or {},
cursor=cursor,
).id