phase6: add provider abstraction, mvp adapter, and ingestion sync tasks
This commit is contained in:
38
apps/ingestion/tasks.py
Normal file
38
apps/ingestion/tasks.py
Normal file
@ -0,0 +1,38 @@
|
||||
from celery import shared_task
|
||||
|
||||
from apps.ingestion.models import IngestionRun
|
||||
from apps.ingestion.services.sync import run_sync_job
|
||||
from apps.providers.exceptions import ProviderRateLimitError, ProviderTransientError
|
||||
|
||||
|
||||
@shared_task(
|
||||
bind=True,
|
||||
autoretry_for=(ProviderRateLimitError, ProviderTransientError),
|
||||
retry_backoff=True,
|
||||
retry_jitter=True,
|
||||
retry_kwargs={"max_retries": 5},
|
||||
)
|
||||
def trigger_full_sync(self, provider_namespace: str, triggered_by_id: int | None = None, context: dict | None = None):
|
||||
return run_sync_job(
|
||||
provider_namespace=provider_namespace,
|
||||
job_type=IngestionRun.JobType.FULL_SYNC,
|
||||
triggered_by_id=triggered_by_id,
|
||||
context=context or {},
|
||||
).id
|
||||
|
||||
|
||||
@shared_task(
|
||||
bind=True,
|
||||
autoretry_for=(ProviderRateLimitError, ProviderTransientError),
|
||||
retry_backoff=True,
|
||||
retry_jitter=True,
|
||||
retry_kwargs={"max_retries": 5},
|
||||
)
|
||||
def trigger_incremental_sync(self, provider_namespace: str, cursor: str | None = None, triggered_by_id: int | None = None, context: dict | None = None):
|
||||
return run_sync_job(
|
||||
provider_namespace=provider_namespace,
|
||||
job_type=IngestionRun.JobType.INCREMENTAL,
|
||||
triggered_by_id=triggered_by_id,
|
||||
context=context or {},
|
||||
cursor=cursor,
|
||||
).id
|
||||
Reference in New Issue
Block a user