from django.conf import settings from django.db import models class IngestionRun(models.Model): class RunStatus(models.TextChoices): PENDING = "pending", "Pending" RUNNING = "running", "Running" SUCCESS = "success", "Success" FAILED = "failed", "Failed" CANCELED = "canceled", "Canceled" class JobType(models.TextChoices): FULL_SYNC = "full_sync", "Full Sync" INCREMENTAL = "incremental", "Incremental" MANUAL = "manual", "Manual" provider_namespace = models.CharField(max_length=80) job_type = models.CharField(max_length=32, choices=JobType.choices) status = models.CharField(max_length=24, choices=RunStatus.choices, default=RunStatus.PENDING) triggered_by = models.ForeignKey( settings.AUTH_USER_MODEL, on_delete=models.SET_NULL, blank=True, null=True, related_name="ingestion_runs", ) started_at = models.DateTimeField(blank=True, null=True) finished_at = models.DateTimeField(blank=True, null=True) records_processed = models.PositiveIntegerField(default=0) records_created = models.PositiveIntegerField(default=0) records_updated = models.PositiveIntegerField(default=0) records_failed = models.PositiveIntegerField(default=0) error_summary = models.TextField(blank=True, default="") context = models.JSONField(default=dict, blank=True) raw_payload = models.JSONField(default=dict, blank=True) created_at = models.DateTimeField(auto_now_add=True) class Meta: ordering = ["-created_at"] indexes = [ models.Index(fields=["provider_namespace", "status"]), models.Index(fields=["provider_namespace", "job_type"]), models.Index(fields=["started_at"]), models.Index(fields=["finished_at"]), ] def __str__(self) -> str: return f"{self.provider_namespace} | {self.job_type} | {self.status}" class IngestionError(models.Model): class Severity(models.TextChoices): WARNING = "warning", "Warning" ERROR = "error", "Error" CRITICAL = "critical", "Critical" ingestion_run = models.ForeignKey( "ingestion.IngestionRun", on_delete=models.CASCADE, related_name="errors", ) provider_namespace = models.CharField(max_length=80) entity_type = models.CharField(max_length=80, blank=True) external_id = models.CharField(max_length=160, blank=True) severity = models.CharField(max_length=16, choices=Severity.choices, default=Severity.ERROR) message = models.TextField() raw_payload = models.JSONField(default=dict, blank=True) occurred_at = models.DateTimeField(auto_now_add=True) class Meta: ordering = ["-occurred_at"] indexes = [ models.Index(fields=["ingestion_run", "occurred_at"]), models.Index(fields=["provider_namespace", "entity_type", "external_id"]), models.Index(fields=["severity"]), ] def __str__(self) -> str: return f"{self.provider_namespace}:{self.entity_type} [{self.severity}]"