81 lines
3.0 KiB
Python
81 lines
3.0 KiB
Python
from django.conf import settings
|
|
from django.db import models
|
|
|
|
|
|
class IngestionRun(models.Model):
|
|
class RunStatus(models.TextChoices):
|
|
PENDING = "pending", "Pending"
|
|
RUNNING = "running", "Running"
|
|
SUCCESS = "success", "Success"
|
|
FAILED = "failed", "Failed"
|
|
CANCELED = "canceled", "Canceled"
|
|
|
|
class JobType(models.TextChoices):
|
|
FULL_SYNC = "full_sync", "Full Sync"
|
|
INCREMENTAL = "incremental", "Incremental"
|
|
MANUAL = "manual", "Manual"
|
|
|
|
provider_namespace = models.CharField(max_length=80)
|
|
job_type = models.CharField(max_length=32, choices=JobType.choices)
|
|
status = models.CharField(max_length=24, choices=RunStatus.choices, default=RunStatus.PENDING)
|
|
triggered_by = models.ForeignKey(
|
|
settings.AUTH_USER_MODEL,
|
|
on_delete=models.SET_NULL,
|
|
blank=True,
|
|
null=True,
|
|
related_name="ingestion_runs",
|
|
)
|
|
started_at = models.DateTimeField(blank=True, null=True)
|
|
finished_at = models.DateTimeField(blank=True, null=True)
|
|
records_processed = models.PositiveIntegerField(default=0)
|
|
records_created = models.PositiveIntegerField(default=0)
|
|
records_updated = models.PositiveIntegerField(default=0)
|
|
records_failed = models.PositiveIntegerField(default=0)
|
|
error_summary = models.TextField(blank=True, default="")
|
|
context = models.JSONField(default=dict, blank=True)
|
|
raw_payload = models.JSONField(default=dict, blank=True)
|
|
created_at = models.DateTimeField(auto_now_add=True)
|
|
|
|
class Meta:
|
|
ordering = ["-created_at"]
|
|
indexes = [
|
|
models.Index(fields=["provider_namespace", "status"]),
|
|
models.Index(fields=["provider_namespace", "job_type"]),
|
|
models.Index(fields=["started_at"]),
|
|
models.Index(fields=["finished_at"]),
|
|
]
|
|
|
|
def __str__(self) -> str:
|
|
return f"{self.provider_namespace} | {self.job_type} | {self.status}"
|
|
|
|
|
|
class IngestionError(models.Model):
|
|
class Severity(models.TextChoices):
|
|
WARNING = "warning", "Warning"
|
|
ERROR = "error", "Error"
|
|
CRITICAL = "critical", "Critical"
|
|
|
|
ingestion_run = models.ForeignKey(
|
|
"ingestion.IngestionRun",
|
|
on_delete=models.CASCADE,
|
|
related_name="errors",
|
|
)
|
|
provider_namespace = models.CharField(max_length=80)
|
|
entity_type = models.CharField(max_length=80, blank=True)
|
|
external_id = models.CharField(max_length=160, blank=True)
|
|
severity = models.CharField(max_length=16, choices=Severity.choices, default=Severity.ERROR)
|
|
message = models.TextField()
|
|
raw_payload = models.JSONField(default=dict, blank=True)
|
|
occurred_at = models.DateTimeField(auto_now_add=True)
|
|
|
|
class Meta:
|
|
ordering = ["-occurred_at"]
|
|
indexes = [
|
|
models.Index(fields=["ingestion_run", "occurred_at"]),
|
|
models.Index(fields=["provider_namespace", "entity_type", "external_id"]),
|
|
models.Index(fields=["severity"]),
|
|
]
|
|
|
|
def __str__(self) -> str:
|
|
return f"{self.provider_namespace}:{self.entity_type} [{self.severity}]"
|