bisco/hoopscout
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add v2 relational domain foundations with import run/file models

Browse Source
This commit is contained in:
Alfredo Di Stasio
2026-03-13 13:54:29 +01:00
parent bb033222e3
commit 6aa66807e9
16 changed files with 429 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
apps/competitions/admin.py
View File

@ -5,16 +5,16 @@ from .models import Competition, Season, TeamSeason
@admin.register(Competition)
class CompetitionAdmin(admin.ModelAdmin):
list_display = ("name", "competition_type", "gender", "country", "is_active")
list_display = ("name", "source_uid", "competition_type", "gender", "country", "is_active")
list_filter = ("competition_type", "gender", "country", "is_active")
search_fields = ("name", "slug")
search_fields = ("name", "slug", "source_uid")
@admin.register(Season)
class SeasonAdmin(admin.ModelAdmin):
list_display = ("label", "start_date", "end_date", "is_current")
list_display = ("label", "source_uid", "start_date", "end_date", "is_current")
list_filter = ("is_current",)
search_fields = ("label",)
search_fields = ("label", "source_uid")
@admin.register(TeamSeason)

32
apps/competitions/migrations/0003_competition_source_uid_season_source_uid_and_more.py Normal file
View File

@ -0,0 +1,32 @@
# Generated by Django 5.2.12 on 2026-03-13 12:44
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('competitions', '0002_initial'),
('players', '0005_player_weight_index'),
]
operations = [
migrations.AddField(
model_name='competition',
name='source_uid',
field=models.CharField(blank=True, max_length=120, null=True, unique=True),
),
migrations.AddField(
model_name='season',
name='source_uid',
field=models.CharField(blank=True, max_length=120, null=True, unique=True),
),
migrations.AddIndex(
model_name='competition',
index=models.Index(fields=['source_uid'], name='competition_source__1c043a_idx'),
),
migrations.AddIndex(
model_name='season',
index=models.Index(fields=['source_uid'], name='competition_source__41e6a6_idx'),
),
]

4
apps/competitions/models.py
View File

@ -14,6 +14,7 @@ class Competition(models.Model):
name = models.CharField(max_length=220)
slug = models.SlugField(max_length=240, unique=True)
source_uid = models.CharField(max_length=120, blank=True, null=True, unique=True)
competition_type = models.CharField(max_length=24, choices=CompetitionType.choices)
gender = models.CharField(max_length=16, choices=Gender.choices, default=Gender.MEN)
level = models.PositiveSmallIntegerField(default=1)
@ -35,6 +36,7 @@ class Competition(models.Model):
]
indexes = [
models.Index(fields=["name"]),
models.Index(fields=["source_uid"]),
models.Index(fields=["country"]),
models.Index(fields=["competition_type"]),
models.Index(fields=["gender"]),
@ -46,6 +48,7 @@ class Competition(models.Model):
class Season(models.Model):
source_uid = models.CharField(max_length=120, blank=True, null=True, unique=True)
label = models.CharField(max_length=40, unique=True)
start_date = models.DateField()
end_date = models.DateField()
@ -57,6 +60,7 @@ class Season(models.Model):
models.CheckConstraint(condition=models.Q(end_date__gte=models.F("start_date")), name="ck_season_dates")
]
indexes = [
models.Index(fields=["source_uid"]),
models.Index(fields=["is_current"]),
models.Index(fields=["start_date"]),
models.Index(fields=["end_date"]),

162
apps/ingestion/admin.py
View File

@ -1,117 +1,99 @@
from django.contrib import admin
from django.contrib import messages
from django.db.models import Count
from apps.providers.registry import get_default_provider_namespace
from .models import IngestionError, IngestionRun
from .tasks import trigger_full_sync, trigger_incremental_sync
from .models import ImportFile, ImportRun, IngestionError, IngestionRun
class IngestionErrorInline(admin.TabularInline):
model = IngestionError
class ImportFileInline(admin.TabularInline):
model = ImportFile
extra = 0
readonly_fields = ("provider_namespace", "entity_type", "external_id", "severity", "message", "occurred_at")
@admin.register(IngestionRun)
class IngestionRunAdmin(admin.ModelAdmin):
list_display = (
"provider_namespace",
"job_type",
readonly_fields = (
"relative_path",
"status",
"records_processed",
"records_created",
"records_updated",
"records_failed",
"error_count",
"short_error_summary",
"checksum",
"file_size_bytes",
"rows_total",
"rows_upserted",
"rows_failed",
"error_message",
"processed_at",
"created_at",
)
@admin.register(ImportRun)
class ImportRunAdmin(admin.ModelAdmin):
list_display = (
"id",
"source",
"status",
"files_total",
"files_processed",
"rows_total",
"rows_upserted",
"rows_failed",
"started_at",
"finished_at",
"created_at",
)
list_filter = ("provider_namespace", "job_type", "status")
search_fields = ("provider_namespace",)
inlines = (IngestionErrorInline,)
list_filter = ("source", "status")
search_fields = ("source", "error_summary")
readonly_fields = (
"provider_namespace",
"job_type",
"source",
"status",
"triggered_by",
"started_at",
"finished_at",
"records_processed",
"records_created",
"records_updated",
"records_failed",
"files_total",
"files_processed",
"rows_total",
"rows_upserted",
"rows_failed",
"error_summary",
"context",
"raw_payload",
"created_at",
)
actions = (
"enqueue_full_sync_default_provider",
"enqueue_incremental_sync_default_provider",
"retry_selected_runs",
inlines = (ImportFileInline,)
@admin.register(ImportFile)
class ImportFileAdmin(admin.ModelAdmin):
list_display = (
"id",
"import_run",
"relative_path",
"status",
"rows_total",
"rows_upserted",
"rows_failed",
"processed_at",
)
list_filter = ("status",)
search_fields = ("relative_path", "checksum", "error_message")
readonly_fields = (
"import_run",
"relative_path",
"status",
"checksum",
"file_size_bytes",
"rows_total",
"rows_upserted",
"rows_failed",
"error_message",
"payload_preview",
"processed_at",
"created_at",
)
@admin.action(description="Queue full sync (default provider)")
def enqueue_full_sync_default_provider(self, request, queryset):
provider_namespace = get_default_provider_namespace()
trigger_full_sync.delay(provider_namespace=provider_namespace, triggered_by_id=request.user.id)
self.message_user(request, f"Queued full sync task for {provider_namespace}.", level=messages.SUCCESS)
@admin.action(description="Queue incremental sync (default provider)")
def enqueue_incremental_sync_default_provider(self, request, queryset):
provider_namespace = get_default_provider_namespace()
trigger_incremental_sync.delay(provider_namespace=provider_namespace, triggered_by_id=request.user.id)
self.message_user(request, f"Queued incremental sync task for {provider_namespace}.", level=messages.SUCCESS)
@admin.action(description="Retry selected ingestion runs")
def retry_selected_runs(self, request, queryset):
count = 0
for run in queryset:
if run.job_type == IngestionRun.JobType.INCREMENTAL:
trigger_incremental_sync.delay(
provider_namespace=run.provider_namespace,
triggered_by_id=request.user.id,
context={"retry_of": run.id},
)
else:
trigger_full_sync.delay(
provider_namespace=run.provider_namespace,
triggered_by_id=request.user.id,
context={"retry_of": run.id},
)
count += 1
self.message_user(request, f"Queued {count} retry task(s).", level=messages.SUCCESS)
def get_queryset(self, request):
queryset = super().get_queryset(request)
return queryset.annotate(_error_count=Count("errors"))
@admin.display(ordering="_error_count", description="Errors")
def error_count(self, obj):
return getattr(obj, "_error_count", 0)
@admin.display(description="Error summary")
def short_error_summary(self, obj):
if not obj.error_summary:
return "-"
return (obj.error_summary[:90] + "...") if len(obj.error_summary) > 90 else obj.error_summary
@admin.register(IngestionRun)
class LegacyIngestionRunAdmin(admin.ModelAdmin):
list_display = ("provider_namespace", "job_type", "status", "started_at", "finished_at")
list_filter = ("provider_namespace", "job_type", "status")
search_fields = ("provider_namespace", "error_summary")
@admin.register(IngestionError)
class IngestionErrorAdmin(admin.ModelAdmin):
class LegacyIngestionErrorAdmin(admin.ModelAdmin):
list_display = ("provider_namespace", "entity_type", "external_id", "severity", "occurred_at")
list_filter = ("severity", "provider_namespace")
search_fields = ("entity_type", "external_id", "message")
readonly_fields = (
"ingestion_run",
"provider_namespace",
"entity_type",
"external_id",
"severity",
"message",
"raw_payload",
"occurred_at",
)

91
apps/ingestion/migrations/0003_importrun_importfile_and_more.py Normal file
View File

@ -0,0 +1,91 @@
# Generated by Django 5.2.12 on 2026-03-13 12:44
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('ingestion', '0002_ingestionrun_error_summary'),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name='ImportRun',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('source', models.CharField(default='snapshot', max_length=80)),
('status', models.CharField(choices=[('pending', 'Pending'), ('running', 'Running'), ('success', 'Success'), ('failed', 'Failed'), ('canceled', 'Canceled')], default='pending', max_length=24)),
('started_at', models.DateTimeField(blank=True, null=True)),
('finished_at', models.DateTimeField(blank=True, null=True)),
('files_total', models.PositiveIntegerField(default=0)),
('files_processed', models.PositiveIntegerField(default=0)),
('rows_total', models.PositiveIntegerField(default=0)),
('rows_upserted', models.PositiveIntegerField(default=0)),
('rows_failed', models.PositiveIntegerField(default=0)),
('error_summary', models.TextField(blank=True, default='')),
('context', models.JSONField(blank=True, default=dict)),
('created_at', models.DateTimeField(auto_now_add=True)),
('triggered_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='import_runs', to=settings.AUTH_USER_MODEL)),
],
options={
'ordering': ['-created_at'],
},
),
migrations.CreateModel(
name='ImportFile',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('relative_path', models.CharField(max_length=260)),
('status', models.CharField(choices=[('pending', 'Pending'), ('processing', 'Processing'), ('success', 'Success'), ('failed', 'Failed'), ('skipped', 'Skipped')], default='pending', max_length=24)),
('checksum', models.CharField(blank=True, max_length=128)),
('file_size_bytes', models.PositiveBigIntegerField(blank=True, null=True)),
('rows_total', models.PositiveIntegerField(default=0)),
('rows_upserted', models.PositiveIntegerField(default=0)),
('rows_failed', models.PositiveIntegerField(default=0)),
('error_message', models.TextField(blank=True)),
('payload_preview', models.JSONField(blank=True, default=dict)),
('processed_at', models.DateTimeField(blank=True, null=True)),
('created_at', models.DateTimeField(auto_now_add=True)),
('import_run', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='files', to='ingestion.importrun')),
],
options={
'ordering': ['-created_at'],
},
),
migrations.AddIndex(
model_name='importrun',
index=models.Index(fields=['source', 'status'], name='ingestion_i_source_61db63_idx'),
),
migrations.AddIndex(
model_name='importrun',
index=models.Index(fields=['created_at'], name='ingestion_i_created_93c115_idx'),
),
migrations.AddIndex(
model_name='importrun',
index=models.Index(fields=['started_at'], name='ingestion_i_started_bf1d94_idx'),
),
migrations.AddIndex(
model_name='importrun',
index=models.Index(fields=['finished_at'], name='ingestion_i_finishe_73cbed_idx'),
),
migrations.AddIndex(
model_name='importfile',
index=models.Index(fields=['import_run', 'status'], name='ingestion_i_import__075f75_idx'),
),
migrations.AddIndex(
model_name='importfile',
index=models.Index(fields=['relative_path'], name='ingestion_i_relativ_183e34_idx'),
),
migrations.AddIndex(
model_name='importfile',
index=models.Index(fields=['processed_at'], name='ingestion_i_process_dfc080_idx'),
),
migrations.AddConstraint(
model_name='importfile',
constraint=models.UniqueConstraint(fields=('import_run', 'relative_path'), name='uq_import_file_per_run_path'),
),
]

84
apps/ingestion/models.py
View File

@ -2,6 +2,90 @@ from django.conf import settings
from django.db import models
class ImportRun(models.Model):
class RunStatus(models.TextChoices):
PENDING = "pending", "Pending"
RUNNING = "running", "Running"
SUCCESS = "success", "Success"
FAILED = "failed", "Failed"
CANCELED = "canceled", "Canceled"
source = models.CharField(max_length=80, default="snapshot")
status = models.CharField(max_length=24, choices=RunStatus.choices, default=RunStatus.PENDING)
triggered_by = models.ForeignKey(
settings.AUTH_USER_MODEL,
on_delete=models.SET_NULL,
blank=True,
null=True,
related_name="import_runs",
)
started_at = models.DateTimeField(blank=True, null=True)
finished_at = models.DateTimeField(blank=True, null=True)
files_total = models.PositiveIntegerField(default=0)
files_processed = models.PositiveIntegerField(default=0)
rows_total = models.PositiveIntegerField(default=0)
rows_upserted = models.PositiveIntegerField(default=0)
rows_failed = models.PositiveIntegerField(default=0)
error_summary = models.TextField(blank=True, default="")
context = models.JSONField(default=dict, blank=True)
created_at = models.DateTimeField(auto_now_add=True)
class Meta:
ordering = ["-created_at"]
indexes = [
models.Index(fields=["source", "status"]),
models.Index(fields=["created_at"]),
models.Index(fields=["started_at"]),
models.Index(fields=["finished_at"]),
]
def __str__(self) -> str:
return f"{self.source} | {self.status} | {self.created_at:%Y-%m-%d %H:%M}"
class ImportFile(models.Model):
class FileStatus(models.TextChoices):
PENDING = "pending", "Pending"
PROCESSING = "processing", "Processing"
SUCCESS = "success", "Success"
FAILED = "failed", "Failed"
SKIPPED = "skipped", "Skipped"
import_run = models.ForeignKey(
"ingestion.ImportRun",
on_delete=models.CASCADE,
related_name="files",
)
relative_path = models.CharField(max_length=260)
status = models.CharField(max_length=24, choices=FileStatus.choices, default=FileStatus.PENDING)
checksum = models.CharField(max_length=128, blank=True)
file_size_bytes = models.PositiveBigIntegerField(blank=True, null=True)
rows_total = models.PositiveIntegerField(default=0)
rows_upserted = models.PositiveIntegerField(default=0)
rows_failed = models.PositiveIntegerField(default=0)
error_message = models.TextField(blank=True)
payload_preview = models.JSONField(default=dict, blank=True)
processed_at = models.DateTimeField(blank=True, null=True)
created_at = models.DateTimeField(auto_now_add=True)
class Meta:
ordering = ["-created_at"]
constraints = [
models.UniqueConstraint(
fields=["import_run", "relative_path"],
name="uq_import_file_per_run_path",
),
]
indexes = [
models.Index(fields=["import_run", "status"]),
models.Index(fields=["relative_path"]),
models.Index(fields=["processed_at"]),
]
def __str__(self) -> str:
return f"{self.relative_path} [{self.status}]"
class IngestionRun(models.Model):
class RunStatus(models.TextChoices):
PENDING = "pending", "Pending"

3
apps/players/admin.py
View File

@ -37,6 +37,7 @@ class PlayerCareerEntryInline(admin.TabularInline):
class PlayerAdmin(admin.ModelAdmin):
list_display = (
"full_name",
"source_uid",
"birth_date",
"nationality",
"nominal_position",
@ -53,7 +54,7 @@ class PlayerAdmin(admin.ModelAdmin):
"origin_competition",
"origin_team",
)
search_fields = ("full_name", "first_name", "last_name")
search_fields = ("full_name", "first_name", "last_name", "source_uid")
inlines = (PlayerAliasInline, PlayerCareerEntryInline)
actions = ("recompute_origin_fields",)

24
apps/players/migrations/0006_player_source_uid_and_more.py Normal file
View File

@ -0,0 +1,24 @@
# Generated by Django 5.2.12 on 2026-03-13 12:44
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('competitions', '0003_competition_source_uid_season_source_uid_and_more'),
('players', '0005_player_weight_index'),
('teams', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='player',
name='source_uid',
field=models.CharField(blank=True, max_length=120, null=True, unique=True),
),
migrations.AddIndex(
model_name='player',
index=models.Index(fields=['source_uid'], name='players_pla_source__93bb47_idx'),
),
]

2
apps/players/models.py
View File

@ -58,6 +58,7 @@ class Player(TimeStampedModel):
first_name = models.CharField(max_length=120)
last_name = models.CharField(max_length=120)
full_name = models.CharField(max_length=260)
source_uid = models.CharField(max_length=120, blank=True, null=True, unique=True)
birth_date = models.DateField(blank=True, null=True)
nationality = models.ForeignKey(
"players.Nationality",
@ -114,6 +115,7 @@ class Player(TimeStampedModel):
]
indexes = [
models.Index(fields=["full_name"]),
models.Index(fields=["source_uid"]),
models.Index(fields=["last_name", "first_name"]),
models.Index(fields=["birth_date"]),
models.Index(fields=["nationality"]),

4
apps/stats/admin.py
View File

@ -5,9 +5,9 @@ from .models import PlayerSeason, PlayerSeasonStats
@admin.register(PlayerSeason)
class PlayerSeasonAdmin(admin.ModelAdmin):
list_display = ("player", "season", "team", "competition", "games_played", "minutes_played")
list_display = ("player", "season", "source_uid", "team", "competition", "games_played", "minutes_played")
list_filter = ("season", "competition")
search_fields = ("player__full_name", "team__name", "competition__name", "season__label")
search_fields = ("player__full_name", "team__name", "competition__name", "season__label", "source_uid")
@admin.register(PlayerSeasonStats)

25
apps/stats/migrations/0003_playerseason_source_uid_and_more.py Normal file
View File

@ -0,0 +1,25 @@
# Generated by Django 5.2.12 on 2026-03-13 12:44
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('competitions', '0003_competition_source_uid_season_source_uid_and_more'),
('players', '0006_player_source_uid_and_more'),
('stats', '0002_playerseasonstats_search_indexes'),
('teams', '0002_team_source_uid_team_teams_team_source__940258_idx'),
]
operations = [
migrations.AddField(
model_name='playerseason',
name='source_uid',
field=models.CharField(blank=True, max_length=160, null=True, unique=True),
),
migrations.AddIndex(
model_name='playerseason',
index=models.Index(fields=['source_uid'], name='stats_playe_source__57b701_idx'),
),
]

2
apps/stats/models.py
View File

@ -4,6 +4,7 @@ from django.db import models
class PlayerSeason(models.Model):
player = models.ForeignKey("players.Player", on_delete=models.CASCADE, related_name="player_seasons")
season = models.ForeignKey("competitions.Season", on_delete=models.CASCADE, related_name="player_seasons")
source_uid = models.CharField(max_length=160, blank=True, null=True, unique=True)
team = models.ForeignKey(
"teams.Team",
on_delete=models.SET_NULL,
@ -31,6 +32,7 @@ class PlayerSeason(models.Model):
)
]
indexes = [
models.Index(fields=["source_uid"]),
models.Index(fields=["player", "season"]),
models.Index(fields=["season", "team"]),
models.Index(fields=["season", "competition"]),

4
apps/teams/admin.py
View File

@ -5,6 +5,6 @@ from .models import Team
@admin.register(Team)
class TeamAdmin(admin.ModelAdmin):
list_display = ("name", "short_name", "country", "is_national_team")
list_display = ("name", "source_uid", "short_name", "country", "is_national_team")
list_filter = ("is_national_team", "country")
search_fields = ("name", "short_name", "slug")
search_fields = ("name", "short_name", "slug", "source_uid")

23
apps/teams/migrations/0002_team_source_uid_team_teams_team_source__940258_idx.py Normal file
View File

@ -0,0 +1,23 @@
# Generated by Django 5.2.12 on 2026-03-13 12:44
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('players', '0006_player_source_uid_and_more'),
('teams', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='team',
name='source_uid',
field=models.CharField(blank=True, max_length=120, null=True, unique=True),
),
migrations.AddIndex(
model_name='team',
index=models.Index(fields=['source_uid'], name='teams_team_source__940258_idx'),
),
]

2
apps/teams/models.py
View File

@ -5,6 +5,7 @@ class Team(models.Model):
name = models.CharField(max_length=200)
short_name = models.CharField(max_length=80, blank=True)
slug = models.SlugField(max_length=220, unique=True)
source_uid = models.CharField(max_length=120, blank=True, null=True, unique=True)
country = models.ForeignKey(
"players.Nationality",
on_delete=models.SET_NULL,
@ -25,6 +26,7 @@ class Team(models.Model):
indexes = [
models.Index(fields=["name"]),
models.Index(fields=["slug"]),
models.Index(fields=["source_uid"]),
models.Index(fields=["country"]),
models.Index(fields=["is_national_team"]),
]