Merge branch 'feature/fix-invalid-datetime-sorting' into develop

app/services/processing.py

@@ -47,7 +47,7 @@ def sort_records(
     reverse = options.order == "desc"
 
     if options.sort_by == "datetime":
-        key_func = _datetime_key
+        return _sort_records_by_datetime(records, reverse)
     elif options.sort_by == "severity":
         key_func = _severity_key
     else:
@@ -56,17 +56,34 @@ def sort_records(
     return sorted(records, key=key_func, reverse=reverse)
 
 
-def _datetime_key(record: dict[str, str]) -> tuple[int, datetime]:
+def _sort_records_by_datetime(
+    records: Iterable[dict[str, str]], reverse: bool
+) -> list[dict[str, str]]:
+    """Sort valid datetimes normally and always place invalid/missing values last."""
+    valid_records: list[tuple[datetime, dict[str, str]]] = []
+    invalid_records: list[dict[str, str]] = []
+
+    for record in records:
+        parsed_datetime = _parse_datetime(record)
+        if parsed_datetime is None:
+            invalid_records.append(record)
+            continue
+        valid_records.append((parsed_datetime, record))
+
+    sorted_valid_records = sorted(valid_records, key=lambda item: item[0], reverse=reverse)
+    return [record for _parsed, record in sorted_valid_records] + invalid_records
+
+
+def _parse_datetime(record: dict[str, str]) -> datetime | None:
     date_value = record.get("v015xxxxdate", "").strip()
     time_value = record.get("time", "").strip()
     if not date_value or not time_value:
-        return (1, datetime.max)
+        return None
 
     try:
-        parsed = datetime.strptime(f"{date_value} {time_value}", "%Y-%m-%d %H:%M:%S")
+        return datetime.strptime(f"{date_value} {time_value}", "%Y-%m-%d %H:%M:%S")
     except ValueError:
-        return (1, datetime.max)
+        return None
-    return (0, parsed)
 
 
 def _severity_key(record: dict[str, str]) -> tuple[int, str]:

tests/test_processing.py

@@ -44,3 +44,65 @@ def test_sort_records_by_severity_desc_uses_defined_ranking():
         "medium",
         "info",
     ]
+
+
+def test_sort_records_by_datetime_asc_places_invalid_records_last():
+    records = [
+        {"v015xxxxdate": "2024-05-03", "time": "08:00:00", "msg": "latest-valid"},
+        {"v015xxxxdate": "", "time": "09:00:00", "msg": "missing-date"},
+        {"v015xxxxdate": "2024-05-01", "time": "10:00:00", "msg": "earliest-valid"},
+        {"v015xxxxdate": "2024-05-02", "time": "", "msg": "missing-time"},
+        {"v015xxxxdate": "bad-date", "time": "99:99:99", "msg": "invalid-datetime"},
+        {"v015xxxxdate": "2024-05-02", "time": "09:30:00", "msg": "middle-valid"},
+    ]
+    options = ProcessingOptions(
+        policy_cs="",
+        policy_ci="",
+        severity_cs="",
+        severity_ci="",
+        sort_by="datetime",
+        order="asc",
+        mode="vendor",
+    )
+
+    sorted_records = sort_records(records, options)
+
+    assert [record["msg"] for record in sorted_records] == [
+        "earliest-valid",
+        "middle-valid",
+        "latest-valid",
+        "missing-date",
+        "missing-time",
+        "invalid-datetime",
+    ]
+
+
+def test_sort_records_by_datetime_desc_places_invalid_records_last():
+    records = [
+        {"v015xxxxdate": "2024-05-03", "time": "08:00:00", "msg": "latest-valid"},
+        {"v015xxxxdate": "", "time": "09:00:00", "msg": "missing-date"},
+        {"v015xxxxdate": "2024-05-01", "time": "10:00:00", "msg": "earliest-valid"},
+        {"v015xxxxdate": "2024-05-02", "time": "", "msg": "missing-time"},
+        {"v015xxxxdate": "bad-date", "time": "99:99:99", "msg": "invalid-datetime"},
+        {"v015xxxxdate": "2024-05-02", "time": "09:30:00", "msg": "middle-valid"},
+    ]
+    options = ProcessingOptions(
+        policy_cs="",
+        policy_ci="",
+        severity_cs="",
+        severity_ci="",
+        sort_by="datetime",
+        order="desc",
+        mode="vendor",
+    )
+
+    sorted_records = sort_records(records, options)
+
+    assert [record["msg"] for record in sorted_records] == [
+        "latest-valid",
+        "middle-valid",
+        "earliest-valid",
+        "missing-date",
+        "missing-time",
+        "invalid-datetime",
+    ]