Reduce conversion memory footprint

app/routes.py

@@ -14,15 +14,10 @@ from flask import (
 from werkzeug.datastructures import FileStorage
 
 from app.constants import MODES, OUTPUT_FORMATS, SORTABLE_FIELDS, SORT_ORDERS
-from app.services.exporter import build_export
-from app.services.parser import LogParseError, parse_log_file
-from app.services.processing import (
-    ProcessingError,
-    ProcessingOptions,
-    filter_records,
-    sort_records,
-)
-from app.services.storage import load_result_metadata, persist_result
+from app.services.conversion import convert_uploaded_log
+from app.services.parser import LogParseError
+from app.services.processing import ProcessingError, ProcessingOptions
+from app.services.storage import load_result_metadata
 
 main_blueprint = Blueprint("main", __name__)
 
@@ -95,7 +90,6 @@ def convert():
     assert uploaded_file is not None
 
     try:
-        records, union_keys = parse_log_file(uploaded_file.stream)
         options = ProcessingOptions(
             policy_cs=form.policy_cs,
             policy_ci=form.policy_ci,
@@ -105,12 +99,12 @@ def convert():
             order=form.order,
             mode=form.mode,
         )
-        filtered_records = filter_records(records, options)
-        sorted_records = sort_records(filtered_records, options)
-        export_result = build_export(sorted_records, union_keys, form.mode, form.output_format)
-        metadata = persist_result(
+        conversion_result = convert_uploaded_log(
+            stream=uploaded_file.stream,
+            options=options,
             output_dir=current_app.config["OUTPUT_DIRECTORY"],
-            export_result=export_result,
+            output_format=form.output_format,
+            preview_record_limit=current_app.config["PREVIEW_RECORD_LIMIT"],
         )
     except (LogParseError, ProcessingError) as exc:
         flash(str(exc), "danger")
@@ -122,15 +116,16 @@ def convert():
         )
         return render_template("index.html", form=form), 400
 
-    preview_limit = current_app.config["PREVIEW_RECORD_LIMIT"]
     return render_template(
         "result.html",
-        result_id=metadata.result_id,
-        preview_text=export_result.preview(preview_limit),
+        result_id=conversion_result.metadata.result_id,
+        preview_text=conversion_result.export_result.preview(
+            current_app.config["PREVIEW_RECORD_LIMIT"]
+        ),
         output_format=form.output_format,
-        record_count=len(sorted_records),
-        parsed_count=len(records),
-        filtered_count=len(sorted_records),
+        record_count=conversion_result.filtered_count,
+        parsed_count=conversion_result.parsed_count,
+        filtered_count=conversion_result.filtered_count,
         mode=form.mode,
         sort_by=form.sort_by,
         order=form.order,