Build Flask WAF log converter app

app/__init__.py

@@ -0,0 +1,28 @@
+from pathlib import Path
+
+from flask import Flask, flash, render_template
+from werkzeug.exceptions import RequestEntityTooLarge
+
+from app.config import Config
+from app.routes import main_blueprint
+
+
+def create_app(config_class: type[Config] = Config) -> Flask:
+    """Application factory used by Flask and Gunicorn."""
+    app = Flask(__name__, instance_relative_config=True)
+    app.config.from_object(config_class)
+
+    output_dir = Path(app.config["OUTPUT_DIRECTORY"])
+    if not output_dir.is_absolute():
+        output_dir = Path(app.instance_path) / output_dir
+    app.config["OUTPUT_DIRECTORY"] = output_dir
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    app.register_blueprint(main_blueprint)
+
+    @app.errorhandler(RequestEntityTooLarge)
+    def handle_file_too_large(_error):
+        flash("The uploaded file is too large.", "danger")
+        return render_template("index.html"), 413
+
+    return app

app/config.py

@@ -0,0 +1,13 @@
+import os
+from pathlib import Path
+
+
+class Config:
+    """Default configuration for local and container usage."""
+
+    SECRET_KEY = os.environ.get("SECRET_KEY", "dev-secret-key-change-me")
+    MAX_CONTENT_LENGTH = int(os.environ.get("MAX_CONTENT_LENGTH", 10 * 1024 * 1024))
+    PREVIEW_RECORD_LIMIT = int(os.environ.get("PREVIEW_RECORD_LIMIT", 5))
+    OUTPUT_DIRECTORY = Path(
+        os.environ.get("OUTPUT_DIRECTORY", Path("instance") / "outputs")
+    )

app/constants.py

@@ -0,0 +1,35 @@
+VENDOR_FIELDS = [
+    "v015xxxxdate",
+    "time",
+    "policy",
+    "http_method",
+    "http_host",
+    "http_url",
+    "http_refer",
+    "service",
+    "backend_service",
+    "msg",
+    "signature_subclass",
+    "signature_id",
+    "owasp_top10",
+    "match_location",
+    "action",
+    "severity_level",
+]
+
+SEVERITY_RANKING = {
+    "critical": 5,
+    "high": 4,
+    "medium": 3,
+    "low": 2,
+    "info": 1,
+    "informational": 1,
+    "unknown": 0,
+    "none": 0,
+    "n/a": 0,
+}
+
+SORTABLE_FIELDS = {"datetime", "severity"}
+SORT_ORDERS = {"asc", "desc"}
+MODES = {"vendor", "full"}
+OUTPUT_FORMATS = {"text", "csv"}

app/routes.py

@@ -0,0 +1,150 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+from flask import (
+    Blueprint,
+    current_app,
+    flash,
+    redirect,
+    render_template,
+    request,
+    send_file,
+    url_for,
+)
+from werkzeug.datastructures import FileStorage
+
+from app.constants import MODES, OUTPUT_FORMATS, SORTABLE_FIELDS, SORT_ORDERS
+from app.services.exporter import build_export
+from app.services.parser import LogParseError, parse_log_file
+from app.services.processing import (
+    ProcessingError,
+    ProcessingOptions,
+    filter_records,
+    sort_records,
+)
+from app.services.storage import load_result_metadata, persist_result
+
+main_blueprint = Blueprint("main", __name__)
+
+
+@dataclass(slots=True)
+class FormData:
+    mode: str
+    output_format: str
+    sort_by: str
+    order: str
+    policy_cs: str
+    policy_ci: str
+    severity_cs: str
+    severity_ci: str
+
+
+def _normalize_form() -> FormData:
+    return FormData(
+        mode=request.form.get("mode", "vendor").strip(),
+        output_format=request.form.get("output_format", "text").strip(),
+        sort_by=request.form.get("sort_by", "datetime").strip(),
+        order=request.form.get("order", "asc").strip(),
+        policy_cs=request.form.get("policy_cs", "").strip(),
+        policy_ci=request.form.get("policy_ci", "").strip(),
+        severity_cs=request.form.get("severity_cs", "").strip(),
+        severity_ci=request.form.get("severity_ci", "").strip(),
+    )
+
+
+def _validate_form(file: FileStorage | None, form: FormData) -> list[str]:
+    errors: list[str] = []
+
+    if file is None or not file.filename:
+        errors.append("Please choose a log file to upload.")
+
+    if form.mode not in MODES:
+        errors.append("Invalid mode selection.")
+    if form.output_format not in OUTPUT_FORMATS:
+        errors.append("Invalid output format selection.")
+    if form.sort_by not in SORTABLE_FIELDS:
+        errors.append("Invalid sort field selection.")
+    if form.order not in SORT_ORDERS:
+        errors.append("Invalid sort order selection.")
+    if form.policy_cs and form.policy_ci:
+        errors.append(
+            "Policy filter must use either case-sensitive or case-insensitive match, not both."
+        )
+    if form.severity_cs and form.severity_ci:
+        errors.append(
+            "Severity filter must use either case-sensitive or case-insensitive match, not both."
+        )
+    return errors
+
+
+@main_blueprint.get("/")
+def index():
+    return render_template("index.html")
+
+
+@main_blueprint.post("/convert")
+def convert():
+    uploaded_file = request.files.get("log_file")
+    form = _normalize_form()
+    errors = _validate_form(uploaded_file, form)
+    if errors:
+        for error in errors:
+            flash(error, "danger")
+        return render_template("index.html", form=form), 400
+
+    assert uploaded_file is not None
+
+    try:
+        records, union_keys = parse_log_file(uploaded_file.stream)
+        options = ProcessingOptions(
+            policy_cs=form.policy_cs,
+            policy_ci=form.policy_ci,
+            severity_cs=form.severity_cs,
+            severity_ci=form.severity_ci,
+            sort_by=form.sort_by,
+            order=form.order,
+            mode=form.mode,
+        )
+        filtered_records = filter_records(records, options)
+        sorted_records = sort_records(filtered_records, options)
+        export_result = build_export(sorted_records, union_keys, form.mode, form.output_format)
+        metadata = persist_result(
+            output_dir=current_app.config["OUTPUT_DIRECTORY"],
+            export_result=export_result,
+        )
+    except (LogParseError, ProcessingError) as exc:
+        flash(str(exc), "danger")
+        return render_template("index.html", form=form), 400
+    except UnicodeDecodeError:
+        flash("The uploaded file is not valid UTF-8 text.", "danger")
+        return render_template("index.html", form=form), 400
+
+    preview_limit = current_app.config["PREVIEW_RECORD_LIMIT"]
+    return render_template(
+        "result.html",
+        result_id=metadata.result_id,
+        preview_text=export_result.preview(preview_limit),
+        output_format=form.output_format,
+        record_count=len(sorted_records),
+        parsed_count=len(records),
+        filtered_count=len(sorted_records),
+        mode=form.mode,
+        sort_by=form.sort_by,
+        order=form.order,
+    )
+
+
+@main_blueprint.get("/download/<result_id>")
+def download(result_id: str):
+    metadata = load_result_metadata(current_app.config["OUTPUT_DIRECTORY"], result_id)
+    if metadata is None:
+        flash("Requested output file could not be found.", "danger")
+        return redirect(url_for("main.index"))
+
+    return send_file(
+        Path(metadata["file_path"]),
+        as_attachment=True,
+        download_name=metadata["download_name"],
+        mimetype=metadata["mimetype"],
+        max_age=0,
+    )

app/services/__init__.py

@@ -0,0 +1 @@
+"""Service layer for parsing, processing, exporting, and file storage."""

app/services/exporter.py

@@ -0,0 +1,69 @@
+import csv
+import io
+from dataclasses import dataclass
+
+from app.constants import VENDOR_FIELDS
+
+
+@dataclass(slots=True)
+class ExportResult:
+    content: str
+    columns: list[str]
+    output_format: str
+
+    def preview(self, record_limit: int) -> str:
+        """Build a small preview string for the result page."""
+        if self.output_format == "text":
+            marker = f"--- record {record_limit + 1} ---"
+            if marker in self.content:
+                return self.content.split(marker, 1)[0].rstrip()
+            return self.content
+
+        lines = self.content.splitlines()
+        if len(lines) <= record_limit + 1:
+            return self.content
+        return "\n".join(lines[: record_limit + 1])
+
+
+def build_export(
+    records: list[dict[str, str]],
+    union_keys: list[str],
+    mode: str,
+    output_format: str,
+) -> ExportResult:
+    columns = VENDOR_FIELDS if mode == "vendor" else union_keys
+
+    if output_format == "text":
+        return ExportResult(
+            content=_render_text(records, columns),
+            columns=columns,
+            output_format=output_format,
+        )
+
+    return ExportResult(
+        content=_render_csv(records, columns),
+        columns=columns,
+        output_format=output_format,
+    )
+
+
+def _render_text(records: list[dict[str, str]], columns: list[str]) -> str:
+    max_key_length = max((len(column) for column in columns), default=0)
+    chunks: list[str] = []
+
+    for index, record in enumerate(records, start=1):
+        chunks.append(f"--- record {index} ---")
+        for column in columns:
+            value = record.get(column, "")
+            chunks.append(f"  {column.ljust(max_key_length)} = {value}")
+
+    return "\n".join(chunks)
+
+
+def _render_csv(records: list[dict[str, str]], columns: list[str]) -> str:
+    buffer = io.StringIO()
+    writer = csv.DictWriter(buffer, fieldnames=columns, extrasaction="ignore")
+    writer.writeheader()
+    for record in records:
+        writer.writerow({column: record.get(column, "") for column in columns})
+    return buffer.getvalue()

app/services/parser.py

@@ -0,0 +1,47 @@
+import shlex
+from collections import OrderedDict
+from io import BufferedIOBase, TextIOBase
+
+
+class LogParseError(ValueError):
+    """Raised when the uploaded log file cannot be parsed."""
+
+
+def parse_log_file(stream: BufferedIOBase | TextIOBase) -> tuple[list[dict[str, str]], list[str]]:
+    """Parse a UTF-8 log file where each line contains shell-like key/value tokens."""
+    raw_bytes = stream.read()
+    if isinstance(raw_bytes, str):
+        content = raw_bytes
+    else:
+        content = raw_bytes.decode("utf-8")
+
+    records: list[dict[str, str]] = []
+    seen_keys: OrderedDict[str, None] = OrderedDict()
+
+    for line_number, raw_line in enumerate(content.splitlines(), start=1):
+        line = raw_line.strip()
+        if not line:
+            continue
+
+        try:
+            tokens = shlex.split(line, posix=True)
+        except ValueError as exc:
+            raise LogParseError(f"Line {line_number}: invalid shell-style quoting.") from exc
+
+        record: dict[str, str] = {}
+        for token in tokens:
+            if "=" not in token:
+                raise LogParseError(
+                    f"Line {line_number}: token '{token}' is missing '='."
+                )
+
+            key, value = token.split("=", 1)
+            if not key:
+                raise LogParseError(f"Line {line_number}: empty key is not allowed.")
+
+            record[key] = value
+            seen_keys.setdefault(key, None)
+
+        records.append(record)
+
+    return records, list(seen_keys.keys())

app/services/processing.py

@@ -0,0 +1,78 @@
+from dataclasses import dataclass
+from datetime import datetime
+
+from app.constants import SEVERITY_RANKING
+
+
+class ProcessingError(ValueError):
+    """Raised when records cannot be processed according to the selected options."""
+
+
+@dataclass(slots=True)
+class ProcessingOptions:
+    policy_cs: str
+    policy_ci: str
+    severity_cs: str
+    severity_ci: str
+    sort_by: str
+    order: str
+    mode: str
+
+
+def filter_records(
+    records: list[dict[str, str]], options: ProcessingOptions
+) -> list[dict[str, str]]:
+    """Apply user-selected filters to parsed records."""
+    filtered: list[dict[str, str]] = []
+
+    for record in records:
+        policy_value = record.get("policy", "")
+        severity_value = record.get("severity_level", "")
+
+        if options.policy_cs and options.policy_cs not in policy_value:
+            continue
+        if options.policy_ci and options.policy_ci.lower() not in policy_value.lower():
+            continue
+        if options.severity_cs and options.severity_cs not in severity_value:
+            continue
+        if options.severity_ci and options.severity_ci.lower() not in severity_value.lower():
+            continue
+
+        filtered.append(record)
+
+    return filtered
+
+
+def sort_records(
+    records: list[dict[str, str]], options: ProcessingOptions
+) -> list[dict[str, str]]:
+    """Sort records by datetime or severity using the requested order."""
+    reverse = options.order == "desc"
+
+    if options.sort_by == "datetime":
+        key_func = _datetime_key
+    elif options.sort_by == "severity":
+        key_func = _severity_key
+    else:
+        raise ProcessingError("Unsupported sort field.")
+
+    return sorted(records, key=key_func, reverse=reverse)
+
+
+def _datetime_key(record: dict[str, str]) -> tuple[int, datetime]:
+    date_value = record.get("v015xxxxdate", "").strip()
+    time_value = record.get("time", "").strip()
+    if not date_value or not time_value:
+        return (1, datetime.max)
+
+    try:
+        parsed = datetime.strptime(f"{date_value} {time_value}", "%Y-%m-%d %H:%M:%S")
+    except ValueError:
+        return (1, datetime.max)
+    return (0, parsed)
+
+
+def _severity_key(record: dict[str, str]) -> tuple[int, str]:
+    raw_value = record.get("severity_level", "").strip().lower()
+    rank = SEVERITY_RANKING.get(raw_value, 0)
+    return (rank, raw_value)

app/services/storage.py

@@ -0,0 +1,43 @@
+import json
+import uuid
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+from app.services.exporter import ExportResult
+
+
+@dataclass(slots=True)
+class ResultMetadata:
+    result_id: str
+    file_path: str
+    download_name: str
+    mimetype: str
+
+
+def persist_result(output_dir: Path, export_result: ExportResult) -> ResultMetadata:
+    """Persist generated output and sidecar metadata in a temporary directory."""
+    result_id = uuid.uuid4().hex
+    extension = "txt" if export_result.output_format == "text" else "csv"
+    mimetype = "text/plain; charset=utf-8" if extension == "txt" else "text/csv; charset=utf-8"
+
+    file_path = output_dir / f"{result_id}.{extension}"
+    metadata_path = output_dir / f"{result_id}.json"
+
+    file_path.write_text(export_result.content, encoding="utf-8")
+    metadata = ResultMetadata(
+        result_id=result_id,
+        file_path=str(file_path),
+        download_name=f"waf-report.{extension}",
+        mimetype=mimetype,
+    )
+    metadata_path.write_text(json.dumps(asdict(metadata)), encoding="utf-8")
+    return metadata
+
+
+def load_result_metadata(output_dir: Path, result_id: str) -> dict[str, str] | None:
+    """Load sidecar metadata for a generated file."""
+    metadata_path = output_dir / f"{result_id}.json"
+    if not metadata_path.exists():
+        return None
+
+    return json.loads(metadata_path.read_text(encoding="utf-8"))

app/templates/base.html

@@ -0,0 +1,38 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title>WAF Log Converter</title>
+    <link
+      href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css"
+      rel="stylesheet"
+      integrity="sha384-QWTKZyjpPEjISv5WaRU9OFeRpok6YctnYmDr5pNlyT2bRjXh0JMhjY6hW+ALEwIH"
+      crossorigin="anonymous"
+    >
+  </head>
+  <body class="bg-body-tertiary">
+    <main class="container py-5">
+      <div class="row justify-content-center">
+        <div class="col-lg-10">
+          <div class="mb-4">
+            <h1 class="display-6 fw-semibold">WAF Log Converter</h1>
+            <p class="text-secondary mb-0">
+              Upload a UTF-8 WAF log file and export a filtered report as readable text or CSV.
+            </p>
+          </div>
+
+          {% with messages = get_flashed_messages(with_categories=true) %}
+            {% if messages %}
+              {% for category, message in messages %}
+                <div class="alert alert-{{ category }}" role="alert">{{ message }}</div>
+              {% endfor %}
+            {% endif %}
+          {% endwith %}
+
+          {% block content %}{% endblock %}
+        </div>
+      </div>
+    </main>
+  </body>
+</html>

app/templates/index.html

@@ -0,0 +1,100 @@
+{% extends "base.html" %}
+{% set form = form or none %}
+{% block content %}
+  <div class="card shadow-sm border-0">
+    <div class="card-body p-4">
+      <form method="post" action="{{ url_for('main.convert') }}" enctype="multipart/form-data" novalidate>
+        <div class="mb-4">
+          <label for="log_file" class="form-label fw-semibold">Log file</label>
+          <input class="form-control" id="log_file" name="log_file" type="file" required>
+          <div class="form-text">Each line must contain one record using shell-like key/value tokens.</div>
+        </div>
+
+        <div class="row g-3">
+          <div class="col-md-3">
+            <label for="mode" class="form-label">Mode</label>
+            <select class="form-select" id="mode" name="mode">
+              <option value="vendor" {% if form and form.mode == "vendor" %}selected{% endif %}>Vendor</option>
+              <option value="full" {% if form and form.mode == "full" %}selected{% endif %}>Full</option>
+            </select>
+          </div>
+          <div class="col-md-3">
+            <label for="output_format" class="form-label">Format</label>
+            <select class="form-select" id="output_format" name="output_format">
+              <option value="text" {% if form and form.output_format == "text" %}selected{% endif %}>Text</option>
+              <option value="csv" {% if form and form.output_format == "csv" %}selected{% endif %}>CSV</option>
+            </select>
+          </div>
+          <div class="col-md-3">
+            <label for="sort_by" class="form-label">Sort by</label>
+            <select class="form-select" id="sort_by" name="sort_by">
+              <option value="datetime" {% if not form or form.sort_by == "datetime" %}selected{% endif %}>Datetime</option>
+              <option value="severity" {% if form and form.sort_by == "severity" %}selected{% endif %}>Severity</option>
+            </select>
+          </div>
+          <div class="col-md-3">
+            <label for="order" class="form-label">Order</label>
+            <select class="form-select" id="order" name="order">
+              <option value="asc" {% if not form or form.order == "asc" %}selected{% endif %}>Ascending</option>
+              <option value="desc" {% if form and form.order == "desc" %}selected{% endif %}>Descending</option>
+            </select>
+          </div>
+        </div>
+
+        <hr class="my-4">
+
+        <div class="row g-3">
+          <div class="col-md-6">
+            <label for="policy_cs" class="form-label">Policy filter, case-sensitive</label>
+            <input
+              class="form-control"
+              id="policy_cs"
+              name="policy_cs"
+              type="text"
+              value="{{ form.policy_cs if form else '' }}"
+            >
+          </div>
+          <div class="col-md-6">
+            <label for="policy_ci" class="form-label">Policy filter, case-insensitive</label>
+            <input
+              class="form-control"
+              id="policy_ci"
+              name="policy_ci"
+              type="text"
+              value="{{ form.policy_ci if form else '' }}"
+            >
+          </div>
+          <div class="col-md-6">
+            <label for="severity_cs" class="form-label">Severity filter, case-sensitive</label>
+            <input
+              class="form-control"
+              id="severity_cs"
+              name="severity_cs"
+              type="text"
+              value="{{ form.severity_cs if form else '' }}"
+            >
+          </div>
+          <div class="col-md-6">
+            <label for="severity_ci" class="form-label">Severity filter, case-insensitive</label>
+            <input
+              class="form-control"
+              id="severity_ci"
+              name="severity_ci"
+              type="text"
+              value="{{ form.severity_ci if form else '' }}"
+            >
+          </div>
+        </div>
+
+        <div class="alert alert-light border mt-4 mb-0" role="note">
+          Use only one policy filter and one severity filter at a time. Matching happens as a partial substring.
+        </div>
+
+        <div class="mt-4 d-flex gap-2">
+          <button class="btn btn-primary" type="submit">Convert log</button>
+          <button class="btn btn-outline-secondary" type="reset">Reset</button>
+        </div>
+      </form>
+    </div>
+  </div>
+{% endblock %}

app/templates/result.html

@@ -0,0 +1,45 @@
+{% extends "base.html" %}
+{% block content %}
+  <div class="row g-4">
+    <div class="col-lg-4">
+      <div class="card shadow-sm border-0 h-100">
+        <div class="card-body">
+          <h2 class="h4">Result summary</h2>
+          <dl class="row mb-4">
+            <dt class="col-sm-5">Parsed records</dt>
+            <dd class="col-sm-7">{{ parsed_count }}</dd>
+            <dt class="col-sm-5">Output records</dt>
+            <dd class="col-sm-7">{{ filtered_count }}</dd>
+            <dt class="col-sm-5">Mode</dt>
+            <dd class="col-sm-7 text-capitalize">{{ mode }}</dd>
+            <dt class="col-sm-5">Format</dt>
+            <dd class="col-sm-7 text-uppercase">{{ output_format }}</dd>
+            <dt class="col-sm-5">Sort</dt>
+            <dd class="col-sm-7">{{ sort_by }} / {{ order }}</dd>
+          </dl>
+
+          <div class="d-grid gap-2">
+            <a class="btn btn-primary" href="{{ url_for('main.download', result_id=result_id) }}">
+              Download export
+            </a>
+            <a class="btn btn-outline-secondary" href="{{ url_for('main.index') }}">
+              Convert another file
+            </a>
+          </div>
+        </div>
+      </div>
+    </div>
+
+    <div class="col-lg-8">
+      <div class="card shadow-sm border-0">
+        <div class="card-body">
+          <div class="d-flex justify-content-between align-items-center mb-3">
+            <h2 class="h4 mb-0">Preview</h2>
+            <span class="badge text-bg-secondary">Showing up to {{ record_count if record_count < 5 else 5 }} records</span>
+          </div>
+          <pre class="bg-dark-subtle p-3 rounded small mb-0" style="white-space: pre-wrap;">{{ preview_text }}</pre>
+        </div>
+      </div>
+    </div>
+  </div>
+{% endblock %}