bisco/webfortilog
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve log upload handling

Browse Source
This commit is contained in:
Alfredo Di Stasio
2026-04-24 15:00:43 +02:00
parent e793b51e4f
commit f64deb9c0d
10 changed files with 237 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tests/conftest.py
View File

@@ -9,7 +9,7 @@ from app import create_app
class TestConfig:
TESTING = True
SECRET_KEY = "test-secret"
MAX_CONTENT_LENGTH = 1024 * 1024
MAX_CONTENT_LENGTH = 100 * 1024 * 1024
PREVIEW_RECORD_LIMIT = 5
OUTPUT_DIRECTORY = "test-outputs"

60
tests/test_app.py
View File

@@ -1,5 +1,7 @@
import io
from app import create_app
SAMPLE_LOG = (
'v015xxxxdate=2024-05-01 time=10:00:00 policy="Prod Policy" '
@@ -23,6 +25,7 @@ def test_index_page_loads(client):
def test_convert_returns_text_preview_and_download_link(client):
log_file = io.BytesIO(SAMPLE_LOG.encode("utf-8"))
response = client.post(
"/convert",
data={
@@ -34,17 +37,20 @@ def test_convert_returns_text_preview_and_download_link(client):
"policy_ci": "prod",
"severity_cs": "",
"severity_ci": "",
"log_file": (io.BytesIO(SAMPLE_LOG.encode("utf-8")), "sample.log"),
"log_file": (log_file, "sample.log"),
},
content_type="multipart/form-data",
)
log_file.close()
assert response.status_code == 200
assert b"Download export" in response.data
assert b"--- record 1 ---" in response.data
response.close()
def test_convert_full_mode_csv_preserves_union_order(client):
log_file = io.BytesIO(SAMPLE_LOG.encode("utf-8"))
response = client.post(
"/convert",
data={
@@ -56,17 +62,20 @@ def test_convert_full_mode_csv_preserves_union_order(client):
"policy_ci": "",
"severity_cs": "",
"severity_ci": "",
"log_file": (io.BytesIO(SAMPLE_LOG.encode("utf-8")), "sample.log"),
"log_file": (log_file, "sample.log"),
},
content_type="multipart/form-data",
)
log_file.close()
assert response.status_code == 200
assert b"TEXT" not in response.data
assert b"Download export" in response.data
response.close()
def test_convert_rejects_mutually_exclusive_filters(client):
log_file = io.BytesIO(SAMPLE_LOG.encode("utf-8"))
response = client.post(
"/convert",
data={
@@ -78,16 +87,19 @@ def test_convert_rejects_mutually_exclusive_filters(client):
"policy_ci": "a",
"severity_cs": "",
"severity_ci": "",
"log_file": (io.BytesIO(SAMPLE_LOG.encode("utf-8")), "sample.log"),
"log_file": (log_file, "sample.log"),
},
content_type="multipart/form-data",
)
log_file.close()
assert response.status_code == 400
assert b"Policy filter must use either case-sensitive or case-insensitive match" in response.data
response.close()
def test_download_route_returns_generated_file(client):
log_file = io.BytesIO(SAMPLE_LOG.encode("utf-8"))
convert_response = client.post(
"/convert",
data={
@@ -99,10 +111,11 @@ def test_download_route_returns_generated_file(client):
"policy_ci": "",
"severity_cs": "",
"severity_ci": "",
"log_file": (io.BytesIO(SAMPLE_LOG.encode("utf-8")), "sample.log"),
"log_file": (log_file, "sample.log"),
},
content_type="multipart/form-data",
)
log_file.close()
html = convert_response.data.decode("utf-8")
marker = '/download/'
@@ -115,4 +128,43 @@ def test_download_route_returns_generated_file(client):
assert download_response.status_code == 200
assert download_response.headers["Content-Type"].startswith("text/csv")
assert b"v015xxxxdate,time,policy" in download_response.data
convert_response.close()
download_response.close()
def test_default_upload_limit_is_100_mib(app):
assert app.config["MAX_CONTENT_LENGTH"] == 100 * 1024 * 1024
def test_too_large_upload_returns_friendly_message(tmp_path):
class SmallLimitConfig:
TESTING = True
SECRET_KEY = "test-secret"
MAX_CONTENT_LENGTH = 128
PREVIEW_RECORD_LIMIT = 5
OUTPUT_DIRECTORY = tmp_path / "tiny-limit-outputs"
app = create_app(SmallLimitConfig)
client = app.test_client()
log_file = io.BytesIO(SAMPLE_LOG.encode("utf-8"))
response = client.post(
"/convert",
data={
"mode": "vendor",
"output_format": "text",
"sort_by": "datetime",
"order": "asc",
"policy_cs": "",
"policy_ci": "",
"severity_cs": "",
"severity_ci": "",
"log_file": (log_file, "sample.log"),
},
content_type="multipart/form-data",
)
log_file.close()
assert response.status_code == 413
assert b"Maximum allowed size is 128 bytes." in response.data
response.close()

15
tests/test_config.py Normal file
View File

@@ -0,0 +1,15 @@
from app.config import _get_max_content_length
def test_max_upload_size_mb_environment_variable(monkeypatch):
monkeypatch.setenv("MAX_UPLOAD_SIZE_MB", "42")
monkeypatch.delenv("MAX_CONTENT_LENGTH", raising=False)
assert _get_max_content_length() == 42 * 1024 * 1024
def test_max_content_length_environment_variable_is_supported(monkeypatch):
monkeypatch.delenv("MAX_UPLOAD_SIZE_MB", raising=False)
monkeypatch.setenv("MAX_CONTENT_LENGTH", "2048")
assert _get_max_content_length() == 2048

30
tests/test_parser.py
View File

@@ -28,3 +28,33 @@ def test_parse_log_file_rejects_tokens_without_equals():
with pytest.raises(LogParseError):
parse_log_file(stream)
def test_parse_log_file_supports_utf8_bom():
stream = io.BytesIO(
b'\xef\xbb\xbfv015xxxxdate=2024-02-15 time=09:10:11 msg="blocked request"\n'
)
records, _union_keys = parse_log_file(stream)
assert records[0]["v015xxxxdate"] == "2024-02-15"
def test_parse_log_file_supports_cp1252_text():
stream = io.BytesIO(
'v015xxxxdate=2024-02-15 time=09:10:11 msg="caf\xe9 request"\n'.encode("cp1252")
)
records, _union_keys = parse_log_file(stream)
assert records[0]["msg"] == "cafe request".replace("e", "é", 1)
def test_parse_log_file_tolerates_unterminated_quotes():
stream = io.BytesIO(
b'v015xxxxdate=2024-02-15 time=09:10:11 msg="broken quoted value\n'
)
records, _union_keys = parse_log_file(stream)
assert records[0]["msg"] == "broken quoted value"