From 8699bc15bc8a7cfa91d692e5e934b1c0821b402d Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Mon, 10 Nov 2025 10:19:34 -0500 Subject: [PATCH 01/20] Rework mime type white list --- src/onegov/form/validators.py | 44 ++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/src/onegov/form/validators.py b/src/onegov/form/validators.py index e84307d6d7..6818a061df 100644 --- a/src/onegov/form/validators.py +++ b/src/onegov/form/validators.py @@ -129,17 +129,49 @@ class WhitelistedMimeType: """ whitelist: Collection[str] = { - 'application/excel', - 'application/vnd.ms-excel', - 'application/msword', + # documents + 'application/msword', # doc 'application/pdf', + 'application/rtf', + 'application/vnd.ms-excel', # xls + ('application/vnd.openxmlformats-officedocument.' + 'presentationml.presentation'), # pptx + ('application/vnd.openxmlformats-officedocument.' + 'spreadsheetml.sheet'), # xlsx + ('application/vnd.openxmlformats-officedocument.' + 'wordprocessingml.document'), # docx + + # archives 'application/zip', + + # text / data + 'text/csv', + 'text/plain', + + # images + 'image/bmp', 'image/gif', - 'image/jpeg', + 'image/jpeg', # jpeg, jpg 'image/png', + 'image/svg', + 'image/svg+xml', + 'image/tiff', + 'image/webp', # shall we allow it? 'image/x-ms-bmp', - 'text/plain', - 'text/csv' + + # audio + 'audio/mp4', + 'audio/mpeg', + 'audio/wav', + 'audio/webm', # weba + + # video + 'video/mp4', + 'video/mpeg', # mpg, mpeg + 'video/ogg', + 'video/quicktime', # mov + 'video/webm', # webm + 'video/x-msvideo', # avi } message = _('Files of this type are not supported.') From 66ac4043a279d1353ceebda34865618cfc403d9d Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Mon, 17 Nov 2025 11:00:03 -0500 Subject: [PATCH 02/20] Ensure mime type validator for file fields in formcode --- tests/onegov/form/test_parser.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/tests/onegov/form/test_parser.py b/tests/onegov/form/test_parser.py index 4f23ff3378..1a65e206c0 100644 --- a/tests/onegov/form/test_parser.py +++ b/tests/onegov/form/test_parser.py @@ -10,7 +10,7 @@ from onegov.form.fields import ( DateTimeLocalField, MultiCheckboxField, TimeField, URLField, VideoURLField) from onegov.form.parser.grammar import field_help_identifier -from onegov.form.validators import LaxDataRequired +from onegov.form.validators import LaxDataRequired, WhitelistedMimeType from onegov.form.validators import ValidDateRange from onegov.pay import Price from textwrap import dedent @@ -341,6 +341,16 @@ def test_parse_fileinput() -> None: assert isinstance(form['file'], FileField) assert form['file'].widget.multiple is False # type: ignore[attr-defined] + # verify mime type validator + field = form._fields['file'] + assert any(isinstance(v, WhitelistedMimeType) for v in field.validators) + assert field.validators[1].whitelist == {'application/msword', 'application/pdf'} + + form = parse_form("File = *.*")() + field = form._fields['file'] + assert any(isinstance(v, WhitelistedMimeType) for v in field.validators) + assert field.validators[1].whitelist == WhitelistedMimeType.whitelist + def test_parse_multiplefileinput() -> None: form = parse_form("Files = *.pdf|*.doc (multiple)")() @@ -349,6 +359,17 @@ def test_parse_multiplefileinput() -> None: assert isinstance(form['files'], FileField) assert form['files'].widget.multiple is True # type: ignore[attr-defined] + # verify mime type validator + field = form._fields['files'] + assert field.validators + assert any(isinstance(v, WhitelistedMimeType) for v in field.validators) + assert field.validators[1].whitelist == {'application/msword', 'application/pdf'} + + form = parse_form("File = *.*")() + field = form._fields['files'] + assert any(isinstance(v, WhitelistedMimeType) for v in field.validators) + assert field.validators[1].whitelist == WhitelistedMimeType.whitelist + def test_parse_radio() -> None: From 5ebe0fbe06214dc2c00f1690ffd29a46468b7dd0 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Thu, 20 Nov 2025 07:21:36 -0500 Subject: [PATCH 03/20] Adds mime type validator by default --- .../election_day/forms/upload/common.py | 4 +- src/onegov/form/fields.py | 4 +- src/onegov/form/validators.py | 13 +++- tests/onegov/form/test_fields.py | 68 +++++++++++++++++-- tests/onegov/form/test_parser.py | 30 ++++---- 5 files changed, 99 insertions(+), 20 deletions(-) diff --git a/src/onegov/election_day/forms/upload/common.py b/src/onegov/election_day/forms/upload/common.py index e50464640e..2a527194b8 100644 --- a/src/onegov/election_day/forms/upload/common.py +++ b/src/onegov/election_day/forms/upload/common.py @@ -11,8 +11,8 @@ 'text/csv' } ALLOWED_MIME_TYPES_XML = { - 'application/xml', - 'text/xml', + 'application/xml', # official, standard + 'text/xml', # deprecated MIME type for XML content 'text/plain' } diff --git a/src/onegov/form/fields.py b/src/onegov/form/fields.py index 8a6aea3e63..449c6b330b 100644 --- a/src/onegov/form/fields.py +++ b/src/onegov/form/fields.py @@ -20,7 +20,7 @@ from onegov.file.utils import IMAGE_MIME_TYPES_AND_SVG from onegov.form import log, _ from onegov.form.utils import path_to_filename -from onegov.form.validators import ValidPhoneNumber +from onegov.form.validators import ValidPhoneNumber, WhitelistedMimeType from onegov.form.widgets import ChosenSelectWidget from onegov.form.widgets import LinkPanelWidget from onegov.form.widgets import DurationInput @@ -260,6 +260,7 @@ class UploadField(FileField): action: Literal['keep', 'replace', 'delete'] file: IO[bytes] | None filename: str | None + validators = [WhitelistedMimeType()] if TYPE_CHECKING: def __init__( @@ -448,6 +449,7 @@ def _add_entry(self, d: _MultiDictLikeWithGetlist, /) -> UploadField: upload_field_class: type[UploadField] = UploadField upload_widget: Widget[UploadField] = UploadWidget() + validators = [WhitelistedMimeType()] def __init__( self, diff --git a/src/onegov/form/validators.py b/src/onegov/form/validators.py index 6818a061df..005f62984a 100644 --- a/src/onegov/form/validators.py +++ b/src/onegov/form/validators.py @@ -141,6 +141,9 @@ class WhitelistedMimeType: ('application/vnd.openxmlformats-officedocument.' 'wordprocessingml.document'), # docx + # xml + 'application/xml', + # archives 'application/zip', @@ -184,7 +187,15 @@ def __call__(self, form: Form, field: Field) -> None: if not field.data: return - if field.data['mimetype'] not in self.whitelist: + if isinstance(field.data, list): # UploadMultipleField + for data in field.data: + if not data: + continue # in case of file deletion + + if data['mimetype'] not in self.whitelist: + raise ValidationError(field.gettext(self.message)) + + elif field.data['mimetype'] not in self.whitelist: raise ValidationError(field.gettext(self.message)) diff --git a/tests/onegov/form/test_fields.py b/tests/onegov/form/test_fields.py index d3dfc55984..b04cc56637 100644 --- a/tests/onegov/form/test_fields.py +++ b/tests/onegov/form/test_fields.py @@ -22,15 +22,19 @@ from onegov.form.fields import UploadField from onegov.form.fields import UploadMultipleField from onegov.form.fields import URLField -from onegov.form.validators import ValidPhoneNumber +from onegov.form.validators import ( + ValidPhoneNumber, WhitelistedMimeType, ExpectedExtensions) from unittest.mock import patch from wtforms.validators import Optional from wtforms.validators import URL -from typing import Any, TYPE_CHECKING +from typing import Any, TYPE_CHECKING, Self + if TYPE_CHECKING: from webob.request import _FieldStorageWithFile + from onegov.form.types import ( + FormT, Validators) class DummyPostData(dict[str, Any]): @@ -56,9 +60,11 @@ def create_file( def test_upload_field() -> None: - def create_field() -> tuple[Form, UploadField]: + def create_field( + validators: Validators[FormT, Self] | None = None + ) -> tuple[Form, UploadField]: form = Form() - field = UploadField() + field = UploadField(validators=validators) field = field.bind(form, 'upload') # type: ignore[attr-defined] return form, field @@ -68,12 +74,17 @@ def create_field() -> tuple[Form, UploadField]: assert data == {} assert field.file is None assert field.filename is None + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) form, field = create_field() data = field.process_fieldstorage('') assert data == {} assert field.file is None assert field.filename is None + assert field.validate(form) + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) textfile = create_file('text/plain', 'foo.txt', b'foo') data = field.process_fieldstorage(textfile) @@ -84,6 +95,9 @@ def create_field() -> tuple[Form, UploadField]: assert dictionary_to_binary(data) == b'foo' # type: ignore[arg-type] assert field.filename == 'foo.txt' assert field.file.read() == b'foo' # type: ignore[attr-defined] + assert field.validate(form) + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) form, field = create_field() textfile = create_file('text/plain', 'C:/mydata/bar.txt', b'bar') @@ -95,6 +109,20 @@ def create_field() -> tuple[Form, UploadField]: assert dictionary_to_binary(data) == b'bar' # type: ignore[arg-type] assert field.filename == 'bar.txt' assert field.file.read() == b'bar' # type: ignore[union-attr] + assert field.validate(form) + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + + # failing mime type validator + form, field = create_field(validators=[ExpectedExtensions(['.pdf'])]) + textfile = create_file('text/plain', 'baz.txt', b'baz') + field.data = field.process_fieldstorage(textfile) + assert field.data['filename'] == 'baz.txt' + assert field.data['mimetype'] == 'text/plain' + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert not field.validate(form) + assert 'Files of this type are not supported.' in field.errors # Test rendering form, field = create_field() @@ -104,6 +132,8 @@ def create_field() -> tuple[Form, UploadField]: field.data = field.process_fieldstorage(textfile) assert 'without-data' in field(force_simple=True) + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) html = field() assert 'with-data' in html @@ -111,6 +141,8 @@ def create_field() -> tuple[Form, UploadField]: assert 'keep' in html assert 'type="file"' in html assert 'value="baz.txt"' not in html + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) html = field(resend_upload=True) assert 'with-data' in html @@ -118,12 +150,16 @@ def create_field() -> tuple[Form, UploadField]: assert 'keep' in html assert 'type="file"' in html assert 'value="baz.txt"' in html + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) # Test submit form, field = create_field() field.process(DummyPostData({})) assert field.validate(form) assert field.data == {} + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) form, field = create_field() field.process(DummyPostData({'upload': 'abcd'})) @@ -132,6 +168,8 @@ def create_field() -> tuple[Form, UploadField]: assert field.data == {} assert field.file is None assert field.filename is None + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) # ... simple form, field = create_field() @@ -146,6 +184,8 @@ def create_field() -> tuple[Form, UploadField]: assert dictionary_to_binary(field.data) == b'foobar' # type: ignore[arg-type] assert field.filename == 'foobar.txt' assert field.file.read() == b'foobar' # type: ignore[union-attr] + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) # ... with select form, field = create_field() @@ -153,6 +193,8 @@ def create_field() -> tuple[Form, UploadField]: field.process(DummyPostData({'upload': ['keep', textfile]})) assert field.validate(form) assert field.action == 'keep' + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) form, field = create_field() textfile = create_file('text/plain', 'foobar.txt', b'foobar') @@ -160,6 +202,8 @@ def create_field() -> tuple[Form, UploadField]: assert field.validate(form) assert field.action == 'delete' assert field.data == {} + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) form, field = create_field() textfile = create_file('text/plain', 'foobar.txt', b'foobar') @@ -173,6 +217,8 @@ def create_field() -> tuple[Form, UploadField]: assert dictionary_to_binary(field.data) == b'foobar' # type: ignore[arg-type] assert field.filename == 'foobar.txt' assert field.file.read() == b'foobar' # type: ignore[union-attr] + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) # ... with select and keep upload previous = field.data @@ -191,6 +237,8 @@ def create_field() -> tuple[Form, UploadField]: assert field.data['mimetype'] == 'text/plain' assert field.data['size'] == 6 assert dictionary_to_binary(field.data) == b'foobar' # type: ignore[arg-type] + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) field.process(DummyPostData({'upload': [ 'delete', @@ -203,6 +251,8 @@ def create_field() -> tuple[Form, UploadField]: assert field2.validate(form) assert field2.action == 'delete' assert field2.data == {} + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) field.process(DummyPostData({'upload': [ 'replace', @@ -221,6 +271,8 @@ def create_field() -> tuple[Form, UploadField]: assert dictionary_to_binary(field2.data) == b'foobaz' # type: ignore[arg-type] assert field2.filename == 'foobaz.txt' assert field2.file.read() == b'foobaz' # type: ignore[union-attr] + assert (field.validators and + any(isinstance(v, WhitelistedMimeType) for v in field.validators)) def test_upload_multiple_field() -> None: @@ -245,6 +297,7 @@ def create_field() -> tuple[Form, UploadMultipleField]: file1 = create_file('text/plain', 'baz.txt', b'baz') file2 = create_file('text/plain', 'foobar.txt', b'foobar') field.process(DummyPostData({'uploads': [file1, file2]})) + assert field.validate(form) assert len(field.data) == 2 assert field.data[0]['filename'] == 'baz.txt' assert field.data[0]['mimetype'] == 'text/plain' @@ -266,7 +319,12 @@ def create_field() -> tuple[Form, UploadMultipleField]: assert file_field2.filename == 'foobar.txt' assert file_field2.file.read() == b'foobar' # type: ignore[union-attr] + # verify attached validators + assert field.validators + assert any(isinstance(v, WhitelistedMimeType) for v in field.validators) + html = field(force_simple=True) + assert field.validate(form) assert 'without-data' in html assert 'multiple' in html assert 'name="uploads"' in html @@ -274,6 +332,7 @@ def create_field() -> tuple[Form, UploadMultipleField]: assert 'name="uploads-0"' not in html html = field() + assert field.validate(form) assert 'with-data' in html assert 'name="uploads-0"' in html assert 'Uploaded file: baz.txt (3 Bytes) ✓' in html @@ -290,6 +349,7 @@ def create_field() -> tuple[Form, UploadMultipleField]: assert 'multiple' in html html = field(resend_upload=True) + assert field.validate(form) assert 'with-data' in html assert 'Uploaded file: baz.txt (3 Bytes) ✓' in html assert 'Uploaded file: foobar.txt (6 Bytes) ✓' in html diff --git a/tests/onegov/form/test_parser.py b/tests/onegov/form/test_parser.py index 1a65e206c0..c1c1fac028 100644 --- a/tests/onegov/form/test_parser.py +++ b/tests/onegov/form/test_parser.py @@ -334,6 +334,10 @@ def test_parse_time() -> None: assert isinstance(form['time'], TimeField) +def _find_validator(field, cls): + return next((v for v in field.validators if isinstance(v, cls)), None) + + def test_parse_fileinput() -> None: form = parse_form("File = *.pdf|*.doc")() @@ -341,10 +345,12 @@ def test_parse_fileinput() -> None: assert isinstance(form['file'], FileField) assert form['file'].widget.multiple is False # type: ignore[attr-defined] - # verify mime type validator - field = form._fields['file'] - assert any(isinstance(v, WhitelistedMimeType) for v in field.validators) - assert field.validators[1].whitelist == {'application/msword', 'application/pdf'} + # verify attached mime type validator + assert form['file'].validators + validator = _find_validator(form['file'], WhitelistedMimeType) + assert validator.whitelist == { + 'application/msword', 'application/pdf' + } form = parse_form("File = *.*")() field = form._fields['file'] @@ -354,25 +360,25 @@ def test_parse_fileinput() -> None: def test_parse_multiplefileinput() -> None: form = parse_form("Files = *.pdf|*.doc (multiple)")() + # form = parse_form("Files = *.pdf (multiple)")() assert form['files'].label.text == 'Files' assert isinstance(form['files'], FileField) assert form['files'].widget.multiple is True # type: ignore[attr-defined] - # verify mime type validator - field = form._fields['files'] - assert field.validators - assert any(isinstance(v, WhitelistedMimeType) for v in field.validators) - assert field.validators[1].whitelist == {'application/msword', 'application/pdf'} + # verify attached mime type validator + validator = _find_validator(form['files'], WhitelistedMimeType) + assert validator.whitelist == { + 'application/msword', 'application/pdf' + } - form = parse_form("File = *.*")() + form = parse_form("Files = *.* (multiple)")() field = form._fields['files'] assert any(isinstance(v, WhitelistedMimeType) for v in field.validators) - assert field.validators[1].whitelist == WhitelistedMimeType.whitelist + assert field.validators[0].whitelist == WhitelistedMimeType.whitelist def test_parse_radio() -> None: - text = dedent(""" Gender = ( ) Male From 7eed62a9be34df6f0c9295a086f752b718340ee7 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Mon, 1 Dec 2025 10:17:31 +0100 Subject: [PATCH 04/20] Fix wrongly attached validators --- src/onegov/form/fields.py | 1 + tests/onegov/form/test_parser.py | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/onegov/form/fields.py b/src/onegov/form/fields.py index 449c6b330b..2550ef800d 100644 --- a/src/onegov/form/fields.py +++ b/src/onegov/form/fields.py @@ -498,6 +498,7 @@ def __init__( widget=widget, # type:ignore[arg-type] render_kw=render_kw, name=name, + validators=validators, _form=_form, _prefix=_prefix, _translations=_translations, diff --git a/tests/onegov/form/test_parser.py b/tests/onegov/form/test_parser.py index c1c1fac028..aeeef4b538 100644 --- a/tests/onegov/form/test_parser.py +++ b/tests/onegov/form/test_parser.py @@ -373,9 +373,8 @@ def test_parse_multiplefileinput() -> None: } form = parse_form("Files = *.* (multiple)")() - field = form._fields['files'] - assert any(isinstance(v, WhitelistedMimeType) for v in field.validators) - assert field.validators[0].whitelist == WhitelistedMimeType.whitelist + validator = _find_validator(form['files'], WhitelistedMimeType) + assert validator.whitelist == WhitelistedMimeType.whitelist def test_parse_radio() -> None: From 4ab42dff8a842408ca722524849937031ec1ce1f Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Mon, 1 Dec 2025 10:48:54 +0100 Subject: [PATCH 05/20] Revert --- src/onegov/form/fields.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/onegov/form/fields.py b/src/onegov/form/fields.py index 2550ef800d..32c4a454bb 100644 --- a/src/onegov/form/fields.py +++ b/src/onegov/form/fields.py @@ -481,7 +481,6 @@ def __init__( # a lot of the arguments we just pass through to the subfield unbound_field = self.upload_field_class( - validators=validators, # type:ignore[arg-type] filters=filters, description=description, widget=upload_widget, From cf252e2df4a4ec4f42eb8132a1bd309092478c94 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Mon, 1 Dec 2025 11:05:32 +0100 Subject: [PATCH 06/20] improve tests and fix linter issues --- tests/onegov/form/test_parser.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/tests/onegov/form/test_parser.py b/tests/onegov/form/test_parser.py index aeeef4b538..b966fce479 100644 --- a/tests/onegov/form/test_parser.py +++ b/tests/onegov/form/test_parser.py @@ -4,6 +4,9 @@ from dateutil.relativedelta import relativedelta from decimal import Decimal + +from wtforms import Field + from onegov.form import Form, errors, find_field from onegov.form import parse_formcode, parse_form, flatten_fieldsets from onegov.form.errors import InvalidIndentSyntax @@ -25,10 +28,13 @@ from wtforms.validators import Regexp -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any + if TYPE_CHECKING: from pyparsing import ParserElement, ParseResults + from onegov.form.types import Validator + def parse(expr: ParserElement, text: str) -> ParseResults: return expr.parseString(text) @@ -334,7 +340,10 @@ def test_parse_time() -> None: assert isinstance(form['time'], TimeField) -def _find_validator(field, cls): +def _find_validator( + field: Field | FileField, + cls: type +) -> Validator[Any, Any] | None: return next((v for v in field.validators if isinstance(v, cls)), None) @@ -348,19 +357,19 @@ def test_parse_fileinput() -> None: # verify attached mime type validator assert form['file'].validators validator = _find_validator(form['file'], WhitelistedMimeType) - assert validator.whitelist == { + assert validator + assert validator.whitelist == { # type:ignore[attr-defined] 'application/msword', 'application/pdf' } form = parse_form("File = *.*")() - field = form._fields['file'] - assert any(isinstance(v, WhitelistedMimeType) for v in field.validators) - assert field.validators[1].whitelist == WhitelistedMimeType.whitelist + validator = _find_validator(form['file'], WhitelistedMimeType) + assert validator + assert validator.whitelist == WhitelistedMimeType.whitelist # type:ignore[attr-defined] def test_parse_multiplefileinput() -> None: form = parse_form("Files = *.pdf|*.doc (multiple)")() - # form = parse_form("Files = *.pdf (multiple)")() assert form['files'].label.text == 'Files' assert isinstance(form['files'], FileField) @@ -368,13 +377,15 @@ def test_parse_multiplefileinput() -> None: # verify attached mime type validator validator = _find_validator(form['files'], WhitelistedMimeType) - assert validator.whitelist == { + assert validator + assert validator.whitelist == { # type:ignore[attr-defined] 'application/msword', 'application/pdf' } form = parse_form("Files = *.* (multiple)")() validator = _find_validator(form['files'], WhitelistedMimeType) - assert validator.whitelist == WhitelistedMimeType.whitelist + assert validator + assert validator.whitelist == WhitelistedMimeType.whitelist # type:ignore[attr-defined] def test_parse_radio() -> None: From 6e9afd6280a0dfdfe10b42ac1874491b0dbe0f5e Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Mon, 1 Dec 2025 14:05:00 +0100 Subject: [PATCH 07/20] Rework tests --- tests/onegov/form/test_fields.py | 82 ++++++++++++++++---------------- tests/onegov/form/test_parser.py | 29 ++++++----- 2 files changed, 59 insertions(+), 52 deletions(-) diff --git a/tests/onegov/form/test_fields.py b/tests/onegov/form/test_fields.py index b04cc56637..2b37849a9e 100644 --- a/tests/onegov/form/test_fields.py +++ b/tests/onegov/form/test_fields.py @@ -6,6 +6,7 @@ from cgi import FieldStorage from copy import deepcopy from datetime import datetime + from onegov.core.utils import Bunch from onegov.core.utils import dictionary_to_binary from onegov.form import Form @@ -25,6 +26,7 @@ from onegov.form.validators import ( ValidPhoneNumber, WhitelistedMimeType, ExpectedExtensions) from unittest.mock import patch +from wtforms import FileField, Field from wtforms.validators import Optional from wtforms.validators import URL @@ -34,7 +36,23 @@ if TYPE_CHECKING: from webob.request import _FieldStorageWithFile from onegov.form.types import ( - FormT, Validators) + FormT, Validators, Validator) + + +def assert_whitelisted_mimetype_validator( + field: UploadField | UploadMultipleField +) -> None: + + validator = find_validator(field, WhitelistedMimeType) + assert validator + assert validator.whitelist == WhitelistedMimeType.whitelist # type:ignore[attr-defined] + + +def find_validator( + field: Field | FileField, + cls: type +) -> Validator[Any, Any] | None: + return next((v for v in field.validators if isinstance(v, cls)), None) class DummyPostData(dict[str, Any]): @@ -74,8 +92,7 @@ def create_field( assert data == {} assert field.file is None assert field.filename is None - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) form, field = create_field() data = field.process_fieldstorage('') @@ -83,8 +100,7 @@ def create_field( assert field.file is None assert field.filename is None assert field.validate(form) - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) textfile = create_file('text/plain', 'foo.txt', b'foo') data = field.process_fieldstorage(textfile) @@ -96,8 +112,7 @@ def create_field( assert field.filename == 'foo.txt' assert field.file.read() == b'foo' # type: ignore[attr-defined] assert field.validate(form) - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) form, field = create_field() textfile = create_file('text/plain', 'C:/mydata/bar.txt', b'bar') @@ -110,17 +125,17 @@ def create_field( assert field.filename == 'bar.txt' assert field.file.read() == b'bar' # type: ignore[union-attr] assert field.validate(form) - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) # failing mime type validator form, field = create_field(validators=[ExpectedExtensions(['.pdf'])]) textfile = create_file('text/plain', 'baz.txt', b'baz') - field.data = field.process_fieldstorage(textfile) - assert field.data['filename'] == 'baz.txt' - assert field.data['mimetype'] == 'text/plain' - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + data = field.data = field.process_fieldstorage(textfile) + assert data['filename'] == 'baz.txt' + assert data['mimetype'] == 'text/plain' + validator = find_validator(field, WhitelistedMimeType) + assert validator + assert validator.whitelist == {'application/pdf'} # type:ignore[attr-defined] assert not field.validate(form) assert 'Files of this type are not supported.' in field.errors @@ -132,8 +147,7 @@ def create_field( field.data = field.process_fieldstorage(textfile) assert 'without-data' in field(force_simple=True) - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) html = field() assert 'with-data' in html @@ -141,8 +155,7 @@ def create_field( assert 'keep' in html assert 'type="file"' in html assert 'value="baz.txt"' not in html - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) html = field(resend_upload=True) assert 'with-data' in html @@ -150,16 +163,14 @@ def create_field( assert 'keep' in html assert 'type="file"' in html assert 'value="baz.txt"' in html - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) # Test submit form, field = create_field() field.process(DummyPostData({})) assert field.validate(form) assert field.data == {} - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) form, field = create_field() field.process(DummyPostData({'upload': 'abcd'})) @@ -168,8 +179,7 @@ def create_field( assert field.data == {} assert field.file is None assert field.filename is None - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) # ... simple form, field = create_field() @@ -184,8 +194,7 @@ def create_field( assert dictionary_to_binary(field.data) == b'foobar' # type: ignore[arg-type] assert field.filename == 'foobar.txt' assert field.file.read() == b'foobar' # type: ignore[union-attr] - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) # ... with select form, field = create_field() @@ -193,8 +202,7 @@ def create_field( field.process(DummyPostData({'upload': ['keep', textfile]})) assert field.validate(form) assert field.action == 'keep' - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) form, field = create_field() textfile = create_file('text/plain', 'foobar.txt', b'foobar') @@ -202,8 +210,7 @@ def create_field( assert field.validate(form) assert field.action == 'delete' assert field.data == {} - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) form, field = create_field() textfile = create_file('text/plain', 'foobar.txt', b'foobar') @@ -217,8 +224,7 @@ def create_field( assert dictionary_to_binary(field.data) == b'foobar' # type: ignore[arg-type] assert field.filename == 'foobar.txt' assert field.file.read() == b'foobar' # type: ignore[union-attr] - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) # ... with select and keep upload previous = field.data @@ -237,8 +243,7 @@ def create_field( assert field.data['mimetype'] == 'text/plain' assert field.data['size'] == 6 assert dictionary_to_binary(field.data) == b'foobar' # type: ignore[arg-type] - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) field.process(DummyPostData({'upload': [ 'delete', @@ -251,8 +256,7 @@ def create_field( assert field2.validate(form) assert field2.action == 'delete' assert field2.data == {} - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) field.process(DummyPostData({'upload': [ 'replace', @@ -271,8 +275,7 @@ def create_field( assert dictionary_to_binary(field2.data) == b'foobaz' # type: ignore[arg-type] assert field2.filename == 'foobaz.txt' assert field2.file.read() == b'foobaz' # type: ignore[union-attr] - assert (field.validators and - any(isinstance(v, WhitelistedMimeType) for v in field.validators)) + assert_whitelisted_mimetype_validator(field) def test_upload_multiple_field() -> None: @@ -320,8 +323,7 @@ def create_field() -> tuple[Form, UploadMultipleField]: assert file_field2.file.read() == b'foobar' # type: ignore[union-attr] # verify attached validators - assert field.validators - assert any(isinstance(v, WhitelistedMimeType) for v in field.validators) + assert_whitelisted_mimetype_validator(field) html = field(force_simple=True) assert field.validate(form) diff --git a/tests/onegov/form/test_parser.py b/tests/onegov/form/test_parser.py index b966fce479..6acea95e59 100644 --- a/tests/onegov/form/test_parser.py +++ b/tests/onegov/form/test_parser.py @@ -11,7 +11,12 @@ from onegov.form import parse_formcode, parse_form, flatten_fieldsets from onegov.form.errors import InvalidIndentSyntax from onegov.form.fields import ( - DateTimeLocalField, MultiCheckboxField, TimeField, URLField, VideoURLField) + DateTimeLocalField, + MultiCheckboxField, + TimeField, + URLField, + VideoURLField, +) from onegov.form.parser.grammar import field_help_identifier from onegov.form.validators import LaxDataRequired, WhitelistedMimeType from onegov.form.validators import ValidDateRange @@ -40,6 +45,13 @@ def parse(expr: ParserElement, text: str) -> ParseResults: return expr.parseString(text) +def find_validator( + field: Field | FileField, + cls: type +) -> Validator[Any, Any] | None: + return next((v for v in field.validators if isinstance(v, cls)), None) + + @pytest.mark.parametrize('comment,output', [ ('<< Some text >>', 'Some text'), ('<< [Z](www.co.me) >>', '[Z](www.co.me)') @@ -340,13 +352,6 @@ def test_parse_time() -> None: assert isinstance(form['time'], TimeField) -def _find_validator( - field: Field | FileField, - cls: type -) -> Validator[Any, Any] | None: - return next((v for v in field.validators if isinstance(v, cls)), None) - - def test_parse_fileinput() -> None: form = parse_form("File = *.pdf|*.doc")() @@ -356,14 +361,14 @@ def test_parse_fileinput() -> None: # verify attached mime type validator assert form['file'].validators - validator = _find_validator(form['file'], WhitelistedMimeType) + validator = find_validator(form['file'], WhitelistedMimeType) assert validator assert validator.whitelist == { # type:ignore[attr-defined] 'application/msword', 'application/pdf' } form = parse_form("File = *.*")() - validator = _find_validator(form['file'], WhitelistedMimeType) + validator = find_validator(form['file'], WhitelistedMimeType) assert validator assert validator.whitelist == WhitelistedMimeType.whitelist # type:ignore[attr-defined] @@ -376,14 +381,14 @@ def test_parse_multiplefileinput() -> None: assert form['files'].widget.multiple is True # type: ignore[attr-defined] # verify attached mime type validator - validator = _find_validator(form['files'], WhitelistedMimeType) + validator = find_validator(form['files'], WhitelistedMimeType) assert validator assert validator.whitelist == { # type:ignore[attr-defined] 'application/msword', 'application/pdf' } form = parse_form("Files = *.* (multiple)")() - validator = _find_validator(form['files'], WhitelistedMimeType) + validator = find_validator(form['files'], WhitelistedMimeType) assert validator assert validator.whitelist == WhitelistedMimeType.whitelist # type:ignore[attr-defined] From 3f718d2612866f77e1c9a01eeeb44a2c3f15626b Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Mon, 1 Dec 2025 14:09:46 +0100 Subject: [PATCH 08/20] Fix file size validator and align --- src/onegov/form/validators.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/onegov/form/validators.py b/src/onegov/form/validators.py index 005f62984a..a9dad595cb 100644 --- a/src/onegov/form/validators.py +++ b/src/onegov/form/validators.py @@ -31,7 +31,8 @@ from wtforms.validators import ValidationError -from typing import Generic, TYPE_CHECKING +from typing import Generic, TYPE_CHECKING, Any + if TYPE_CHECKING: from collections.abc import Collection, Sequence from onegov.core.orm import Base @@ -114,7 +115,18 @@ def __call__(self, form: Form, field: Field) -> None: if not field.data: return - if field.data.get('size', 0) > self.max_bytes: + if isinstance(field.data, list): # UploadMultipleField + for data in field.data: + if not data: + continue # in case of file deletion + + self.validate_filesize(field, data) + + else: + self.validate_filesize(field, field.data) + + def validate_filesize(self, field: Field, data: dict[Any, Any]) -> None: + if data.get('size', 0) > self.max_bytes: message = field.gettext(self.message).format( humanize.naturalsize(self.max_bytes) ) @@ -192,11 +204,15 @@ def __call__(self, form: Form, field: Field) -> None: if not data: continue # in case of file deletion - if data['mimetype'] not in self.whitelist: - raise ValidationError(field.gettext(self.message)) + self.validate_mimetype(field, data) + + else: + self.validate_mimetype(field, field.data) - elif field.data['mimetype'] not in self.whitelist: - raise ValidationError(field.gettext(self.message)) + def validate_mimetype(self, field: Field, data: dict[Any, Any]) -> None: + if data['mimetype'] not in self.whitelist: + message = field.gettext(self.message) + raise ValidationError(field.gettext(message)) class ExpectedExtensions(WhitelistedMimeType): From 557202052ffd71071ec7b08a1fd2d3791fb38298 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Tue, 2 Dec 2025 09:20:53 +0100 Subject: [PATCH 09/20] Set mime types for all upload fields --- src/onegov/agency/forms/agency.py | 7 +- src/onegov/election_day/forms/election.py | 4 +- .../election_day/forms/election_compound.py | 8 +- src/onegov/election_day/forms/vote.py | 4 +- src/onegov/form/validators.py | 117 +++++++++++------- src/onegov/landsgemeinde/forms/agenda.py | 6 +- src/onegov/landsgemeinde/forms/assembly.py | 14 +-- src/onegov/org/forms/directory.py | 2 +- src/onegov/org/forms/event.py | 14 +-- src/onegov/org/forms/parliamentarian.py | 5 +- src/onegov/pas/forms/data_import.py | 4 + src/onegov/pas/forms/parliamentarian.py | 5 +- .../forms/accreditation.py | 24 ++-- .../translator_directory/forms/mutation.py | 24 ++-- 14 files changed, 129 insertions(+), 109 deletions(-) diff --git a/src/onegov/agency/forms/agency.py b/src/onegov/agency/forms/agency.py index b80029ee0a..8dc860f930 100644 --- a/src/onegov/agency/forms/agency.py +++ b/src/onegov/agency/forms/agency.py @@ -16,7 +16,7 @@ from onegov.form.fields import ChosenSelectField, HtmlField from onegov.form.fields import MultiCheckboxField from onegov.form.fields import UploadField -from onegov.form.validators import FileSizeLimit +from onegov.form.validators import FileSizeLimit, MIME_TYPES_IMAGE from onegov.form.validators import WhitelistedMimeType from onegov.gis import CoordinatesField from sqlalchemy import func @@ -73,10 +73,7 @@ class ExtendedAgencyForm(Form): organigram = UploadField( label=_('Organigram'), validators=[ - WhitelistedMimeType({ - 'image/jpeg', - 'image/png', - }), + WhitelistedMimeType(MIME_TYPES_IMAGE), FileSizeLimit(1 * 1024 * 1024) ] ) diff --git a/src/onegov/election_day/forms/election.py b/src/onegov/election_day/forms/election.py index 2c720e4933..9d4ff54478 100644 --- a/src/onegov/election_day/forms/election.py +++ b/src/onegov/election_day/forms/election.py @@ -13,7 +13,7 @@ from onegov.form.fields import ChosenSelectMultipleField from onegov.form.fields import PanelField from onegov.form.fields import UploadField -from onegov.form.validators import FileSizeLimit +from onegov.form.validators import FileSizeLimit, MIME_TYPES_PDF from onegov.form.validators import WhitelistedMimeType from re import findall from sqlalchemy import or_ @@ -323,7 +323,7 @@ class ElectionForm(Form): explanations_pdf = UploadField( label=_('Explanations (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024) ], fieldset=_('Related link') diff --git a/src/onegov/election_day/forms/election_compound.py b/src/onegov/election_day/forms/election_compound.py index 61be3c5e4c..76db82f6e8 100644 --- a/src/onegov/election_day/forms/election_compound.py +++ b/src/onegov/election_day/forms/election_compound.py @@ -12,7 +12,7 @@ from onegov.form.fields import ChosenSelectMultipleField from onegov.form.fields import PanelField from onegov.form.fields import UploadField -from onegov.form.validators import FileSizeLimit +from onegov.form.validators import FileSizeLimit, MIME_TYPES_PDF from onegov.form.validators import WhitelistedMimeType from re import findall from sqlalchemy import or_ @@ -228,7 +228,7 @@ class ElectionCompoundForm(Form): explanations_pdf = UploadField( label=_('Explanations (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024) ], fieldset=_('Related link') @@ -237,7 +237,7 @@ class ElectionCompoundForm(Form): upper_apportionment_pdf = UploadField( label=_('Upper apportionment (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024) ], fieldset=_('Related link'), @@ -247,7 +247,7 @@ class ElectionCompoundForm(Form): lower_apportionment_pdf = UploadField( label=_('Lower apportionment (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024) ], fieldset=_('Related link'), diff --git a/src/onegov/election_day/forms/vote.py b/src/onegov/election_day/forms/vote.py index 1719e07baa..1b5df4a516 100644 --- a/src/onegov/election_day/forms/vote.py +++ b/src/onegov/election_day/forms/vote.py @@ -10,7 +10,7 @@ from onegov.form.fields import ChosenSelectField from onegov.form.fields import PanelField from onegov.form.fields import UploadField -from onegov.form.validators import FileSizeLimit +from onegov.form.validators import FileSizeLimit, MIME_TYPES_PDF from onegov.form.validators import WhitelistedMimeType from wtforms.fields import BooleanField from wtforms.fields import DateField @@ -280,7 +280,7 @@ class VoteForm(Form): explanations_pdf = UploadField( label=_('Explanations (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024) ], fieldset=_('Related link') diff --git a/src/onegov/form/validators.py b/src/onegov/form/validators.py index a9dad595cb..5056747e44 100644 --- a/src/onegov/form/validators.py +++ b/src/onegov/form/validators.py @@ -133,6 +133,70 @@ def validate_filesize(self, field: Field, data: dict[Any, Any]) -> None: raise ValidationError(message) +MIME_TYPES_PDF = { + 'application/pdf', +} + +# for now not allowed by default +MIME_TYPES_JSON = { + 'application/json', +} + +MIME_TYPES_DOCUMENT = { + 'application/msword', # doc + 'application/rtf', + *MIME_TYPES_PDF, + 'application/vnd.ms-excel', # xls + ('application/vnd.openxmlformats-officedocument.' + 'presentationml.presentation'), # pptx + ('application/vnd.openxmlformats-officedocument.' + 'spreadsheetml.sheet'), # xlsx + ('application/vnd.openxmlformats-officedocument.' + 'wordprocessingml.document'), # docx +} + +MIME_TYPES_XML = { + 'application/xml', +} + +MIME_TYPES_ARCHIVE = { + 'application/zip', +} + +MIME_TYPES_TEXT_DATA = { + 'text/csv', + 'text/plain', +} + +MIME_TYPES_IMAGE = { + 'image/bmp', + 'image/gif', + 'image/jpeg', # jpeg, jpg + 'image/png', + 'image/svg', + 'image/svg+xml', + 'image/tiff', + 'image/webp', # shall we allow it? + 'image/x-ms-bmp', +} + +MIME_TYPES_AUDIO = { + 'audio/mp4', + 'audio/mpeg', + 'audio/wav', + 'audio/webm', # weba +} + +MIME_TYPES_VIDEO = { + 'video/mp4', + 'video/mpeg', # mpg, mpeg + 'video/ogg', + 'video/quicktime', # mov + 'video/webm', # webm + 'video/x-msvideo', # avi +} + + class WhitelistedMimeType: """ Makes sure an uploaded file is in a whitelist of allowed mimetypes. @@ -141,52 +205,13 @@ class WhitelistedMimeType: """ whitelist: Collection[str] = { - # documents - 'application/msword', # doc - 'application/pdf', - 'application/rtf', - 'application/vnd.ms-excel', # xls - ('application/vnd.openxmlformats-officedocument.' - 'presentationml.presentation'), # pptx - ('application/vnd.openxmlformats-officedocument.' - 'spreadsheetml.sheet'), # xlsx - ('application/vnd.openxmlformats-officedocument.' - 'wordprocessingml.document'), # docx - - # xml - 'application/xml', - - # archives - 'application/zip', - - # text / data - 'text/csv', - 'text/plain', - - # images - 'image/bmp', - 'image/gif', - 'image/jpeg', # jpeg, jpg - 'image/png', - 'image/svg', - 'image/svg+xml', - 'image/tiff', - 'image/webp', # shall we allow it? - 'image/x-ms-bmp', - - # audio - 'audio/mp4', - 'audio/mpeg', - 'audio/wav', - 'audio/webm', # weba - - # video - 'video/mp4', - 'video/mpeg', # mpg, mpeg - 'video/ogg', - 'video/quicktime', # mov - 'video/webm', # webm - 'video/x-msvideo', # avi + *MIME_TYPES_DOCUMENT, + *MIME_TYPES_XML, + *MIME_TYPES_ARCHIVE, + *MIME_TYPES_TEXT_DATA, + *MIME_TYPES_IMAGE, + *MIME_TYPES_AUDIO, + *MIME_TYPES_VIDEO, } message = _('Files of this type are not supported.') diff --git a/src/onegov/landsgemeinde/forms/agenda.py b/src/onegov/landsgemeinde/forms/agenda.py index e663d99d88..76114e2cce 100644 --- a/src/onegov/landsgemeinde/forms/agenda.py +++ b/src/onegov/landsgemeinde/forms/agenda.py @@ -19,7 +19,7 @@ from onegov.form.fields import TimeField from onegov.form.fields import UploadField from onegov.form.forms import NamedFileForm -from onegov.form.validators import FileSizeLimit +from onegov.form.validators import FileSizeLimit, MIME_TYPES_PDF, MIME_TYPES_ARCHIVE from onegov.form.validators import WhitelistedMimeType from onegov.landsgemeinde import _ from onegov.landsgemeinde.layouts import DefaultLayout @@ -80,7 +80,7 @@ class AgendaItemForm(NamedFileForm): label=_('Excerpt from the Memorial (PDF)'), fieldset=_('Memorial'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024) ] ) @@ -226,7 +226,7 @@ class AgendaItemUploadForm(Form): label=_('Agenda Item ZIP'), fieldset=_('Import'), validators=[ - WhitelistedMimeType({'application/zip'}), + WhitelistedMimeType(MIME_TYPES_ARCHIVE), FileSizeLimit(100 * 1024 * 1024) ] ) diff --git a/src/onegov/landsgemeinde/forms/assembly.py b/src/onegov/landsgemeinde/forms/assembly.py index ca4375e0fe..7449d0dd91 100644 --- a/src/onegov/landsgemeinde/forms/assembly.py +++ b/src/onegov/landsgemeinde/forms/assembly.py @@ -7,7 +7,7 @@ from onegov.form.fields import TimeField from onegov.form.fields import UploadField from onegov.form.forms import NamedFileForm -from onegov.form.validators import FileSizeLimit +from onegov.form.validators import FileSizeLimit, MIME_TYPES_PDF, MIME_TYPES_AUDIO, MIME_TYPES_ARCHIVE from onegov.form.validators import WhitelistedMimeType from onegov.landsgemeinde import _ from onegov.landsgemeinde.layouts import DefaultLayout @@ -83,7 +83,7 @@ class AssemblyForm(NamedFileForm): label=_('Memorial part 1 (PDF)'), fieldset=_('Downloads'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024) ] ) @@ -92,7 +92,7 @@ class AssemblyForm(NamedFileForm): label=_('Memorial part 2 (PDF)'), fieldset=_('Downloads'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024) ] ) @@ -101,7 +101,7 @@ class AssemblyForm(NamedFileForm): label=_('Supplement to the memorial (PDF)'), fieldset=_('Downloads'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024) ] ) @@ -110,7 +110,7 @@ class AssemblyForm(NamedFileForm): label=_('Protocol (PDF)'), fieldset=_('Downloads'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024) ] ) @@ -119,7 +119,7 @@ class AssemblyForm(NamedFileForm): label=_('Audio (MP3)'), fieldset=_('Downloads'), validators=[ - WhitelistedMimeType({'audio/mpeg'}), + WhitelistedMimeType(MIME_TYPES_AUDIO), FileSizeLimit(600 * 1024 * 1024) ] ) @@ -128,7 +128,7 @@ class AssemblyForm(NamedFileForm): label=_('Memorial as audio for the visually impaired and blind'), fieldset=_('Downloads'), validators=[ - WhitelistedMimeType({'application/zip'}), + WhitelistedMimeType(MIME_TYPES_ARCHIVE), FileSizeLimit(600 * 1024 * 1024) ] ) diff --git a/src/onegov/org/forms/directory.py b/src/onegov/org/forms/directory.py index aa6c707814..f6cc53a24d 100644 --- a/src/onegov/org/forms/directory.py +++ b/src/onegov/org/forms/directory.py @@ -17,7 +17,7 @@ from onegov.form.fields import IconField, MultiCheckboxField from onegov.form.fields import UploadField from onegov.form.filters import as_float -from onegov.form.validators import FileSizeLimit +from onegov.form.validators import FileSizeLimit, MIME_TYPES_ARCHIVE from onegov.form.validators import ValidFormDefinition from onegov.form.validators import WhitelistedMimeType from onegov.org import _ diff --git a/src/onegov/org/forms/event.py b/src/onegov/org/forms/event.py index 4f571e1c00..888c50e681 100644 --- a/src/onegov/org/forms/event.py +++ b/src/onegov/org/forms/event.py @@ -537,19 +537,7 @@ class EventImportForm(Form): label=_('Import'), validators=[ DataRequired(), - WhitelistedMimeType({ - 'application/excel', - 'application/vnd.ms-excel', - ( - 'application/' - 'vnd.openxmlformats-officedocument.spreadsheetml.sheet' - ), - 'application/vnd.ms-office', - 'application/octet-stream', - 'application/zip', - 'text/csv', - 'text/plain', - }), + WhitelistedMimeType(), FileSizeLimit(10 * 1024 * 1024) ], render_kw={'force_simple': True} diff --git a/src/onegov/org/forms/parliamentarian.py b/src/onegov/org/forms/parliamentarian.py index b9ccf61ab5..f0bf28d7ae 100644 --- a/src/onegov/org/forms/parliamentarian.py +++ b/src/onegov/org/forms/parliamentarian.py @@ -6,7 +6,7 @@ from onegov.form.fields import TranslatedSelectField from onegov.form.fields import UploadField from onegov.form.forms import NamedFileForm -from onegov.form.validators import ValidPhoneNumber +from onegov.form.validators import ValidPhoneNumber, WhitelistedMimeType, MIME_TYPES_IMAGE from onegov.org import _ from onegov.parliament.models.parliamentarian import GENDERS from wtforms.fields import DateField @@ -49,6 +49,9 @@ class ParliamentarianForm(NamedFileForm): picture = UploadField( label=_('Picture'), fieldset=_('Basic properties'), + validators=[ + WhitelistedMimeType(MIME_TYPES_IMAGE) + ] ) party = StringField( diff --git a/src/onegov/pas/forms/data_import.py b/src/onegov/pas/forms/data_import.py index e5de118dcd..8394ce54f2 100644 --- a/src/onegov/pas/forms/data_import.py +++ b/src/onegov/pas/forms/data_import.py @@ -4,6 +4,7 @@ from onegov.core.utils import dictionary_to_binary from onegov.form import Form from onegov.form.fields import UploadMultipleField +from onegov.form.validators import WhitelistedMimeType, MIME_TYPES_JSON from onegov.pas import _ from onegov.pas.importer.json_import import ( MembershipData, @@ -24,6 +25,7 @@ class DataImportForm(Form): people_source = UploadMultipleField( label=_('People Data (JSON)'), description=_('JSON file containing parliamentarian data.'), + validators=[WhitelistedMimeType(MIME_TYPES_JSON)] ) organizations_source = UploadMultipleField( label=_('Organizations Data (JSON)'), @@ -31,6 +33,7 @@ class DataImportForm(Form): 'JSON file containing organization data (commissions, ' 'parties, etc.).' ), + validators=[WhitelistedMimeType(MIME_TYPES_JSON)] ) memberships_source = UploadMultipleField( label=_('Memberships Data (JSON)'), @@ -38,6 +41,7 @@ class DataImportForm(Form): 'JSON file containing membership data (who is member of ' 'what organization).' ), + validators=[WhitelistedMimeType(MIME_TYPES_JSON)] ) validate_schema = BooleanField( diff --git a/src/onegov/pas/forms/parliamentarian.py b/src/onegov/pas/forms/parliamentarian.py index 0f184186bc..5a87ada37b 100644 --- a/src/onegov/pas/forms/parliamentarian.py +++ b/src/onegov/pas/forms/parliamentarian.py @@ -4,7 +4,7 @@ from onegov.form.fields import TranslatedSelectField from onegov.form.fields import UploadField from onegov.form.forms import NamedFileForm -from onegov.form.validators import ValidPhoneNumber +from onegov.form.validators import ValidPhoneNumber, MIME_TYPES_IMAGE, WhitelistedMimeType from onegov.parliament.models.parliamentarian import GENDERS from onegov.parliament.models.parliamentarian import SHIPPING_METHODS from onegov.pas.collections.parliamentarian import ( @@ -65,6 +65,9 @@ class PASParliamentarianForm(NamedFileForm): picture = UploadField( label=_('Picture'), fieldset=_('Basic properties'), + validators=[ + WhitelistedMimeType(MIME_TYPES_IMAGE), + ] ) shipping_method = TranslatedSelectField( diff --git a/src/onegov/translator_directory/forms/accreditation.py b/src/onegov/translator_directory/forms/accreditation.py index c54c2500ea..0c07f83c06 100644 --- a/src/onegov/translator_directory/forms/accreditation.py +++ b/src/onegov/translator_directory/forms/accreditation.py @@ -14,7 +14,7 @@ from onegov.form.fields import PanelField from onegov.form.fields import TagsField from onegov.form.fields import UploadField -from onegov.form.validators import FileSizeLimit +from onegov.form.validators import FileSizeLimit, MIME_TYPES_PDF from onegov.form.validators import Stdnum from onegov.form.validators import StrictOptional from onegov.form.validators import ValidPhoneNumber @@ -356,7 +356,7 @@ class RequestAccreditationForm(Form, DrivingDistanceMixin): declaration_of_authorization = UploadField( label=_('Signed declaration of authorization (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), DataRequired(), ], @@ -367,7 +367,7 @@ class RequestAccreditationForm(Form, DrivingDistanceMixin): letter_of_motivation = UploadField( label=_('Short letter of motivation (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), DataRequired(), ], @@ -378,7 +378,7 @@ class RequestAccreditationForm(Form, DrivingDistanceMixin): resume = UploadField( label=_('Resume (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), DataRequired(), ], @@ -393,7 +393,7 @@ class RequestAccreditationForm(Form, DrivingDistanceMixin): 'level C2 are mandatory for non-native speakers.' ), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), DataRequired(), ], @@ -404,7 +404,7 @@ class RequestAccreditationForm(Form, DrivingDistanceMixin): social_security_card = UploadField( label=_('Social security card (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), DataRequired(), ], @@ -415,7 +415,7 @@ class RequestAccreditationForm(Form, DrivingDistanceMixin): passport = UploadField( label=_('Identity card, passport or foreigner identity card (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), DataRequired(), ], @@ -426,7 +426,7 @@ class RequestAccreditationForm(Form, DrivingDistanceMixin): passport_photo = UploadField( label=_('Current passport photo (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), DataRequired(), ], @@ -438,7 +438,7 @@ class RequestAccreditationForm(Form, DrivingDistanceMixin): label=_('Current extract from the debt collection register (PDF)'), description=_('Maximum 6 months since issue.'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), DataRequired(), ], @@ -453,7 +453,7 @@ class RequestAccreditationForm(Form, DrivingDistanceMixin): 'www.strafregister.admin.ch' ), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), DataRequired(), ], @@ -465,7 +465,7 @@ class RequestAccreditationForm(Form, DrivingDistanceMixin): label=_('Certificate of Capability (PDF)'), description=_('Available from the municipal or city administration.'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), DataRequired(), ], @@ -479,7 +479,7 @@ class RequestAccreditationForm(Form, DrivingDistanceMixin): 'self-employment' ), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), DataRequired(), ], diff --git a/src/onegov/translator_directory/forms/mutation.py b/src/onegov/translator_directory/forms/mutation.py index f32560861b..052ae20b39 100644 --- a/src/onegov/translator_directory/forms/mutation.py +++ b/src/onegov/translator_directory/forms/mutation.py @@ -12,7 +12,7 @@ from onegov.form.fields import MultiCheckboxField from onegov.form.fields import TagsField from onegov.form.fields import UploadField -from onegov.form.validators import FileSizeLimit +from onegov.form.validators import FileSizeLimit, MIME_TYPES_PDF from onegov.form.validators import Stdnum from onegov.form.validators import ValidPhoneNumber from onegov.form.validators import ValidSwissSocialSecurityNumber @@ -256,7 +256,7 @@ def as_file(field: UploadField, category: str) -> File | None: declaration_of_authorization = UploadField( label=_('Signed declaration of authorization (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), Optional(), ], @@ -267,7 +267,7 @@ def as_file(field: UploadField, category: str) -> File | None: letter_of_motivation = UploadField( label=_('Short letter of motivation (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), Optional(), ], @@ -278,7 +278,7 @@ def as_file(field: UploadField, category: str) -> File | None: resume = UploadField( label=_('Resume (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), Optional(), ], @@ -293,7 +293,7 @@ def as_file(field: UploadField, category: str) -> File | None: 'level C2 are mandatory for non-native speakers.' ), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), Optional(), ], @@ -304,7 +304,7 @@ def as_file(field: UploadField, category: str) -> File | None: social_security_card = UploadField( label=_('Social security card (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), Optional(), ], @@ -315,7 +315,7 @@ def as_file(field: UploadField, category: str) -> File | None: passport = UploadField( label=_('Identity card, passport or foreigner identity card (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), Optional(), ], @@ -326,7 +326,7 @@ def as_file(field: UploadField, category: str) -> File | None: passport_photo = UploadField( label=_('Current passport photo (PDF)'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), Optional(), ], @@ -338,7 +338,7 @@ def as_file(field: UploadField, category: str) -> File | None: label=_('Current extract from the debt collection register (PDF)'), description=_('Maximum 6 months since issue.'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), Optional(), ], @@ -353,7 +353,7 @@ def as_file(field: UploadField, category: str) -> File | None: 'www.strafregister.admin.ch' ), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), Optional(), ], @@ -365,7 +365,7 @@ def as_file(field: UploadField, category: str) -> File | None: label=_('Certificate of Capability (PDF)'), description=_('Available from the municipal or city administration.'), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), Optional(), ], @@ -379,7 +379,7 @@ def as_file(field: UploadField, category: str) -> File | None: 'self-employment' ), validators=[ - WhitelistedMimeType({'application/pdf'}), + WhitelistedMimeType(MIME_TYPES_PDF), FileSizeLimit(100 * 1024 * 1024), Optional(), ], From 1293c8e0cb52dfd26911c2c5fda4fe590b9719ce Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Thu, 4 Dec 2025 11:57:55 +0100 Subject: [PATCH 10/20] Fix missing validator --- src/onegov/form/fields.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/onegov/form/fields.py b/src/onegov/form/fields.py index 32c4a454bb..b8731d6ea3 100644 --- a/src/onegov/form/fields.py +++ b/src/onegov/form/fields.py @@ -485,6 +485,7 @@ def __init__( description=description, widget=upload_widget, render_kw=render_kw, + validators=validators, **extra_arguments ) super().__init__( From af685f5e5802eeaf00509869b8adb242c45434b6 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Thu, 4 Dec 2025 11:58:53 +0100 Subject: [PATCH 11/20] Cleanup unused import --- src/onegov/org/forms/directory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/onegov/org/forms/directory.py b/src/onegov/org/forms/directory.py index f6cc53a24d..aa6c707814 100644 --- a/src/onegov/org/forms/directory.py +++ b/src/onegov/org/forms/directory.py @@ -17,7 +17,7 @@ from onegov.form.fields import IconField, MultiCheckboxField from onegov.form.fields import UploadField from onegov.form.filters import as_float -from onegov.form.validators import FileSizeLimit, MIME_TYPES_ARCHIVE +from onegov.form.validators import FileSizeLimit from onegov.form.validators import ValidFormDefinition from onegov.form.validators import WhitelistedMimeType from onegov.org import _ From a50f6f0e3c44ddab3e7222bd708213fa7a667104 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Thu, 4 Dec 2025 12:01:50 +0100 Subject: [PATCH 12/20] Add fixme --- src/onegov/pas/views/data_import.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/onegov/pas/views/data_import.py b/src/onegov/pas/views/data_import.py index 25b16ee0f9..0445ddcd4e 100644 --- a/src/onegov/pas/views/data_import.py +++ b/src/onegov/pas/views/data_import.py @@ -95,6 +95,7 @@ def handle_data_import( error_message = None + # FIXME: why not use `form.submitted(request)` like in many places? if request.method == 'POST' and form.validate(): try: # Load and concatenate data from uploaded files From 458a82d85ce9aa9943c6a9ea497e652174fb004f Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Thu, 4 Dec 2025 12:12:06 +0100 Subject: [PATCH 13/20] Fix linting errors --- src/onegov/form/fields.py | 2 +- src/onegov/landsgemeinde/forms/agenda.py | 8 ++++++-- src/onegov/landsgemeinde/forms/assembly.py | 9 +++++++-- src/onegov/org/forms/parliamentarian.py | 6 +++++- src/onegov/pas/forms/parliamentarian.py | 6 +++++- 5 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/onegov/form/fields.py b/src/onegov/form/fields.py index b8731d6ea3..89b6821803 100644 --- a/src/onegov/form/fields.py +++ b/src/onegov/form/fields.py @@ -485,7 +485,7 @@ def __init__( description=description, widget=upload_widget, render_kw=render_kw, - validators=validators, + validators=validators, # type:ignore[arg-type] **extra_arguments ) super().__init__( diff --git a/src/onegov/landsgemeinde/forms/agenda.py b/src/onegov/landsgemeinde/forms/agenda.py index 76114e2cce..07f590355c 100644 --- a/src/onegov/landsgemeinde/forms/agenda.py +++ b/src/onegov/landsgemeinde/forms/agenda.py @@ -19,8 +19,12 @@ from onegov.form.fields import TimeField from onegov.form.fields import UploadField from onegov.form.forms import NamedFileForm -from onegov.form.validators import FileSizeLimit, MIME_TYPES_PDF, MIME_TYPES_ARCHIVE -from onegov.form.validators import WhitelistedMimeType +from onegov.form.validators import ( + FileSizeLimit, + MIME_TYPES_PDF, + MIME_TYPES_ARCHIVE, + WhitelistedMimeType +) from onegov.landsgemeinde import _ from onegov.landsgemeinde.layouts import DefaultLayout from onegov.landsgemeinde.models import AgendaItem, LandsgemeindeFile diff --git a/src/onegov/landsgemeinde/forms/assembly.py b/src/onegov/landsgemeinde/forms/assembly.py index 7449d0dd91..e69affc5bc 100644 --- a/src/onegov/landsgemeinde/forms/assembly.py +++ b/src/onegov/landsgemeinde/forms/assembly.py @@ -7,8 +7,13 @@ from onegov.form.fields import TimeField from onegov.form.fields import UploadField from onegov.form.forms import NamedFileForm -from onegov.form.validators import FileSizeLimit, MIME_TYPES_PDF, MIME_TYPES_AUDIO, MIME_TYPES_ARCHIVE -from onegov.form.validators import WhitelistedMimeType +from onegov.form.validators import ( + FileSizeLimit, + MIME_TYPES_PDF, + MIME_TYPES_AUDIO, + MIME_TYPES_ARCHIVE, + WhitelistedMimeType +) from onegov.landsgemeinde import _ from onegov.landsgemeinde.layouts import DefaultLayout from onegov.landsgemeinde.models import Assembly, LandsgemeindeFile diff --git a/src/onegov/org/forms/parliamentarian.py b/src/onegov/org/forms/parliamentarian.py index f0bf28d7ae..39c44867ef 100644 --- a/src/onegov/org/forms/parliamentarian.py +++ b/src/onegov/org/forms/parliamentarian.py @@ -6,7 +6,11 @@ from onegov.form.fields import TranslatedSelectField from onegov.form.fields import UploadField from onegov.form.forms import NamedFileForm -from onegov.form.validators import ValidPhoneNumber, WhitelistedMimeType, MIME_TYPES_IMAGE +from onegov.form.validators import ( + MIME_TYPES_IMAGE, + ValidPhoneNumber, + WhitelistedMimeType +) from onegov.org import _ from onegov.parliament.models.parliamentarian import GENDERS from wtforms.fields import DateField diff --git a/src/onegov/pas/forms/parliamentarian.py b/src/onegov/pas/forms/parliamentarian.py index 5a87ada37b..4d0a85563f 100644 --- a/src/onegov/pas/forms/parliamentarian.py +++ b/src/onegov/pas/forms/parliamentarian.py @@ -4,7 +4,11 @@ from onegov.form.fields import TranslatedSelectField from onegov.form.fields import UploadField from onegov.form.forms import NamedFileForm -from onegov.form.validators import ValidPhoneNumber, MIME_TYPES_IMAGE, WhitelistedMimeType +from onegov.form.validators import ( + ValidPhoneNumber, + MIME_TYPES_IMAGE, + WhitelistedMimeType +) from onegov.parliament.models.parliamentarian import GENDERS from onegov.parliament.models.parliamentarian import SHIPPING_METHODS from onegov.pas.collections.parliamentarian import ( From 8d99429ee851494ef02c5d6e0de2640cd5e347bc Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Thu, 4 Dec 2025 12:39:15 +0100 Subject: [PATCH 14/20] Extend test --- tests/onegov/form/test_fields.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/onegov/form/test_fields.py b/tests/onegov/form/test_fields.py index 2b37849a9e..afe35b5dbf 100644 --- a/tests/onegov/form/test_fields.py +++ b/tests/onegov/form/test_fields.py @@ -279,12 +279,26 @@ def create_field( def test_upload_multiple_field() -> None: - def create_field() -> tuple[Form, UploadMultipleField]: + def create_field( + validators: Validators[FormT, Self] | None = None + ) -> tuple[Form, UploadMultipleField]: form = Form() - field = UploadMultipleField() + field = UploadMultipleField(validators=validators) field = field.bind(form, 'uploads') # type: ignore[attr-defined] return form, field + # failing mime type validator + form, field = create_field(validators=[ExpectedExtensions(['.json'])]) + file1 = create_file('text/plain', 'baz.txt', b'baz') + field.process(DummyPostData({'uploads': [file1]})) + assert not field.validate(form) + assert len(field.data) == 1 + assert field.data[0]['filename'] == 'baz.txt' + assert field.data[0]['mimetype'] == 'text/plain' + validator = find_validator(field, WhitelistedMimeType) + assert validator + assert validator.whitelist == {'application/json'} # type:ignore[attr-defined] + # Test rendering and initial submit form, field = create_field() field.process(None) @@ -321,8 +335,6 @@ def create_field() -> tuple[Form, UploadMultipleField]: assert dictionary_to_binary(file_field2.data) == b'foobar' # type: ignore[arg-type] assert file_field2.filename == 'foobar.txt' assert file_field2.file.read() == b'foobar' # type: ignore[union-attr] - - # verify attached validators assert_whitelisted_mimetype_validator(field) html = field(force_simple=True) From c31d0e3d5abb2796241bf3a548392b9ae428dd1e Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Thu, 4 Dec 2025 12:57:25 +0100 Subject: [PATCH 15/20] Fix more linter issues --- tests/onegov/form/test_fields.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/onegov/form/test_fields.py b/tests/onegov/form/test_fields.py index afe35b5dbf..4758c102c5 100644 --- a/tests/onegov/form/test_fields.py +++ b/tests/onegov/form/test_fields.py @@ -79,7 +79,7 @@ def create_file( def test_upload_field() -> None: def create_field( - validators: Validators[FormT, Self] | None = None + validators: Validators[FormT, Self] | None = None # type:ignore[misc] ) -> tuple[Form, UploadField]: form = Form() field = UploadField(validators=validators) @@ -280,7 +280,7 @@ def create_field( def test_upload_multiple_field() -> None: def create_field( - validators: Validators[FormT, Self] | None = None + validators: Validators[FormT, Self] | None = None # type:ignore[misc] ) -> tuple[Form, UploadMultipleField]: form = Form() field = UploadMultipleField(validators=validators) From eee980c1376789db10e7a4f6e7aa35036028be28 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Fri, 5 Dec 2025 10:00:45 +0100 Subject: [PATCH 16/20] Remove validators from field list --- src/onegov/form/fields.py | 1 - tests/onegov/form/test_fields.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/onegov/form/fields.py b/src/onegov/form/fields.py index 89b6821803..db15643a94 100644 --- a/src/onegov/form/fields.py +++ b/src/onegov/form/fields.py @@ -498,7 +498,6 @@ def __init__( widget=widget, # type:ignore[arg-type] render_kw=render_kw, name=name, - validators=validators, _form=_form, _prefix=_prefix, _translations=_translations, diff --git a/tests/onegov/form/test_fields.py b/tests/onegov/form/test_fields.py index 4758c102c5..fff4a52342 100644 --- a/tests/onegov/form/test_fields.py +++ b/tests/onegov/form/test_fields.py @@ -335,7 +335,8 @@ def create_field( assert dictionary_to_binary(file_field2.data) == b'foobar' # type: ignore[arg-type] assert file_field2.filename == 'foobar.txt' assert file_field2.file.read() == b'foobar' # type: ignore[union-attr] - assert_whitelisted_mimetype_validator(field) + for subfield in field: + assert_whitelisted_mimetype_validator(subfield) html = field(force_simple=True) assert field.validate(form) From 248819adefac1ce0fb6a8752bb56c34aa7691c07 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Fri, 5 Dec 2025 10:31:43 +0100 Subject: [PATCH 17/20] Add old ms office doc types --- src/onegov/form/validators.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/onegov/form/validators.py b/src/onegov/form/validators.py index 5056747e44..d8d4a923c0 100644 --- a/src/onegov/form/validators.py +++ b/src/onegov/form/validators.py @@ -153,6 +153,9 @@ def validate_filesize(self, field: Field, data: dict[Any, Any]) -> None: 'spreadsheetml.sheet'), # xlsx ('application/vnd.openxmlformats-officedocument.' 'wordprocessingml.document'), # docx + 'application/CDFV2', # old ms office docs + 'application/x-ole-storage', # old ms office docs + 'application/CDFV2-unknown' # old ms office docs } MIME_TYPES_XML = { From 5621994564090923622218ce64ea7493440ef7a6 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Fri, 5 Dec 2025 10:32:05 +0100 Subject: [PATCH 18/20] Remove non-standard svg type --- src/onegov/form/validators.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/onegov/form/validators.py b/src/onegov/form/validators.py index d8d4a923c0..79b61eb2e9 100644 --- a/src/onegov/form/validators.py +++ b/src/onegov/form/validators.py @@ -176,7 +176,6 @@ def validate_filesize(self, field: Field, data: dict[Any, Any]) -> None: 'image/gif', 'image/jpeg', # jpeg, jpg 'image/png', - 'image/svg', 'image/svg+xml', 'image/tiff', 'image/webp', # shall we allow it? From 21274d1e9bd22fec8c59d2407d22302447efbf65 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Fri, 5 Dec 2025 11:04:38 +0100 Subject: [PATCH 19/20] Update supported image mime type --- src/onegov/file/utils.py | 9 +-------- src/onegov/form/validators.py | 11 ++++------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/src/onegov/file/utils.py b/src/onegov/file/utils.py index f9daa6a386..75a8ce1549 100644 --- a/src/onegov/file/utils.py +++ b/src/onegov/file/utils.py @@ -74,15 +74,8 @@ def get_supported_image_mime_types() -> set[str]: # Not all PIL formats register a mime type, fill in the blanks ourselves. supported_types = { - 'image/bmp', - 'image/x-bmp', 'image/x-MS-bmp', - 'image/x-icon', - 'image/x-ico', - 'image/x-win-bitmap', - 'image/x-pcx', - 'image/x-portable-pixmap', - 'image/x-tga' + 'image/x-xcf', } for mime in Image.MIME.values(): diff --git a/src/onegov/form/validators.py b/src/onegov/form/validators.py index 79b61eb2e9..d984dc618c 100644 --- a/src/onegov/form/validators.py +++ b/src/onegov/form/validators.py @@ -12,6 +12,8 @@ from decimal import Decimal from dateutil.relativedelta import relativedelta from mimetypes import types_map + +from onegov.file.utils import get_supported_image_mime_types from onegov.form import _ from onegov.form.errors import (DuplicateLabelError, InvalidIndentSyntax, EmptyFieldsetError) @@ -172,14 +174,9 @@ def validate_filesize(self, field: Field, data: dict[Any, Any]) -> None: } MIME_TYPES_IMAGE = { - 'image/bmp', - 'image/gif', - 'image/jpeg', # jpeg, jpg - 'image/png', + # allowed types based on PIL + *get_supported_image_mime_types(), 'image/svg+xml', - 'image/tiff', - 'image/webp', # shall we allow it? - 'image/x-ms-bmp', } MIME_TYPES_AUDIO = { From c4a1eefc2f7115dc6b4010126dbb30e535cd8040 Mon Sep 17 00:00:00 2001 From: Reto Tschuppert Date: Fri, 5 Dec 2025 11:16:03 +0100 Subject: [PATCH 20/20] Remove unused json validator --- src/onegov/pas/forms/data_import.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/onegov/pas/forms/data_import.py b/src/onegov/pas/forms/data_import.py index 8394ce54f2..1728f3ebcb 100644 --- a/src/onegov/pas/forms/data_import.py +++ b/src/onegov/pas/forms/data_import.py @@ -4,7 +4,6 @@ from onegov.core.utils import dictionary_to_binary from onegov.form import Form from onegov.form.fields import UploadMultipleField -from onegov.form.validators import WhitelistedMimeType, MIME_TYPES_JSON from onegov.pas import _ from onegov.pas.importer.json_import import ( MembershipData, @@ -25,7 +24,7 @@ class DataImportForm(Form): people_source = UploadMultipleField( label=_('People Data (JSON)'), description=_('JSON file containing parliamentarian data.'), - validators=[WhitelistedMimeType(MIME_TYPES_JSON)] + validators=[] # no validators as files are not stored ) organizations_source = UploadMultipleField( label=_('Organizations Data (JSON)'), @@ -33,7 +32,7 @@ class DataImportForm(Form): 'JSON file containing organization data (commissions, ' 'parties, etc.).' ), - validators=[WhitelistedMimeType(MIME_TYPES_JSON)] + validators=[] # no validators as files are not stored ) memberships_source = UploadMultipleField( label=_('Memberships Data (JSON)'), @@ -41,7 +40,7 @@ class DataImportForm(Form): 'JSON file containing membership data (who is member of ' 'what organization).' ), - validators=[WhitelistedMimeType(MIME_TYPES_JSON)] + validators=[] # no validators as files are not stored ) validate_schema = BooleanField(