Rework mime type white list

src/onegov/agency/forms/agency.py

@@ -16,7 +16,7 @@
 from onegov.form.fields import ChosenSelectField, HtmlField
 from onegov.form.fields import MultiCheckboxField
 from onegov.form.fields import UploadField
-from onegov.form.validators import FileSizeLimit
+from onegov.form.validators import FileSizeLimit, MIME_TYPES_IMAGE
 from onegov.form.validators import WhitelistedMimeType
 from onegov.gis import CoordinatesField
 from sqlalchemy import func
@@ -73,10 +73,7 @@ class ExtendedAgencyForm(Form):
     organigram = UploadField(
         label=_('Organigram'),
         validators=[
-            WhitelistedMimeType({
-                'image/jpeg',
-                'image/png',
-            }),
+            WhitelistedMimeType(MIME_TYPES_IMAGE),
             FileSizeLimit(1 * 1024 * 1024)
         ]
     )

src/onegov/election_day/forms/election.py

@@ -13,7 +13,7 @@
 from onegov.form.fields import ChosenSelectMultipleField
 from onegov.form.fields import PanelField
 from onegov.form.fields import UploadField
-from onegov.form.validators import FileSizeLimit
+from onegov.form.validators import FileSizeLimit, MIME_TYPES_PDF
 from onegov.form.validators import WhitelistedMimeType
 from re import findall
 from sqlalchemy import or_
@@ -323,7 +323,7 @@ class ElectionForm(Form):
     explanations_pdf = UploadField(
         label=_('Explanations (PDF)'),
         validators=[
-            WhitelistedMimeType({'application/pdf'}),
+            WhitelistedMimeType(MIME_TYPES_PDF),
             FileSizeLimit(100 * 1024 * 1024)
         ],
         fieldset=_('Related link')

src/onegov/election_day/forms/election_compound.py

@@ -12,7 +12,7 @@
 from onegov.form.fields import ChosenSelectMultipleField
 from onegov.form.fields import PanelField
 from onegov.form.fields import UploadField
-from onegov.form.validators import FileSizeLimit
+from onegov.form.validators import FileSizeLimit, MIME_TYPES_PDF
 from onegov.form.validators import WhitelistedMimeType
 from re import findall
 from sqlalchemy import or_
@@ -228,7 +228,7 @@ class ElectionCompoundForm(Form):
     explanations_pdf = UploadField(
         label=_('Explanations (PDF)'),
         validators=[
-            WhitelistedMimeType({'application/pdf'}),
+            WhitelistedMimeType(MIME_TYPES_PDF),
             FileSizeLimit(100 * 1024 * 1024)
         ],
         fieldset=_('Related link')
@@ -237,7 +237,7 @@ class ElectionCompoundForm(Form):
     upper_apportionment_pdf = UploadField(
         label=_('Upper apportionment (PDF)'),
         validators=[
-            WhitelistedMimeType({'application/pdf'}),
+            WhitelistedMimeType(MIME_TYPES_PDF),
             FileSizeLimit(100 * 1024 * 1024)
         ],
         fieldset=_('Related link'),
@@ -247,7 +247,7 @@ class ElectionCompoundForm(Form):
     lower_apportionment_pdf = UploadField(
         label=_('Lower apportionment (PDF)'),
         validators=[
-            WhitelistedMimeType({'application/pdf'}),
+            WhitelistedMimeType(MIME_TYPES_PDF),
             FileSizeLimit(100 * 1024 * 1024)
         ],
         fieldset=_('Related link'),

src/onegov/election_day/forms/upload/common.py

@@ -11,8 +11,8 @@
     'text/csv'
 }
 ALLOWED_MIME_TYPES_XML = {
-    'application/xml',
-    'text/xml',
+    'application/xml',  # official, standard
+    'text/xml',  # deprecated MIME type for XML content
     'text/plain'
 }
 

src/onegov/election_day/forms/vote.py

@@ -10,7 +10,7 @@
 from onegov.form.fields import ChosenSelectField
 from onegov.form.fields import PanelField
 from onegov.form.fields import UploadField
-from onegov.form.validators import FileSizeLimit
+from onegov.form.validators import FileSizeLimit, MIME_TYPES_PDF
 from onegov.form.validators import WhitelistedMimeType
 from wtforms.fields import BooleanField
 from wtforms.fields import DateField
@@ -280,7 +280,7 @@ class VoteForm(Form):
     explanations_pdf = UploadField(
         label=_('Explanations (PDF)'),
         validators=[
-            WhitelistedMimeType({'application/pdf'}),
+            WhitelistedMimeType(MIME_TYPES_PDF),
             FileSizeLimit(100 * 1024 * 1024)
         ],
         fieldset=_('Related link')

src/onegov/file/utils.py

@@ -74,15 +74,8 @@ def get_supported_image_mime_types() -> set[str]:
 
     # Not all PIL formats register a mime type, fill in the blanks ourselves.
     supported_types = {
-        'image/bmp',
-        'image/x-bmp',
         'image/x-MS-bmp',
-        'image/x-icon',
-        'image/x-ico',
-        'image/x-win-bitmap',
-        'image/x-pcx',
-        'image/x-portable-pixmap',
-        'image/x-tga'
+        'image/x-xcf',
     }
 
     for mime in Image.MIME.values():

src/onegov/form/fields.py

@@ -20,7 +20,7 @@
 from onegov.file.utils import IMAGE_MIME_TYPES_AND_SVG
 from onegov.form import log, _
 from onegov.form.utils import path_to_filename
-from onegov.form.validators import ValidPhoneNumber
+from onegov.form.validators import ValidPhoneNumber, WhitelistedMimeType
 from onegov.form.widgets import ChosenSelectWidget
 from onegov.form.widgets import LinkPanelWidget
 from onegov.form.widgets import DurationInput
@@ -260,6 +260,7 @@ class UploadField(FileField):
     action: Literal['keep', 'replace', 'delete']
     file: IO[bytes] | None
     filename: str | None
+    validators = [WhitelistedMimeType()]
 
     if TYPE_CHECKING:
         def __init__(
@@ -448,6 +449,7 @@ def _add_entry(self, d: _MultiDictLikeWithGetlist, /) -> UploadField:
 
     upload_field_class: type[UploadField] = UploadField
     upload_widget: Widget[UploadField] = UploadWidget()
+    validators = [WhitelistedMimeType()]
 
     def __init__(
         self,
@@ -479,11 +481,11 @@ def __init__(
 
         # a lot of the arguments we just pass through to the subfield
         unbound_field = self.upload_field_class(
-            validators=validators,  # type:ignore[arg-type]
             filters=filters,
             description=description,
             widget=upload_widget,
             render_kw=render_kw,
+            validators=validators,  # type:ignore[arg-type]
             **extra_arguments
         )
         super().__init__(

src/onegov/form/validators.py

@@ -12,6 +12,8 @@
 from decimal import Decimal
 from dateutil.relativedelta import relativedelta
 from mimetypes import types_map
+
+from onegov.file.utils import get_supported_image_mime_types
 from onegov.form import _
 from onegov.form.errors import (DuplicateLabelError, InvalidIndentSyntax,
                                 EmptyFieldsetError)
@@ -31,7 +33,8 @@
 from wtforms.validators import ValidationError
 
 
-from typing import Generic, TYPE_CHECKING
+from typing import Generic, TYPE_CHECKING, Any
+
 if TYPE_CHECKING:
     from collections.abc import Collection, Sequence
     from onegov.core.orm import Base
@@ -114,13 +117,85 @@ def __call__(self, form: Form, field: Field) -> None:
         if not field.data:
             return
 
-        if field.data.get('size', 0) > self.max_bytes:
+        if isinstance(field.data, list):  # UploadMultipleField
+            for data in field.data:
+                if not data:
+                    continue  # in case of file deletion
+
+                self.validate_filesize(field, data)
+
+        else:
+            self.validate_filesize(field, field.data)
+
+    def validate_filesize(self, field: Field, data: dict[Any, Any]) -> None:
+        if data.get('size', 0) > self.max_bytes:
             message = field.gettext(self.message).format(
                 humanize.naturalsize(self.max_bytes)
             )
             raise ValidationError(message)
 
 
+MIME_TYPES_PDF = {
+    'application/pdf',
+}
+
+# for now not allowed by default
+MIME_TYPES_JSON = {
+    'application/json',
+}
+
+MIME_TYPES_DOCUMENT = {
+    'application/msword',  # doc
+    'application/rtf',
+    *MIME_TYPES_PDF,
+    'application/vnd.ms-excel',  # xls
+    ('application/vnd.openxmlformats-officedocument.'
+     'presentationml.presentation'),  # pptx
+    ('application/vnd.openxmlformats-officedocument.'
+     'spreadsheetml.sheet'),  # xlsx
+    ('application/vnd.openxmlformats-officedocument.'
+     'wordprocessingml.document'),  # docx
+    'application/CDFV2',  # old ms office docs
+    'application/x-ole-storage',  # old ms office docs
+    'application/CDFV2-unknown'  # old ms office docs
+}
+
+MIME_TYPES_XML = {
+    'application/xml',
+}
+
+MIME_TYPES_ARCHIVE = {
+    'application/zip',
+}
+
+MIME_TYPES_TEXT_DATA = {
+    'text/csv',
+    'text/plain',
+}
+
+MIME_TYPES_IMAGE = {
+    # allowed types based on PIL
+    *get_supported_image_mime_types(),
+    'image/svg+xml',
+}
+
+MIME_TYPES_AUDIO = {
+    'audio/mp4',
+    'audio/mpeg',
+    'audio/wav',
+    'audio/webm',  # weba
+}
+
+MIME_TYPES_VIDEO = {
+    'video/mp4',
+    'video/mpeg',  # mpg, mpeg
+    'video/ogg',
+    'video/quicktime',  # mov
+    'video/webm',  # webm
+    'video/x-msvideo',  # avi
+}
+
+
 class WhitelistedMimeType:
     """ Makes sure an uploaded file is in a whitelist of allowed mimetypes.
 
@@ -129,17 +204,13 @@ class WhitelistedMimeType:
     """
 
     whitelist: Collection[str] = {
-        'application/excel',
-        'application/vnd.ms-excel',
-        'application/msword',
-        'application/pdf',
-        'application/zip',
-        'image/gif',
-        'image/jpeg',
-        'image/png',
-        'image/x-ms-bmp',
-        'text/plain',
-        'text/csv'
+        *MIME_TYPES_DOCUMENT,
+        *MIME_TYPES_XML,
+        *MIME_TYPES_ARCHIVE,
+        *MIME_TYPES_TEXT_DATA,
+        *MIME_TYPES_IMAGE,
+        *MIME_TYPES_AUDIO,
+        *MIME_TYPES_VIDEO,
     }
 
     message = _('Files of this type are not supported.')
@@ -152,8 +223,20 @@ def __call__(self, form: Form, field: Field) -> None:
         if not field.data:
             return
 
-        if field.data['mimetype'] not in self.whitelist:
-            raise ValidationError(field.gettext(self.message))
+        if isinstance(field.data, list):  # UploadMultipleField
+            for data in field.data:
+                if not data:
+                    continue  # in case of file deletion
+
+                self.validate_mimetype(field, data)
+
+        else:
+            self.validate_mimetype(field, field.data)
+
+    def validate_mimetype(self, field: Field, data: dict[Any, Any]) -> None:
+        if data['mimetype'] not in self.whitelist:
+            message = field.gettext(self.message)
+            raise ValidationError(field.gettext(message))
 
 
 class ExpectedExtensions(WhitelistedMimeType):

src/onegov/landsgemeinde/forms/agenda.py

@@ -19,8 +19,12 @@
 from onegov.form.fields import TimeField
 from onegov.form.fields import UploadField
 from onegov.form.forms import NamedFileForm
-from onegov.form.validators import FileSizeLimit
-from onegov.form.validators import WhitelistedMimeType
+from onegov.form.validators import (
+    FileSizeLimit,
+    MIME_TYPES_PDF,
+    MIME_TYPES_ARCHIVE,
+    WhitelistedMimeType
+)
 from onegov.landsgemeinde import _
 from onegov.landsgemeinde.layouts import DefaultLayout
 from onegov.landsgemeinde.models import AgendaItem, LandsgemeindeFile
@@ -80,7 +84,7 @@ class AgendaItemForm(NamedFileForm):
         label=_('Excerpt from the Memorial (PDF)'),
         fieldset=_('Memorial'),
         validators=[
-            WhitelistedMimeType({'application/pdf'}),
+            WhitelistedMimeType(MIME_TYPES_PDF),
             FileSizeLimit(100 * 1024 * 1024)
         ]
     )
@@ -226,7 +230,7 @@ class AgendaItemUploadForm(Form):
         label=_('Agenda Item ZIP'),
         fieldset=_('Import'),
         validators=[
-            WhitelistedMimeType({'application/zip'}),
+            WhitelistedMimeType(MIME_TYPES_ARCHIVE),
             FileSizeLimit(100 * 1024 * 1024)
         ]
     )