Merge develop to main

.github/workflows/main.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.9]
+        python-version: [3.12]
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/tests.yml

@@ -20,18 +20,28 @@ jobs:
           pip freeze | tee pip_freeze.log
       - name: Unit tests
         run: python3 manage.py test -v2
+
       - name: Regression tests
+        id: regression_tests
         run: bash RegressionTests/run.sh
         # Continue even if tests fail, so that we can collect test outputs for debugging
         continue-on-error: true
+
       - name: Collect outputs
+        # This step will run even if the regression tests failed
         run: |
           find . -type f \( -name "*.log" -o -name "*.out" -o -name "*.diff" \) -print | cut -c3- > listing.txt
           echo "Creating an artifact with the following files:"
           cat listing.txt
           7z a -tzip regression-tests-appraise.zip @listing.txt
+
       - name: Publish outputs
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: regression-tests-appraise
           path: regression-tests-appraise.zip
+
+      # Enforce the failure
+      - name: Check on failures
+        if: steps.regression_tests.outcome == 'failure'
+        run: exit 1

Appraise/settings.py

@@ -9,6 +9,7 @@
 For the full list of settings and their values, see
 https://docs.djangoproject.com/en/1.11/ref/settings/
 """
+
 import logging
 import os
 import warnings
@@ -37,7 +38,9 @@
 
 ALLOWED_HOSTS = os.environ.get('APPRAISE_ALLOWED_HOSTS', '127.0.0.1').split(',')
 
-CSRF_TRUSTED_ORIGINS = os.environ.get('APPRAISE_CSRF_TRUSTED_ORIGINS', 'https://*.127.0.0.1').split(',')
+CSRF_TRUSTED_ORIGINS = os.environ.get(
+    'APPRAISE_CSRF_TRUSTED_ORIGINS', 'https://*.127.0.0.1'
+).split(',')
 
 WSGI_APPLICATION = os.environ.get(
     'APPRAISE_WSGI_APPLICATION', 'Appraise.wsgi.application'
@@ -208,7 +211,7 @@
 
 # Base context for all views.
 BASE_CONTEXT = {
-    'commit_tag': '#wmt24dev',
+    'commit_tag': '#wmt25dev',
     'title': 'Appraise evaluation system',
     'static_url': STATIC_URL,
 }

Appraise/urls.py

@@ -3,6 +3,7 @@
 
 See LICENSE for usage details
 """
+
 # pylint: disable=unused-import,import-error
 from django.conf.urls import handler404
 from django.conf.urls import handler500
@@ -187,8 +188,8 @@
         name='pairwise-assessment-document',
     ),
     re_path(
-        r'^campaign-status/(?P<campaign_name>[a-zA-Z0-9]+)/'
-        r'(?P<sort_key>[0123456])?/?$',
+        r'^campaign-status/(?P<campaign_name>[a-zA-Z0-9]+(,[a-zA-Z0-9]+)*)/'
+        r'(?P<sort_key>[a-zA-Z0-9_])?/?$',
         campaign_views.campaign_status,
         name='campaign_status',
     ),

Appraise/utils.py

@@ -3,6 +3,7 @@
 
 See LICENSE for usage details
 """
+
 import logging
 
 from Appraise.settings import LOG_HANDLER
@@ -33,8 +34,8 @@ def _compute_user_total_annotation_time(timestamps):
     def _clamp_time(seconds):
         # if a segment takes longer than 10 minutes, set it to 5 minutes
         # it's likely due to inactivity
-        if seconds >= 10*60:
-            return 5*60
+        if seconds >= 10 * 60:
+            return 5 * 60
         else:
             return seconds
 
@@ -54,4 +55,4 @@ def _clamp_time(seconds):
         # Update the previous end timestamp
         previous_end_timestamp = end_timestamp
 
-    return total_annotation_time
+    return total_annotation_time

Appraise/wsgi.py

@@ -6,6 +6,7 @@
 For more information on this file, see
 https://docs.djangoproject.com/en/1.11/howto/deployment/wsgi/
 """
+
 import os
 
 from django.core.wsgi import get_wsgi_application

Campaign/admin.py

@@ -1,6 +1,7 @@
 """
 Campaign admin.py
 """
+
 # pylint: disable=C0330,import-error
 from django.contrib import admin
 from django.contrib.admin.filters import AllValuesFieldListFilter
@@ -10,14 +11,18 @@
 from Campaign.models import CampaignTeam
 from Campaign.models import TrustedUser
 from EvalData.admin import BaseMetadataAdmin
-
+from django.http import HttpResponse
+import csv
+import zipfile
+from io import StringIO
+import importlib
 
 class DropdownFilter(AllValuesFieldListFilter):
     """
     Experimental dropdown filter.
     """
 
-    template = 'Campaign/filter_select.html'
+    template = "Campaign/filter_select.html"
 
 
 class CampaignTeamAdmin(BaseMetadataAdmin):
@@ -26,33 +31,33 @@ class CampaignTeamAdmin(BaseMetadataAdmin):
     """
 
     list_display = [
-        'teamName',
-        'owner',
-        'teamMembers',
-        'requiredAnnotations',
-        'requiredHours',
-        'completionStatus',
+        "teamName",
+        "owner",
+        "teamMembers",
+        "requiredAnnotations",
+        "requiredHours",
+        "completionStatus",
     ] + BaseMetadataAdmin.list_display  # type: ignore
-    list_filter = ['owner'] + BaseMetadataAdmin.list_filter  # type: ignore
+    list_filter = ["owner"] + BaseMetadataAdmin.list_filter  # type: ignore
     search_fields = [
-        'teamName',
-        'owner__username',
-        'owner__first_name',
-        'owner__last_name',
+        "teamName",
+        "owner__username",
+        "owner__first_name",
+        "owner__last_name",
     ] + BaseMetadataAdmin.search_fields  # type: ignore
 
-    filter_horizontal = ['members']
+    filter_horizontal = ["members"]
 
     fieldsets = (
         (
             None,
             {
-                'fields': (
-                    'teamName',
-                    'owner',
-                    'members',
-                    'requiredAnnotations',
-                    'requiredHours',
+                "fields": (
+                    "teamName",
+                    "owner",
+                    "members",
+                    "requiredAnnotations",
+                    "requiredHours",
                 )
             },
         ),
@@ -65,22 +70,22 @@ class CampaignDataAdmin(BaseMetadataAdmin):
     """
 
     list_display = [
-        'dataName',
-        'market',
-        'metadata',
-        'dataValid',
-        'dataReady',
+        "dataName",
+        "market",
+        "metadata",
+        "dataValid",
+        "dataReady",
     ] + BaseMetadataAdmin.list_display  # type: ignore
     list_filter = [
-        'dataValid',
-        'dataReady',
+        "dataValid",
+        "dataReady",
     ] + BaseMetadataAdmin.list_filter  # type: ignore
     search_fields = [
         # nothing model specific
     ] + BaseMetadataAdmin.search_fields  # type: ignore
 
     fieldsets = (
-        (None, {'fields': ('dataFile', 'market', 'metadata')}),
+        (None, {"fields": ("dataFile", "market", "metadata")}),
     ) + BaseMetadataAdmin.fieldsets  # type: ignore
 
 
@@ -89,47 +94,98 @@ class CampaignAdmin(BaseMetadataAdmin):
     Model admin for Campaign instances.
     """
 
-    list_display = ['campaignName'] + BaseMetadataAdmin.list_display + ['id']  # type: ignore
+    list_display = ["campaignName"] + \
+        BaseMetadataAdmin.list_display + ["id"]  # type: ignore
     list_filter = [
         # nothing model specific
     ] + BaseMetadataAdmin.list_filter  # type: ignore
     search_fields = [
         # nothing model specific
     ] + BaseMetadataAdmin.search_fields  # type: ignore
 
-    filter_horizontal = ['batches']
+    filter_horizontal = ["batches"]
 
     fieldsets = (
         (
             None,
             {
-                'fields': (
-                    'campaignName',
-                    'packageFile',
-                    'teams',
-                    'batches',
-                    'campaignOptions',
+                "fields": (
+                    "campaignName",
+                    "packageFile",
+                    "teams",
+                    "batches",
+                    "campaignOptions",
                 )
             },
         ),
     ) + BaseMetadataAdmin.fieldsets  # type: ignore
 
 
+    actions = ["export_results"]
+
+    def _retrieve_csv(self, current_campaign):
+        # Get the task type  corresponding to the campaign
+        qs_name = current_campaign.get_campaign_type().lower()
+        qs_attr = "evaldata_{0}_campaign".format(qs_name)
+        qs_obj = getattr(current_campaign, qs_attr, None)
+        cls = type(qs_obj.all()[0])
+        cls_name = cls.__name__
+        cls_name = cls_name.replace("Task", "Result")
+        module = importlib.import_module(cls.__module__)
+        cls = getattr(module, cls_name)
+
+        # Now get the content
+        f = StringIO()
+        writer = csv.writer(f)
+        csv_content = cls.get_system_data(current_campaign.id, extended_csv=True)
+        for r in csv_content:
+            writer.writerow(r)
+
+        f.seek(0)
+        return f
+
+
+    def export_results(self, request, queryset):
+        if len(queryset) == 1:
+            current_campaign = queryset[0]
+            csv_content = self._retrieve_csv(current_campaign)
+            filename = f"results_{current_campaign.campaignName}.csv"
+            response = HttpResponse(csv_content, content_type="text/csv")
+            response["Content-Disposition"] = f"attachment; filename={filename}"
+        else:
+            response = HttpResponse(content_type='application/zip')
+            response['Content-Disposition'] = 'attachment; filename="campaign_results.zip"'
+
+            # Create a zip file with selected objects
+            with zipfile.ZipFile(response, 'w') as zipf:
+                for current_campaign in queryset:
+
+                    csv_content = self._retrieve_csv(current_campaign)
+                    # Add objects to the zip file, customize as per your model's data
+                    # For example, you can add an object's name and description to a text file in the zip
+                    filename = f"results_{current_campaign.campaignName}.csv"
+                    zipf.writestr(filename, csv_content.getvalue())
+        return response
+
+    export_results.short_description = "Download results"
+
+
+
 class TrustedUserAdmin(admin.ModelAdmin):
     """
     Model admin for Campaign instances.
     """
 
-    list_display = ['user', 'campaign']
+    list_display = ["user", "campaign"]
     list_filter = [
-        ('campaign__campaignName', DropdownFilter),
-        #      'campaign'
+        ("campaign__campaignName", DropdownFilter),
+        #      "campaign"
     ]
     search_fields = [  # type: ignore
         # nothing model specific
     ]
 
-    fieldsets = ((None, {'fields': ('user', 'campaign')}),)
+    fieldsets = ((None, {"fields": ("user", "campaign")}),)
 
 
 admin.site.register(CampaignTeam, CampaignTeamAdmin)

Campaign/management/commands/ComputeSystemScores.py

@@ -10,6 +10,7 @@
 from EvalData.models import DirectAssessmentResult
 from EvalData.models import DirectAssessmentTask
 
+
 # pylint: disable=C0111,C0330,E1101
 class Command(BaseCommand):
     help = 'Computes system scores over all results'

Campaign/management/commands/ComputeWMT21Results.py

@@ -463,7 +463,7 @@ def handle(self, *args, **options):
             wins_for_system = defaultdict(list)
             losses_for_system = defaultdict(list)
             p_level = 0.05
-            for (sysA, sysB) in combinations_with_replacement(system_ids, 2):
+            for sysA, sysB in combinations_with_replacement(system_ids, 2):
                 sysA_ids = set([x[0] for x in system_z_scores[sysA]])
                 sysB_ids = set([x[0] for x in system_z_scores[sysB]])
                 good_ids = set.intersection(sysA_ids, sysB_ids)

Campaign/management/commands/ComputeZScores.py

@@ -427,7 +427,7 @@ def handle(self, *args, **options):
 
             wins_for_system = defaultdict(list)
             p_level = 0.05
-            for (sysA, sysB) in combinations_with_replacement(system_ids, 2):
+            for sysA, sysB in combinations_with_replacement(system_ids, 2):
                 sysA_ids = set([x[0] for x in system_z_scores[sysA]])
                 sysB_ids = set([x[0] for x in system_z_scores[sysB]])
                 good_ids = set.intersection(sysA_ids, sysB_ids)
@@ -577,7 +577,7 @@ def sort_by_wins_and_z_score(x, y):
                 key = system_id[:4].upper()
                 vsystems[key].extend(system_z_scores[system_id])
 
-            for (sysA, sysB) in combinations_with_replacement(
+            for sysA, sysB in combinations_with_replacement(
                 ['GOOG', 'CAND', 'PROD'], 2
             ):
                 sysA_scores = [x[1] for x in vsystems[sysA]]

Campaign/management/commands/InitCampaignMMT18Task1.py

@@ -31,6 +31,7 @@
 }
 REDUNDANCY = 1
 
+
 # pylint: disable=C0111,C0330,E1101
 class Command(BaseCommand):
     help = 'Initialises campaign MMT18 Task #1'

Campaign/management/commands/InitCampaignMMT18Task1b.py

@@ -27,6 +27,7 @@
 }
 REDUNDANCY = 1
 
+
 # pylint: disable=C0111,C0330,E1101
 class Command(BaseCommand):
     help = 'Initialises campaign MMT18 Task #1.b'