Preparation for WMT25

EvalData/models/direct_assessment_document.py

@@ -255,11 +255,12 @@ def next_document_for_user_mqmesa(self, user):
         Used for MQM/ESA views
         Specifically a tuple with:
             next_item,
-            completed_items,
-            completed_docs,
+            items_completed,
+            items_total,
+            docs_completed,
+            docs_total,
             doc_items,
             doc_items_results,
-            total_docs,
         """
 
         # get all items (100) and try to find resul
@@ -274,16 +275,19 @@ def next_document_for_user_mqmesa(self, user):
         ]
         unfinished_items = [i for i, r in all_items if not r]
 
-        docs_total = len({i.documentID for i, r in all_items})
+        # documentID + targetID uniquely identifies documents
+        docs_total = len({(i.documentID, i.targetID) for i, r in all_items})
         items_completed = len([i for i, r in all_items if r and r.completed])
         docs_completed = docs_total - len(
-            {i.documentID for i, r in all_items if r is None or not r.completed}
+            {(i.documentID, i.targetID) for i, r in all_items if r is None or not r.completed}
         )
+        items_total = len(all_items)
 
         if not unfinished_items:
             return (
                 None,
                 items_completed,
+                items_total,
                 docs_completed,
                 [],
                 [],
@@ -309,10 +313,11 @@ def next_document_for_user_mqmesa(self, user):
         return (
             next_item,  # the first unannotated item for the user
             items_completed,  # the number of completed items in the task
+            items_total,
             docs_completed,  # the number of completed documents in the task
+            docs_total,  # the total number of documents in the task
             doc_items,  # all items from the current document
             doc_items_results,  # all score results from the current document
-            docs_total,  # the total number of documents in the task
         )
 
     def get_results_for_each_item(self, block_items, user):

EvalView/static/EvalView/css/direct-assessment-document-mqm-esa.css

@@ -73,6 +73,13 @@
     width: 100%;
 }
 
+.source-text > img {
+    display: block;
+    margin-left: auto;
+    margin-right: auto;
+    width: 45%;
+}
+
 .tutorial-text {
     text-align: center;
     color: #257;

EvalView/static/EvalView/js/direct-assessment-document-mqm-esa.js

@@ -75,6 +75,8 @@ const ERROR_TYPES = {
     },
     "Other": {},
 }
+
+
 Object.keys(SEVERITY_TO_COLOR).map((key) => {
     $(`#instruction_sev_${key}`).css("background-color", SEVERITY_TO_COLOR[key])
 })
@@ -311,8 +313,14 @@ class MQMItemHandler {
         }
         this.mqm_submitted = structuredClone(this.mqm)
         this.mqm_orig = JSON.parse(JSON.parse(this.el.children('#mqm-payload-orig').html()))
-        this.text_source_orig = decodeEntities(JSON.parse(this.el.children('#text-source-payload').html()).trim())
-        this.source_video = JSON.parse(this.el.children('#text-source-payload').html()).trim().startsWith("<video")
+
+        let _src_raw = JSON.parse(this.el.children('#text-source-payload').html()).trim()
+        this.text_source_orig = decodeEntities(_src_raw)
+        this.source_is_multimodal = (
+            _src_raw.startsWith("<video") ||
+            _src_raw.startsWith("<audio") ||
+            _src_raw.startsWith("<img")
+        )
         // NOTE: we don't decode entities for the target text, which might cause false positive annotated errors
         this.text_target_orig = JSON.parse(this.el.children('#text-target-payload').html()).trim()
         this.SELECTION_STATE = []
@@ -335,9 +343,11 @@ class MQMItemHandler {
         let score = parseFloat(this.el.children('#score-payload').html())
 
 
-
         // setup_span_structure
         let html_target = this.text_target_orig.split("").map((v, i) => {
+            if (v == "\n") {
+                return "<br>" // preserve newlines
+            }
             return `<span class="mqm_char" id="target_char_${i}" char_id="${i}">${v}</span>`
         }).join("") + " <span class='mqm_char span_missing' id='target_char_missing' char_id='missing'>[MISSING]</span>"
         this.el_target.html(html_target)
@@ -357,8 +367,11 @@ class MQMItemHandler {
         }
 
         // handle character alignment estimation
-        if (!this.source_video) {
+        if (!this.source_is_multimodal) {
             let html_source = this.text_source_orig.split("").map((v, i) => {
+                if (v == "\n") {
+                    return "<br>" // preserve newlines
+                }
                 return `<span class="mqm_char_src" id="source_char_${i}" char_id="${i}">${v}</span>`
             }).join("")
             this.el_source.html(html_source)

EvalView/templates/EvalView/_instructions-esa.html

@@ -1,13 +1,13 @@
 <div class="row">
   <div class="col-md-12">
     <ul class="list-unstyled">
-      <li>Highligh errors:
+      <li><strong>Highlighting errors:</strong>
         <ul>
           <li><strong>Select</strong> the part of translation where you have identified a <strong>translation error</strong> (drag or click start & end).</li>
           <li><strong>Click</strong> on the highlight to change error severity (minor/major) or remove the highlight.</li>
         </ul>
       </li> 
-      <li>Choose error severity:
+      <li><strong>Choose error severity:</strong>
         <ul>
           <li><span class="instruction_sev" id="instruction_sev_minor">Minor errors:</span> Style, grammar, word choice could be better or more natural. </li>
           <li><span class="instruction_sev" id="instruction_sev_major">Major errors:</span>: The meaning is changed significantly and/or the part is really hard to understand. </li>

EvalView/templates/EvalView/direct-assessment-document-mqm-esa.html

@@ -22,7 +22,7 @@
             <td style="width:33%;text-align:left;">
                 <strong id="task_progress">
                     Completed {{docs_completed}}/{{docs_total}} documents,
-                    {{items_completed}}/100 segments
+                    {{items_completed}}/{{items_total}} segments
                 </strong>
             </td>
             <td style="width:33%;text-align:center;">

EvalView/views.py

@@ -1133,10 +1133,11 @@ def direct_assessment_document_mqmesa(campaign, current_task, request):
     (
         next_item,
         items_completed,
+        items_total,
         docs_completed,
+        docs_total,
         doc_items,
         doc_items_results,
-        docs_total,
     ) = current_task.next_document_for_user_mqmesa(request.user)
 
     if not next_item:
@@ -1151,11 +1152,15 @@ def direct_assessment_document_mqmesa(campaign, current_task, request):
             # Send response to the Ajax POST request
             return JsonResponse(context)
 
-    # TODO: hotfix for WMT24
+    # TODO: hotfix for WMT24 and WMT25
     # Tracking issue: https://github.com/AppraiseDev/Appraise/issues/185
     for item in doc_items:
-        # don't escape HTML video or images
-        if item.sourceText.strip().startswith("<video") or item.sourceText.strip().startswith("<img"):
+        # don't escape HTML video, audio or images
+        if (
+            item.sourceText.strip().startswith("<video") or
+            item.sourceText.strip().startswith("<audio") or
+            item.sourceText.strip().startswith("<img")
+        ):
             continue
         item.sourceText = escape(item.sourceText)
 
@@ -1201,6 +1206,7 @@ def direct_assessment_document_mqmesa(campaign, current_task, request):
         'task_id': next_item.id,
         'document_id': next_item.documentID,
         'items_completed': items_completed,
+        'items_total': items_total,
         'docs_completed': docs_completed,
         'docs_total': docs_total,
         'source_language': source_language,