ci: merge main to release

.github/workflows/dev-assets-sync-nightly.yml

@@ -47,17 +47,3 @@ jobs:
           file: dev/shared-assets-sync/Dockerfile
           push: true
           tags: ghcr.io/ietf-tools/datatracker-rsync-assets:latest
-
-  sync:
-    name: Run assets rsync
-    if: ${{ always() }}
-    runs-on: [self-hosted, dev-server]
-    needs: [build]
-    steps:
-    - name: Run rsync
-      env:
-        DEBIAN_FRONTEND: noninteractive
-      run: |
-        docker pull ghcr.io/ietf-tools/datatracker-rsync-assets:latest
-        docker run --rm -v dt-assets:/assets ghcr.io/ietf-tools/datatracker-rsync-assets:latest
-        docker image prune -a -f

README.md

@@ -5,10 +5,10 @@
 [![Release](https://img.shields.io/github/release/ietf-tools/datatracker.svg?style=flat&maxAge=300)](https://github.com/ietf-tools/datatracker/releases)
 [![License](https://img.shields.io/github/license/ietf-tools/datatracker)](https://github.com/ietf-tools/datatracker/blob/main/LICENSE)
 [![Code Coverage](https://codecov.io/gh/ietf-tools/datatracker/branch/feat/bs5/graph/badge.svg?token=V4DXB0Q28C)](https://codecov.io/gh/ietf-tools/datatracker)  
-[![Python Version](https://img.shields.io/badge/python-3.9-blue?logo=python&logoColor=white)](#prerequisites)
+[![Python Version](https://img.shields.io/badge/python-3.12-blue?logo=python&logoColor=white)](#prerequisites)
 [![Django Version](https://img.shields.io/badge/django-4.x-51be95?logo=django&logoColor=white)](#prerequisites)
 [![Node Version](https://img.shields.io/badge/node.js-16.x-green?logo=node.js&logoColor=white)](#prerequisites)
-[![MariaDB Version](https://img.shields.io/badge/postgres-16-blue?logo=postgresql&logoColor=white)](#prerequisites)
+[![MariaDB Version](https://img.shields.io/badge/postgres-17-blue?logo=postgresql&logoColor=white)](#prerequisites)
 
 ##### The day-to-day front-end to the IETF database for people who work on IETF standards.
 

ietf/api/urls.py

@@ -49,6 +49,9 @@
     url(r'^group/role-holder-addresses/$', api_views.role_holder_addresses),
     # Let IESG members set positions programmatically
     url(r'^iesg/position', views_ballot.api_set_position),
+    # Find the blob to store for a given materials document path
+    url(r'^meeting/(?:(?P<num>(?:interim-)?[a-z0-9-]+)/)?materials/%(document)s(?P<ext>\.[A-Za-z0-9]+)?/resolve-cached/$' % settings.URL_REGEXPS, meeting_views.api_resolve_materials_name_cached),
+    url(r'^meeting/blob/(?P<bucket>[a-z0-9-]+)/(?P<name>[a-z][a-z0-9.-]+)$', meeting_views.api_retrieve_materials_blob),
     # Let Meetecho set session video URLs
     url(r'^meeting/session/video/url$', meeting_views.api_set_session_video_url),
     # Let Meetecho tell us the name of its recordings

ietf/blobdb/admin.py

@@ -3,7 +3,7 @@
 from django.db.models.functions import Length
 from rangefilter.filters import DateRangeQuickSelectListFilterBuilder
 
-from .models import Blob
+from .models import Blob, ResolvedMaterial
 
 
 @admin.register(Blob)
@@ -29,3 +29,12 @@ def get_queryset(self, request):
     def object_size(self, instance):
         """Get the size of the object"""
         return instance.object_size  # annotation added in get_queryset()
+
+
+@admin.register(ResolvedMaterial)
+class ResolvedMaterialAdmin(admin.ModelAdmin):
+    model = ResolvedMaterial
+    list_display = ["name", "meeting_number", "bucket", "blob"]
+    list_filter = ["meeting_number", "bucket"]
+    search_fields = ["name", "blob"]
+    ordering = ["name"]

ietf/blobdb/migrations/0002_resolvedmaterial.py

@@ -0,0 +1,48 @@
+# Copyright The IETF Trust 2025, All Rights Reserved
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("blobdb", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="ResolvedMaterial",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("name", models.CharField(help_text="Name to resolve", max_length=300)),
+                (
+                    "meeting_number",
+                    models.CharField(
+                        help_text="Meeting material is related to", max_length=64
+                    ),
+                ),
+                (
+                    "bucket",
+                    models.CharField(help_text="Resolved bucket name", max_length=255),
+                ),
+                (
+                    "blob",
+                    models.CharField(help_text="Resolved blob name", max_length=300),
+                ),
+            ],
+        ),
+        migrations.AddConstraint(
+            model_name="resolvedmaterial",
+            constraint=models.UniqueConstraint(
+                fields=("name", "meeting_number"), name="unique_name_per_meeting"
+            ),
+        ),
+    ]

ietf/blobdb/models.py

@@ -96,3 +96,23 @@ def _emit_blob_change_event(self, using=None):
             ),
             using=using,
         )
+
+
+class ResolvedMaterial(models.Model):
+    # A Document name can be 255 characters; allow this name to be a bit longer
+    name = models.CharField(max_length=300, help_text="Name to resolve")
+    meeting_number = models.CharField(
+        max_length=64, help_text="Meeting material is related to"
+    )
+    bucket = models.CharField(max_length=255, help_text="Resolved bucket name")
+    blob = models.CharField(max_length=300, help_text="Resolved blob name")
+
+    class Meta:
+        constraints = [
+            models.UniqueConstraint(
+                fields=["name", "meeting_number"], name="unique_name_per_meeting"
+            )
+        ]
+
+    def __str__(self):
+        return f"{self.name}@{self.meeting_number} -> {self.bucket}:{self.blob}"

ietf/doc/models.py

@@ -913,6 +913,7 @@ def role_for_doc(self):
             roles.append('Action Holder')
         return ', '.join(roles) 
 
+# N.B., at least a couple dozen documents exist that do not satisfy this validator
 validate_docname = RegexValidator(
     r'^[-a-z0-9]+$',
     "Provide a valid document name consisting of lowercase letters, numbers and hyphens.",
@@ -1588,9 +1589,17 @@ class BofreqResponsibleDocEvent(DocEvent):
     """ Capture the responsible leadership (IAB and IESG members) for a BOF Request """
     responsible = models.ManyToManyField('person.Person', blank=True)
 
+
+class StoredObjectQuerySet(models.QuerySet):
+    def exclude_deleted(self):
+        return self.filter(deleted__isnull=True)
+
+
 class StoredObject(models.Model):
     """Hold metadata about objects placed in object storage"""
 
+    objects = StoredObjectQuerySet.as_manager() 
+
     store = models.CharField(max_length=256)
     name = models.CharField(max_length=1024, null=False, blank=False) # N.B. the 1024 limit on name comes from S3
     sha384 = models.CharField(max_length=96)

ietf/doc/storage.py

@@ -32,7 +32,7 @@ def __init__(self, file, name, mtime=None, content_type="", store=None, doc_name
     @classmethod
     def from_storedobject(cls, file, name, store):
         """Alternate constructor for objects that already exist in the StoredObject table"""
-        stored_object = StoredObject.objects.filter(store=store, name=name, deleted__isnull=True).first()
+        stored_object = StoredObject.objects.exclude_deleted().filter(store=store, name=name).first()
         if stored_object is None:
             raise FileNotFoundError(f"StoredObject for {store}:{name} does not exist or was deleted")
         file = cls(file, name, store, doc_name=stored_object.doc_name, doc_rev=stored_object.doc_rev)
@@ -140,7 +140,11 @@ def _save_stored_object(self, name, content) -> StoredObject:
                 ),
             ),
         )
-        if not created:
+        if not created and (
+            record.sha384 != content.custom_metadata["sha384"]
+            or record.len != int(content.custom_metadata["len"])
+            or record.deleted is not None
+        ):
             record.sha384 = content.custom_metadata["sha384"]
             record.len = int(content.custom_metadata["len"])
             record.modified = now
@@ -160,7 +164,7 @@ def _delete_stored_object(self, name) -> Optional[StoredObject]:
         else:
             now = timezone.now()
             # Note that existing_record is a queryset that will have one matching object
-            existing_record.filter(deleted__isnull=True).update(deleted=now)
+            existing_record.exclude_deleted().update(deleted=now)
         return existing_record.first()
 
     def _save(self, name, content):

ietf/doc/storage_utils.py

@@ -12,6 +12,14 @@
 from ietf.utils.log import log
 
 
+class StorageUtilsError(Exception):
+    pass
+
+
+class AlreadyExistsError(StorageUtilsError):
+    pass
+
+
 def _get_storage(kind: str) -> Storage:
     if kind in settings.ARTIFACT_STORAGE_NAMES:
         return storages[kind]
@@ -70,7 +78,7 @@ def store_file(
             # debug.show('f"Asked to store {name} in {kind}: is_new={is_new}, allow_overwrite={allow_overwrite}"')
             if not allow_overwrite and not is_new:
                 debug.show('f"Failed to save {kind}:{name} - name already exists in store"')
-                raise RuntimeError(f"Failed to save {kind}:{name} - name already exists in store")
+                raise AlreadyExistsError(f"Failed to save {kind}:{name} - name already exists in store")
             new_name = _get_storage(kind).save(
                 name,
                 StoredObjectFile(
@@ -85,7 +93,7 @@ def store_file(
             if new_name != name:
                 complaint = f"Error encountered saving '{name}' - results stored in '{new_name}' instead."
                 debug.show("complaint")
-                raise RuntimeError(complaint)
+                raise StorageUtilsError(complaint)
         except Exception as err:
             log(f"Blobstore Error: Failed to store file {kind}:{name}: {repr(err)}")
             if settings.SERVER_MODE == "development":

ietf/doc/views_material.py

@@ -22,6 +22,7 @@
 from ietf.doc.utils import add_state_change_event, check_common_doc_name_rules
 from ietf.group.models import Group
 from ietf.group.utils import can_manage_materials
+from ietf.meeting.utils import resolve_uploaded_material
 from ietf.utils import log
 from ietf.utils.decorators import ignore_view_kwargs
 from ietf.utils.meetecho import MeetechoAPIError, SlidesManager
@@ -179,6 +180,9 @@ def edit_material(request, name=None, acronym=None, action=None, doc_type=None):
                             "There was an error creating a hardlink at %s pointing to %s: %s"
                             % (ftp_filepath, filepath, ex)
                         )
+                else:
+                    for meeting in set([s.meeting for s in doc.session_set.all()]):
+                        resolve_uploaded_material(meeting=meeting, doc=doc)
 
             if prev_rev != doc.rev:
                 e = NewRevisionDocEvent(type="new_revision", doc=doc, rev=doc.rev)

ietf/liaisons/forms.py

@@ -495,14 +495,18 @@ def set_from_fields(self):
             self.fields['from_groups'].initial = qs
 
         # Note that the IAB chair currently doesn't get to work with incoming liaison statements
-        if not (
-            has_role(self.user, "Secretariat")
-            or has_role(self.user, "Liaison Coordinator")
-        ):
-            self.fields["from_contact"].initial = (
-                self.person.role_set.filter(group=qs[0]).first().email.formatted_email()
-            )
-            self.fields["from_contact"].widget.attrs["disabled"] = True
+
+        # Removing this block at the request of the IAB - as a workaround until the new liaison tool is
+        # create, anyone with access to the form can set any from_contact value
+        #
+        # if not (
+        #     has_role(self.user, "Secretariat")
+        #     or has_role(self.user, "Liaison Coordinator")
+        # ):
+        #     self.fields["from_contact"].initial = (
+        #         self.person.role_set.filter(group=qs[0]).first().email.formatted_email()
+        #     )
+        #     self.fields["from_contact"].widget.attrs["disabled"] = True
 
     def set_to_fields(self):
         '''Set to_groups and to_contacts options and initial value based on user

ietf/meeting/resources.py

@@ -11,11 +11,15 @@
 
 from ietf import api
 
-from ietf.meeting.models import ( Meeting, ResourceAssociation, Constraint, Room, Schedule, Session,
-                                TimeSlot, SchedTimeSessAssignment, SessionPresentation, FloorPlan,
-                                UrlResource, ImportantDate, SlideSubmission, SchedulingEvent,
-                                BusinessConstraint, ProceedingsMaterial, MeetingHost, Attended,
-                                Registration, RegistrationTicket)
+from ietf.meeting.models import (Meeting, ResourceAssociation, Constraint, Room,
+                                 Schedule, Session,
+                                 TimeSlot, SchedTimeSessAssignment, SessionPresentation,
+                                 FloorPlan,
+                                 UrlResource, ImportantDate, SlideSubmission,
+                                 SchedulingEvent,
+                                 BusinessConstraint, ProceedingsMaterial, MeetingHost,
+                                 Attended,
+                                 Registration, RegistrationTicket)
 
 from ietf.name.resources import MeetingTypeNameResource
 class MeetingResource(ModelResource):