From 0e4a0323918b5f2b215ea2991f64791482ae712d Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 26 Sep 2023 10:44:51 -0300 Subject: [PATCH 01/64] feat: Add rpc_person API call --- ietf/api/urls.py | 1 + ietf/api/views.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/ietf/api/urls.py b/ietf/api/urls.py index 7ee55cf708..21e4ec4d6d 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -60,6 +60,7 @@ url(r'^rfcdiff-latest-json/(?P[Rr][Ff][Cc] [0-9]+?)(\.txt|\.html)?/?$', api_views.rfcdiff_latest_json), # direct authentication url(r'^directauth/?$', api_views.directauth), + url(r'^rpc/person/(?P[0-9]+)$', api_views.rpc_person), ] # Additional (standard) Tastypie endpoints diff --git a/ietf/api/views.py b/ietf/api/views.py index f6221b5e2e..868fef2505 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -14,7 +14,7 @@ from django.contrib.auth.models import User from django.core.exceptions import ValidationError from django.core.validators import validate_email -from django.http import HttpResponse, Http404 +from django.http import HttpResponse, Http404, JsonResponse, HttpResponseForbidden from django.shortcuts import render, get_object_or_404 from django.urls import reverse from django.utils.decorators import method_decorator @@ -435,3 +435,14 @@ def directauth(request): else: return HttpResponse(status=405) + +@csrf_exempt +def rpc_person(request, person_id): + authtoken = request.META.get("HTTP_X_API_KEY", None) + if authtoken is None or not is_valid_token("ietf.api.views.rpc_person", authtoken): + return HttpResponseForbidden() + person = get_object_or_404(Person, pk=person_id) + return JsonResponse({ + "id": person.id, + "plain_name": person.plain_name(), + }) From e92136cb41ed9526f86eba1227830c69e017da2d Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 26 Sep 2023 10:46:02 -0300 Subject: [PATCH 02/64] chore: Add OpenAPI yaml for RPC API --- rpcapi.yaml | 68 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 rpcapi.yaml diff --git a/rpcapi.yaml b/rpcapi.yaml new file mode 100644 index 0000000000..9c02e12c72 --- /dev/null +++ b/rpcapi.yaml @@ -0,0 +1,68 @@ +openapi: 3.0.3 +info: + title: Datatracker RPC API + description: Datatracker RPC API + version: 1.0.0 +servers: + - url: 'http://localhost:8000/api/rpc/' +paths: + /person/{personId}: + get: + operationId: get_person_by_id + summary: Find person by ID + description: Returns a single person + parameters: + - name: personId + in: path + description: ID of person to return + required: true + schema: + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/Person' + +# /subject/person{subjectId}: +# get: +# operationId: get_subject_person_by_id +# summary: Find person for a subject by ID +# description: Returns a single person +# parameters: +# - name: subjectId +# in: path +# description: subject ID of person to return +# required: true +# schema: +# type: string +# responses: +# '200': +# description: OK +# content: +# application/json: +# schema: +# $ref: '#/components/schemas/Person' + +components: + schemas: + Person: + type: object + properties: + id: + type: integer + example: 1234 + plain_name: + type: string + example: John Doe + + securitySchemes: + ApiKeyAuth: + type: apiKey + in: header + name: X-API-KEY + +security: + - ApiKeyAuth: [] From 0e309d00d61917d01b06d77f0f957b7b3f2ee33d Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 26 Sep 2023 11:46:18 -0500 Subject: [PATCH 03/64] feat: api endpoint for docs submitted to the rpc --- docker/configs/settings_local.py | 5 +++++ ietf/api/urls.py | 2 ++ ietf/api/views.py | 24 ++++++++++++++++++++++++ ietf/doc/models.py | 9 +++++++++ rpcapi.yaml | 32 ++++++++++++++++++++++++++++++++ 5 files changed, 72 insertions(+) diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index 07c16c2e9a..7b4803c3c0 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -58,3 +58,8 @@ STATIC_IETF_ORG = "/_static" STATIC_IETF_ORG_INTERNAL = "http://static" + +APP_API_TOKENS = { + "ietf.api.views.rpc_person" : ["devtoken"], + "ietf.api.views.submitted_to_rpc" : ["devtoken"] +} diff --git a/ietf/api/urls.py b/ietf/api/urls.py index 21e4ec4d6d..5650a51449 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -61,6 +61,8 @@ # direct authentication url(r'^directauth/?$', api_views.directauth), url(r'^rpc/person/(?P[0-9]+)$', api_views.rpc_person), + url(r'^rpc/doc/submitted_to_rpc/$', api_views.submitted_to_rpc), + ] # Additional (standard) Tastypie endpoints diff --git a/ietf/api/views.py b/ietf/api/views.py index 868fef2505..46b53f5560 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -14,6 +14,7 @@ from django.contrib.auth.models import User from django.core.exceptions import ValidationError from django.core.validators import validate_email +from django.db.models import Q from django.http import HttpResponse, Http404, JsonResponse, HttpResponseForbidden from django.shortcuts import render, get_object_or_404 from django.urls import reverse @@ -34,6 +35,7 @@ from ietf.api import _api_list from ietf.api.serializer import JsonExportMixin from ietf.api.ietf_utils import is_valid_token +from ietf.doc.models import Document from ietf.doc.utils import fuzzy_find_documents from ietf.ietfauth.views import send_account_creation_email from ietf.ietfauth.utils import role_required @@ -446,3 +448,25 @@ def rpc_person(request, person_id): "id": person.id, "plain_name": person.plain_name(), }) + +@csrf_exempt +def submitted_to_rpc(request): + """ Return documents in datatracker that have been submitted to the RPC but are not yet in the queue + + Those queries overreturn - there may be things, particularly not from the IETF stream that are already in the queue. + """ + authtoken = request.META.get("HTTP_X_API_KEY", None) + if authtoken is None or not is_valid_token("ietf.api.views.submitted_to_rpc", authtoken): + return HttpResponseForbidden() + ietf_docs = Q(states__type_id="draft-iesg",states__slug__in=["ann"]) + irtf_iab_ise_docs = Q(states__type_id__in=["draft-stream-iab","draft-stream-irtf","draft-stream-ise"],states__slug__in=["rfc-edit"]) + #TODO: Need a way to talk about editorial stream docs + docs = Document.objects.filter(type_id="draft").filter(ietf_docs|irtf_iab_ise_docs) + response = {"submitted_to_rpc": []} + for doc in docs: + if not doc.sent_to_rfc_editor_event(): + debug.show("doc") + debug.show("doc.stream") + response["submitted_to_rpc"].append({"name":doc.name, "pk": doc.pk, "stream": doc.stream_id, "submitted": f"{doc.sent_to_rfc_editor_event().time:%Y-%m-%d}"}) #TODO reconcile timezone + + return JsonResponse(response) diff --git a/ietf/doc/models.py b/ietf/doc/models.py index e35eb12612..98f7744c07 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -660,6 +660,15 @@ def referenced_by(self): def referenced_by_rfcs(self): return self.relations_that(('refnorm','refinfo','refunk','refold')).filter(source__states__type__slug='draft',source__states__slug='rfc') + + def sent_to_rfc_editor_event(self): + if self.stream_id == "ietf": + return self.docevent_set.filter(type="iesg_approved").order_by("-time").first() + elif self.stream_id in ["iab", "irtf", "ise"]: + return self.docevent_set.filter(type="requested_publication").order_by("-time").first() + #elif self.stream_id == "editorial": #TODO + else: + return None class Meta: abstract = True diff --git a/rpcapi.yaml b/rpcapi.yaml index 9c02e12c72..887ee12373 100644 --- a/rpcapi.yaml +++ b/rpcapi.yaml @@ -26,6 +26,19 @@ paths: schema: $ref: '#/components/schemas/Person' + /doc/submitted_to_rpc: + get: + operationId: submitted_to_rpc + summary: List documents ready to enter the RFC Editor Queue + description: List documents ready to enter the RFC Editor Queue + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SubmittedToQueue' + # /subject/person{subjectId}: # get: # operationId: get_subject_person_by_id @@ -58,6 +71,25 @@ components: type: string example: John Doe + SubmittedToQueue: + type: object + properties: + submitted_to_rpc: + type: array + items: + type: object + properties: + name: + type: string + pk: + type: integer + stream: + type: string + submitted: + type: string + format: date + + securitySchemes: ApiKeyAuth: type: apiKey From 3998f34815ef8311cb16f025e3b1088511840b9a Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 26 Sep 2023 12:29:16 -0500 Subject: [PATCH 04/64] chore: remove some debug --- ietf/api/views.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ietf/api/views.py b/ietf/api/views.py index 46b53f5560..27d2457c3c 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -464,9 +464,6 @@ def submitted_to_rpc(request): docs = Document.objects.filter(type_id="draft").filter(ietf_docs|irtf_iab_ise_docs) response = {"submitted_to_rpc": []} for doc in docs: - if not doc.sent_to_rfc_editor_event(): - debug.show("doc") - debug.show("doc.stream") response["submitted_to_rpc"].append({"name":doc.name, "pk": doc.pk, "stream": doc.stream_id, "submitted": f"{doc.sent_to_rfc_editor_event().time:%Y-%m-%d}"}) #TODO reconcile timezone return JsonResponse(response) From 76f043ae0ea5ae2021f32f6a29ea29c440842750 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 26 Sep 2023 14:05:07 -0500 Subject: [PATCH 05/64] feat: api for created demo people --- docker/configs/settings_local.py | 3 ++- ietf/api/urls.py | 2 ++ ietf/api/views.py | 21 +++++++++++++++++++++ rpcapi.yaml | 27 +++++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 1 deletion(-) diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index 7b4803c3c0..eb2dfbbe7c 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -61,5 +61,6 @@ APP_API_TOKENS = { "ietf.api.views.rpc_person" : ["devtoken"], - "ietf.api.views.submitted_to_rpc" : ["devtoken"] + "ietf.api.views.submitted_to_rpc" : ["devtoken"], + "ietf.api.views.create_demo_person" : ["devtoken"] } diff --git a/ietf/api/urls.py b/ietf/api/urls.py index 5650a51449..6920b7364f 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -62,6 +62,8 @@ url(r'^directauth/?$', api_views.directauth), url(r'^rpc/person/(?P[0-9]+)$', api_views.rpc_person), url(r'^rpc/doc/submitted_to_rpc/$', api_views.submitted_to_rpc), + url(r'^rpc/person/create_demo_person/$', api_views.create_demo_person), + ] diff --git a/ietf/api/views.py b/ietf/api/views.py index 27d2457c3c..211a70ce78 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -31,6 +31,7 @@ import debug # pyflakes:ignore import ietf +from ietf.person.factories import PersonFactory from ietf.person.models import Person, Email from ietf.api import _api_list from ietf.api.serializer import JsonExportMixin @@ -467,3 +468,23 @@ def submitted_to_rpc(request): response["submitted_to_rpc"].append({"name":doc.name, "pk": doc.pk, "stream": doc.stream_id, "submitted": f"{doc.sent_to_rfc_editor_event().time:%Y-%m-%d}"}) #TODO reconcile timezone return JsonResponse(response) + +@csrf_exempt +def create_demo_person(request): + """ Helper for creating rpc demo objects - SHOULD NOT MAKE IT INTO PRODUCTION + + """ + authtoken = request.META.get("HTTP_X_API_KEY", None) + if authtoken is None or not is_valid_token("ietf.api.views.submitted_to_rpc", authtoken): + return HttpResponseForbidden() + if request.method != "POST": + return HttpResponseForbidden() + + request_params = json.loads(request.body) + name = request_params["name"] + if Person.objects.filter(name=name).exists(): + return HttpResponseForbidden() + person = PersonFactory(name=name) + + + return JsonResponse({"user_id":person.user.pk,"person_pk":person.pk}, status=201) diff --git a/rpcapi.yaml b/rpcapi.yaml index 887ee12373..1c6c7c7845 100644 --- a/rpcapi.yaml +++ b/rpcapi.yaml @@ -26,6 +26,33 @@ paths: schema: $ref: '#/components/schemas/Person' + /person/create_demo_person/: + post: + operationId: create_demo_person + summary: Build a datatraker Person for RPC demo purposes + description: returns a datatracker User id for a person created with the given name + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + name: + type: string + responses: + '201': + description: OK + content: + application/json: + schema: + type: object + properties: + user_id: + type: integer + person_pk: + type: integer + /doc/submitted_to_rpc: get: operationId: submitted_to_rpc From ec324e0b957ab1374af71dadf1ae2bade477c3f0 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 26 Sep 2023 16:41:12 -0500 Subject: [PATCH 06/64] feat: optimize fetching persons --- ietf/api/urls.py | 1 + ietf/api/views.py | 18 ++++++++++++++++++ rpcapi.yaml | 24 ++++++++++++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/ietf/api/urls.py b/ietf/api/urls.py index 6920b7364f..772cf59d09 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -61,6 +61,7 @@ # direct authentication url(r'^directauth/?$', api_views.directauth), url(r'^rpc/person/(?P[0-9]+)$', api_views.rpc_person), + url(r'^rpc/persons/$', api_views.rpc_persons), url(r'^rpc/doc/submitted_to_rpc/$', api_views.submitted_to_rpc), url(r'^rpc/person/create_demo_person/$', api_views.create_demo_person), diff --git a/ietf/api/views.py b/ietf/api/views.py index 211a70ce78..ea1aa5da33 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -450,6 +450,24 @@ def rpc_person(request, person_id): "plain_name": person.plain_name(), }) +@csrf_exempt +def rpc_persons(request): + """ Get a batch of rpc person names + + """ + authtoken = request.META.get("HTTP_X_API_KEY", None) + if authtoken is None or not is_valid_token("ietf.api.views.rpc_person", authtoken): + return HttpResponseForbidden() + if request.method != "POST": + return HttpResponseForbidden() + + pks = json.loads(request.body) + response = dict() + for p in Person.objects.filter(pk__in=pks): + response[str(p.pk)] = p.plain_name() + return JsonResponse(response) + + @csrf_exempt def submitted_to_rpc(request): """ Return documents in datatracker that have been submitted to the RPC but are not yet in the queue diff --git a/rpcapi.yaml b/rpcapi.yaml index 1c6c7c7845..f6a0db088b 100644 --- a/rpcapi.yaml +++ b/rpcapi.yaml @@ -26,6 +26,30 @@ paths: schema: $ref: '#/components/schemas/Person' + /persons/: + post: + operationId: get_persons + summary: Get a batch of persons + description: returns a dict of person pks to person names + requestBody: + required: true + content: + application/json: + schema: + type: array + items: + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + additionalProperties: + type: string + + /person/create_demo_person/: post: operationId: create_demo_person From f27ff983ec6e5fedb529a6a698669f9dec1faa87 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 28 Sep 2023 12:43:08 -0500 Subject: [PATCH 07/64] feat: api for creating demo drafts (#6402) --- ietf/api/urls.py | 1 + ietf/api/views.py | 35 +++++++++++++++++++++++++++++++++-- rpcapi.yaml | 36 +++++++++++++++++++++++++++++++++++- 3 files changed, 69 insertions(+), 3 deletions(-) diff --git a/ietf/api/urls.py b/ietf/api/urls.py index 772cf59d09..f53bcdce5e 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -64,6 +64,7 @@ url(r'^rpc/persons/$', api_views.rpc_persons), url(r'^rpc/doc/submitted_to_rpc/$', api_views.submitted_to_rpc), url(r'^rpc/person/create_demo_person/$', api_views.create_demo_person), + url(r'^rpc/doc/create_demo_draft/$', api_views.create_demo_draft), ] diff --git a/ietf/api/views.py b/ietf/api/views.py index ea1aa5da33..825b469f52 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -31,11 +31,12 @@ import debug # pyflakes:ignore import ietf -from ietf.person.factories import PersonFactory +from ietf.person.factories import PersonFactory # DO NOT MERGE INTO MAIN from ietf.person.models import Person, Email from ietf.api import _api_list from ietf.api.serializer import JsonExportMixin from ietf.api.ietf_utils import is_valid_token +from ietf.doc.factories import WgDraftFactory # DO NOT MERGE INTO MAIN from ietf.doc.models import Document from ietf.doc.utils import fuzzy_find_documents from ietf.ietfauth.views import send_account_creation_email @@ -505,4 +506,34 @@ def create_demo_person(request): person = PersonFactory(name=name) - return JsonResponse({"user_id":person.user.pk,"person_pk":person.pk}, status=201) + return JsonResponse({"user_id":person.user.pk,"person_pk":person.pk}) + +@csrf_exempt +def create_demo_draft(request): + """ Helper for creating rpc demo objects - SHOULD NOT MAKE IT INTO PRODUCTION + + """ + authtoken = request.META.get("HTTP_X_API_KEY", None) + if authtoken is None or not is_valid_token("ietf.api.views.submitted_to_rpc", authtoken): + return HttpResponseForbidden() + if request.method != "POST": + return HttpResponseForbidden() + + request_params = json.loads(request.body) + name = request_params.get("name") + states = request_params.get("states") + exists = False + doc = None + if not name: + return HttpResponse(status=400, content="Name is required") + doc = Document.objects.filter(name=name).first() + if doc: + exists = True + else: + kwargs = {"name": name} + if states: + kwargs["states"] = states + doc = WgDraftFactory(**kwargs) + return JsonResponse({ "doc_id":doc.pk, "name":doc.name }) + + diff --git a/rpcapi.yaml b/rpcapi.yaml index f6a0db088b..383fdb6541 100644 --- a/rpcapi.yaml +++ b/rpcapi.yaml @@ -65,7 +65,7 @@ paths: name: type: string responses: - '201': + '200': description: OK content: application/json: @@ -77,6 +77,40 @@ paths: person_pk: type: integer + /doc/create_demo_draft/: + post: + operationId: create_demo_draft + summary: Build a datatraker WG draft for RPC demo purposes + description: returns a datatracker document id for a draft created with the provided name and states. The arguments, if present, are passed directly to the WgDraftFactory + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + name: + type: string + states: + type: array + items: + type: array + items: + type: string + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + properties: + doc_id: + type: integer + name: + type: string +s + /doc/submitted_to_rpc: get: operationId: submitted_to_rpc From 9e9a787268d37e841ef7bda23c3b39923d6d0b22 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 28 Sep 2023 14:48:37 -0300 Subject: [PATCH 08/64] fix: Typo in rpcapi.yaml --- rpcapi.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/rpcapi.yaml b/rpcapi.yaml index 383fdb6541..6fe632438a 100644 --- a/rpcapi.yaml +++ b/rpcapi.yaml @@ -109,7 +109,6 @@ paths: type: integer name: type: string -s /doc/submitted_to_rpc: get: From fd90d8b3b8e6f613e5a2772caa22b3df92833cce Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 28 Sep 2023 15:05:05 -0300 Subject: [PATCH 09/64] refactor: Allow existing Person in create_demo_person (#6398) --- ietf/api/views.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/ietf/api/views.py b/ietf/api/views.py index 825b469f52..9d6b94f59a 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -488,6 +488,7 @@ def submitted_to_rpc(request): return JsonResponse(response) + @csrf_exempt def create_demo_person(request): """ Helper for creating rpc demo objects - SHOULD NOT MAKE IT INTO PRODUCTION @@ -501,12 +502,9 @@ def create_demo_person(request): request_params = json.loads(request.body) name = request_params["name"] - if Person.objects.filter(name=name).exists(): - return HttpResponseForbidden() - person = PersonFactory(name=name) - + person = Person.objects.filter(name=name).first() or PersonFactory(name=name) + return JsonResponse({"user_id":person.user.pk,"person_pk":person.pk}, status=201) - return JsonResponse({"user_id":person.user.pk,"person_pk":person.pk}) @csrf_exempt def create_demo_draft(request): @@ -535,5 +533,3 @@ def create_demo_draft(request): kwargs["states"] = states doc = WgDraftFactory(**kwargs) return JsonResponse({ "doc_id":doc.pk, "name":doc.name }) - - From fbc1dde3937086da28d1a8606ff453b8ecb1f3a5 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 28 Sep 2023 13:58:39 -0500 Subject: [PATCH 10/64] fix: include pubreq docevents in demo drafts (#6404) --- ietf/api/views.py | 10 ++++++++-- rpcapi.yaml | 2 ++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ietf/api/views.py b/ietf/api/views.py index 9d6b94f59a..b45c14fa3d 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -520,6 +520,7 @@ def create_demo_draft(request): request_params = json.loads(request.body) name = request_params.get("name") states = request_params.get("states") + stream_id = request_params.get("stream_id", "ietf") exists = False doc = None if not name: @@ -528,8 +529,13 @@ def create_demo_draft(request): if doc: exists = True else: - kwargs = {"name": name} + kwargs = {"name": name, "stream_id": stream_id} if states: kwargs["states"] = states - doc = WgDraftFactory(**kwargs) + doc = WgDraftFactory(**kwargs) # Yes, things may be a little strange if the stream isn't IETF, but until we nned something different... + event_type = "iesg_approved" if stream_id == "ietf" else "requested_publication" + if not doc.docevent_set.filter(type=event_type).exists() # Not using get_or_crete here on purpose - these are wobbly facades we're creating + doc.docevent_set.create(type=event_type, by_id=1, desc="Sent off to the RPC") return JsonResponse({ "doc_id":doc.pk, "name":doc.name }) + + diff --git a/rpcapi.yaml b/rpcapi.yaml index 6fe632438a..a170f06eb2 100644 --- a/rpcapi.yaml +++ b/rpcapi.yaml @@ -91,6 +91,8 @@ paths: properties: name: type: string + stream_id: + type: string states: type: array items: From 57f5c9adaab303770ccf08cef539b98d6f789edf Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 28 Sep 2023 16:14:53 -0300 Subject: [PATCH 11/64] fix: Minor API fixes (#6405) --- ietf/api/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ietf/api/views.py b/ietf/api/views.py index b45c14fa3d..c451b888ca 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -503,7 +503,7 @@ def create_demo_person(request): request_params = json.loads(request.body) name = request_params["name"] person = Person.objects.filter(name=name).first() or PersonFactory(name=name) - return JsonResponse({"user_id":person.user.pk,"person_pk":person.pk}, status=201) + return JsonResponse({"user_id":person.user.pk,"person_pk":person.pk}) @csrf_exempt @@ -534,7 +534,7 @@ def create_demo_draft(request): kwargs["states"] = states doc = WgDraftFactory(**kwargs) # Yes, things may be a little strange if the stream isn't IETF, but until we nned something different... event_type = "iesg_approved" if stream_id == "ietf" else "requested_publication" - if not doc.docevent_set.filter(type=event_type).exists() # Not using get_or_crete here on purpose - these are wobbly facades we're creating + if not doc.docevent_set.filter(type=event_type).exists(): # Not using get_or_create here on purpose - these are wobbly facades we're creating doc.docevent_set.create(type=event_type, by_id=1, desc="Sent off to the RPC") return JsonResponse({ "doc_id":doc.pk, "name":doc.name }) From 603ba8ce87eb25a43e294f23b3646f0a362f0a5e Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Sun, 1 Oct 2023 10:18:21 -0300 Subject: [PATCH 12/64] chore: Document 404 from rpcapi get_person_by_id --- rpcapi.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rpcapi.yaml b/rpcapi.yaml index a170f06eb2..ea74b17dcb 100644 --- a/rpcapi.yaml +++ b/rpcapi.yaml @@ -25,6 +25,8 @@ paths: application/json: schema: $ref: '#/components/schemas/Person' + '404': + description: Not found /persons/: post: @@ -80,7 +82,7 @@ paths: /doc/create_demo_draft/: post: operationId: create_demo_draft - summary: Build a datatraker WG draft for RPC demo purposes + summary: Build a datatracker WG draft for RPC demo purposes description: returns a datatracker document id for a draft created with the provided name and states. The arguments, if present, are passed directly to the WgDraftFactory requestBody: required: true From 084e8632f8e3a9c588ab33afcdd5b0ee168c7f67 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 3 Oct 2023 17:22:41 -0500 Subject: [PATCH 13/64] feat: adding rev to demo doc creation (#6425) * feat: adding rev to demo doc creation * fix: remove attempt to control required --- ietf/api/views.py | 3 +++ rpcapi.yaml | 2 ++ 2 files changed, 5 insertions(+) diff --git a/ietf/api/views.py b/ietf/api/views.py index c451b888ca..9a41b7e3e5 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -519,6 +519,7 @@ def create_demo_draft(request): request_params = json.loads(request.body) name = request_params.get("name") + rev = request_params.get("rev") states = request_params.get("states") stream_id = request_params.get("stream_id", "ietf") exists = False @@ -532,6 +533,8 @@ def create_demo_draft(request): kwargs = {"name": name, "stream_id": stream_id} if states: kwargs["states"] = states + if rev: + kwargs["rev"] = rev doc = WgDraftFactory(**kwargs) # Yes, things may be a little strange if the stream isn't IETF, but until we nned something different... event_type = "iesg_approved" if stream_id == "ietf" else "requested_publication" if not doc.docevent_set.filter(type=event_type).exists(): # Not using get_or_create here on purpose - these are wobbly facades we're creating diff --git a/rpcapi.yaml b/rpcapi.yaml index ea74b17dcb..b3d5b663af 100644 --- a/rpcapi.yaml +++ b/rpcapi.yaml @@ -95,6 +95,8 @@ paths: type: string stream_id: type: string + rev: + type: string states: type: array items: From af0ea91048dd371aed833d4d1beab4ea7393782a Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 5 Oct 2023 10:56:15 -0300 Subject: [PATCH 14/64] refactor: Replace api token checks with decorator (#6434) * feat: Add @requires_api_token decorator * refactor: Use @requires_api_token * refactor: Tweak api token endpoints This might be drifting from the design intent, but at least uses the defined endpoint values. Further cleanup may well be needed. --- docker/configs/settings_local.py | 2 +- ietf/api/ietf_utils.py | 15 +++++++++++++++ ietf/api/views.py | 28 +++++++--------------------- 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index eb2dfbbe7c..e570946e41 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -62,5 +62,5 @@ APP_API_TOKENS = { "ietf.api.views.rpc_person" : ["devtoken"], "ietf.api.views.submitted_to_rpc" : ["devtoken"], - "ietf.api.views.create_demo_person" : ["devtoken"] + "ietf.api.views.create_demo_resources" : ["devtoken"], # remove for production! } diff --git a/ietf/api/ietf_utils.py b/ietf/api/ietf_utils.py index 06b9d76aff..cb6de2921c 100644 --- a/ietf/api/ietf_utils.py +++ b/ietf/api/ietf_utils.py @@ -3,7 +3,11 @@ # This is not utils.py because Tastypie implicitly consumes ietf.api.utils. # See ietf.api.__init__.py for details. +from functools import wraps + from django.conf import settings +from django.http import HttpResponseForbidden + def is_valid_token(endpoint, token): # This is where we would consider integration with vault @@ -13,3 +17,14 @@ def is_valid_token(endpoint, token): if endpoint in token_store and token in token_store[endpoint]: return True return False + +def requires_api_token(endpoint): + def decorate(f): + @wraps(f) + def wrapped(request, *args, **kwargs): + authtoken = request.META.get("HTTP_X_API_KEY", None) + if authtoken is None or not is_valid_token(endpoint, authtoken): + return HttpResponseForbidden() + return f(request, *args, **kwargs) + return wrapped + return decorate diff --git a/ietf/api/views.py b/ietf/api/views.py index 9a41b7e3e5..a0e98065e1 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -35,7 +35,7 @@ from ietf.person.models import Person, Email from ietf.api import _api_list from ietf.api.serializer import JsonExportMixin -from ietf.api.ietf_utils import is_valid_token +from ietf.api.ietf_utils import is_valid_token, requires_api_token from ietf.doc.factories import WgDraftFactory # DO NOT MERGE INTO MAIN from ietf.doc.models import Document from ietf.doc.utils import fuzzy_find_documents @@ -441,10 +441,8 @@ def directauth(request): return HttpResponse(status=405) @csrf_exempt +@requires_api_token("ietf.api.views.rpc_person") def rpc_person(request, person_id): - authtoken = request.META.get("HTTP_X_API_KEY", None) - if authtoken is None or not is_valid_token("ietf.api.views.rpc_person", authtoken): - return HttpResponseForbidden() person = get_object_or_404(Person, pk=person_id) return JsonResponse({ "id": person.id, @@ -452,13 +450,9 @@ def rpc_person(request, person_id): }) @csrf_exempt +@requires_api_token("ietf.api.views.rpc_person") def rpc_persons(request): - """ Get a batch of rpc person names - - """ - authtoken = request.META.get("HTTP_X_API_KEY", None) - if authtoken is None or not is_valid_token("ietf.api.views.rpc_person", authtoken): - return HttpResponseForbidden() + """ Get a batch of rpc person names""" if request.method != "POST": return HttpResponseForbidden() @@ -470,14 +464,12 @@ def rpc_persons(request): @csrf_exempt +@requires_api_token("ietf.api.views.submitted_to_rpc") def submitted_to_rpc(request): """ Return documents in datatracker that have been submitted to the RPC but are not yet in the queue Those queries overreturn - there may be things, particularly not from the IETF stream that are already in the queue. """ - authtoken = request.META.get("HTTP_X_API_KEY", None) - if authtoken is None or not is_valid_token("ietf.api.views.submitted_to_rpc", authtoken): - return HttpResponseForbidden() ietf_docs = Q(states__type_id="draft-iesg",states__slug__in=["ann"]) irtf_iab_ise_docs = Q(states__type_id__in=["draft-stream-iab","draft-stream-irtf","draft-stream-ise"],states__slug__in=["rfc-edit"]) #TODO: Need a way to talk about editorial stream docs @@ -490,13 +482,11 @@ def submitted_to_rpc(request): @csrf_exempt +@requires_api_token("ietf.api.views.create_demo_resources") def create_demo_person(request): """ Helper for creating rpc demo objects - SHOULD NOT MAKE IT INTO PRODUCTION """ - authtoken = request.META.get("HTTP_X_API_KEY", None) - if authtoken is None or not is_valid_token("ietf.api.views.submitted_to_rpc", authtoken): - return HttpResponseForbidden() if request.method != "POST": return HttpResponseForbidden() @@ -507,13 +497,11 @@ def create_demo_person(request): @csrf_exempt +@requires_api_token("ietf.api.views.create_demo_resources") def create_demo_draft(request): """ Helper for creating rpc demo objects - SHOULD NOT MAKE IT INTO PRODUCTION """ - authtoken = request.META.get("HTTP_X_API_KEY", None) - if authtoken is None or not is_valid_token("ietf.api.views.submitted_to_rpc", authtoken): - return HttpResponseForbidden() if request.method != "POST": return HttpResponseForbidden() @@ -540,5 +528,3 @@ def create_demo_draft(request): if not doc.docevent_set.filter(type=event_type).exists(): # Not using get_or_create here on purpose - these are wobbly facades we're creating doc.docevent_set.create(type=event_type, by_id=1, desc="Sent off to the RPC") return JsonResponse({ "doc_id":doc.pk, "name":doc.name }) - - From 8c29605c659452ad8c2cb5c31a089a619a12ea42 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 5 Oct 2023 16:27:31 -0300 Subject: [PATCH 15/64] refactor: Improve usability of @requires_api_token --- ietf/api/ietf_utils.py | 43 ++++++++++++++++++++++++++++++++++++++++-- ietf/api/views.py | 4 ++-- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/ietf/api/ietf_utils.py b/ietf/api/ietf_utils.py index cb6de2921c..3f6d4762e0 100644 --- a/ietf/api/ietf_utils.py +++ b/ietf/api/ietf_utils.py @@ -4,6 +4,7 @@ # See ietf.api.__init__.py for details. from functools import wraps +from typing import Callable, Optional, Union from django.conf import settings from django.http import HttpResponseForbidden @@ -18,13 +19,51 @@ def is_valid_token(endpoint, token): return True return False -def requires_api_token(endpoint): + +def requires_api_token(func_or_endpoint: Optional[Union[Callable, str]] = None): + """Validate API token before executing the wrapped method + + Usage: + * Basic: endpoint defaults to the qualified name of the wrapped method. E.g., in ietf.api.views, + + @requires_api_token + def my_view(request): + ... + + will require a token for "ietf.api.views.my_view" + + * Custom endpoint: specify the endpoint explicitly + + @requires_api_token("ietf.api.views.some_other_thing") + def my_view(request): + ... + + will require a token for "ietf.api.views.some_other_thing" + """ + def decorate(f): + if _endpoint is None: + fname = getattr(f, "__qualname__", None) + if fname is None: + raise TypeError("Cannot automatically decorate function that does not support __qualname__. Explicitly set the endpoint.") + endpoint = "{}.{}".format(f.__module__, fname) + else: + endpoint = _endpoint + @wraps(f) def wrapped(request, *args, **kwargs): authtoken = request.META.get("HTTP_X_API_KEY", None) if authtoken is None or not is_valid_token(endpoint, authtoken): return HttpResponseForbidden() return f(request, *args, **kwargs) + return wrapped - return decorate + + # Magic to allow decorator to be used with or without parentheses + if callable(func_or_endpoint): + func = func_or_endpoint + _endpoint = None + return decorate(func) + else: + _endpoint = func_or_endpoint + return decorate diff --git a/ietf/api/views.py b/ietf/api/views.py index a0e98065e1..79948014a7 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -441,7 +441,7 @@ def directauth(request): return HttpResponse(status=405) @csrf_exempt -@requires_api_token("ietf.api.views.rpc_person") +@requires_api_token def rpc_person(request, person_id): person = get_object_or_404(Person, pk=person_id) return JsonResponse({ @@ -464,7 +464,7 @@ def rpc_persons(request): @csrf_exempt -@requires_api_token("ietf.api.views.submitted_to_rpc") +@requires_api_token def submitted_to_rpc(request): """ Return documents in datatracker that have been submitted to the RPC but are not yet in the queue From 272db6de00c82ee1f3c31536bb45208fd1b0a0ce Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 10 Oct 2023 13:01:48 -0300 Subject: [PATCH 16/64] feat: get_draft_by_id api call (#6446) --- docker/configs/settings_local.py | 1 + ietf/api/urls.py | 1 + ietf/api/views.py | 45 ++++++++++++++++++----- rpcapi.yaml | 62 +++++++++++++++++++++----------- 4 files changed, 79 insertions(+), 30 deletions(-) diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index e570946e41..766e39d1a9 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -61,6 +61,7 @@ APP_API_TOKENS = { "ietf.api.views.rpc_person" : ["devtoken"], + "ietf.api.views.rpc_draft" : ["devtoken"], "ietf.api.views.submitted_to_rpc" : ["devtoken"], "ietf.api.views.create_demo_resources" : ["devtoken"], # remove for production! } diff --git a/ietf/api/urls.py b/ietf/api/urls.py index f53bcdce5e..545842266b 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -63,6 +63,7 @@ url(r'^rpc/person/(?P[0-9]+)$', api_views.rpc_person), url(r'^rpc/persons/$', api_views.rpc_persons), url(r'^rpc/doc/submitted_to_rpc/$', api_views.submitted_to_rpc), + url(r'^rpc/doc/drafts/(?P[0-9]+)$', api_views.rpc_draft), url(r'^rpc/person/create_demo_person/$', api_views.create_demo_person), url(r'^rpc/doc/create_demo_draft/$', api_views.create_demo_draft), diff --git a/ietf/api/views.py b/ietf/api/views.py index 79948014a7..54d75d4acf 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -15,7 +15,13 @@ from django.core.exceptions import ValidationError from django.core.validators import validate_email from django.db.models import Q -from django.http import HttpResponse, Http404, JsonResponse, HttpResponseForbidden +from django.http import ( + HttpResponse, + Http404, + JsonResponse, + HttpResponseNotAllowed, + HttpResponseNotFound, +) from django.shortcuts import render, get_object_or_404 from django.urls import reverse from django.utils.decorators import method_decorator @@ -454,7 +460,7 @@ def rpc_person(request, person_id): def rpc_persons(request): """ Get a batch of rpc person names""" if request.method != "POST": - return HttpResponseForbidden() + return HttpResponseNotAllowed(["POST"]) pks = json.loads(request.body) response = dict() @@ -462,6 +468,30 @@ def rpc_persons(request): response[str(p.pk)] = p.plain_name() return JsonResponse(response) +@csrf_exempt +@requires_api_token +def rpc_draft(request, doc_id): + if request.method != "GET": + return HttpResponseNotAllowed(["GET"]) + + try: + d = Document.objects.get(pk=doc_id, type_id="draft") + except Document.DoesNotExist: + return HttpResponseNotFound() + return JsonResponse({ + "id": d.pk, + "name": d.name, + "rev": d.rev, + "stream": d.stream.slug, + "title": d.title, + "pages": d.pages, + "authors": [ + { + "id": p.pk, + "plain_name": p.plain_name(), + } for p in d.documentauthor_set.all() + ] + }) @csrf_exempt @requires_api_token @@ -488,7 +518,7 @@ def create_demo_person(request): """ if request.method != "POST": - return HttpResponseForbidden() + return HttpResponseNotAllowed(["POST"]) request_params = json.loads(request.body) name = request_params["name"] @@ -503,21 +533,18 @@ def create_demo_draft(request): """ if request.method != "POST": - return HttpResponseForbidden() - + return HttpResponseNotAllowed(["POST"]) + request_params = json.loads(request.body) name = request_params.get("name") rev = request_params.get("rev") states = request_params.get("states") stream_id = request_params.get("stream_id", "ietf") - exists = False doc = None if not name: return HttpResponse(status=400, content="Name is required") doc = Document.objects.filter(name=name).first() - if doc: - exists = True - else: + if not doc: kwargs = {"name": name, "stream_id": stream_id} if states: kwargs["states"] = states diff --git a/rpcapi.yaml b/rpcapi.yaml index b3d5b663af..e314e4cae8 100644 --- a/rpcapi.yaml +++ b/rpcapi.yaml @@ -55,7 +55,7 @@ paths: /person/create_demo_person/: post: operationId: create_demo_person - summary: Build a datatraker Person for RPC demo purposes + summary: Build a datatracker Person for RPC demo purposes description: returns a datatracker User id for a person created with the given name requestBody: required: true @@ -129,25 +129,27 @@ paths: schema: $ref: '#/components/schemas/SubmittedToQueue' -# /subject/person{subjectId}: -# get: -# operationId: get_subject_person_by_id -# summary: Find person for a subject by ID -# description: Returns a single person -# parameters: -# - name: subjectId -# in: path -# description: subject ID of person to return -# required: true -# schema: -# type: string -# responses: -# '200': -# description: OK -# content: -# application/json: -# schema: -# $ref: '#/components/schemas/Person' + /doc/drafts/{docId}: + get: + operationId: get_draft_by_id + summary: Get a draft + description: Returns the draft for the requested ID + parameters: + - name: docId + in: path + description: ID of draft to retrieve + required: true + schema: + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/Draft' + '404': + description: Not found components: schemas: @@ -178,7 +180,25 @@ components: submitted: type: string format: date - + Draft: + type: object + properties: + id: + type: integer + name: + type: string + rev: + type: string + stream: + type: string + title: + type: string + pages: + type: integer + authors: + type: array + items: + $ref: '#/components/schemas/Person' securitySchemes: ApiKeyAuth: From 4bc5ec65d9da61e184dfcad52401888f0839c113 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 17 Oct 2023 12:51:24 -0500 Subject: [PATCH 17/64] fix: construct the cdn photo url correctly (#6491) --- ietf/person/models.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ietf/person/models.py b/ietf/person/models.py index 22c63d4a0f..3120c529b7 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -253,11 +253,16 @@ def available_api_endpoints(self): def cdn_photo_url(self, size=80): if self.photo: if settings.SERVE_CDN_PHOTOS: + if settings.SERVER_MODE != "production": + original_media_dir = settings.MEDIA_URL + settings.MEDIA_URL = "https://www.ietf.org/lib/dt/media/" source_url = self.photo.url if source_url.startswith(settings.IETF_HOST_URL): source_url = source_url[len(settings.IETF_HOST_URL):] elif source_url.startswith('/'): source_url = source_url[1:] + if settings.SERVER_MODE != "production": + settings.MEDIA_URL = original_media_dir return f'{settings.IETF_HOST_URL}cdn-cgi/image/fit=scale-down,width={size},height={size}/{source_url}' else: datatracker_photo_path = urlreverse('ietf.person.views.photo', kwargs={'email_or_name': self.email()}) From 5c0cdecc4baf9f09785343152838c7a22e183d21 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 20 Oct 2023 10:56:28 -0500 Subject: [PATCH 18/64] chore: restructure the rpc api (#6505) --- dev/deploy-to-container/settings_local.py | 4 + docker/configs/settings_local.py | 5 +- ietf/api/urls.py | 9 +- ietf/api/urls_rpc.py | 14 +++ ietf/api/views.py | 117 +------------------- ietf/api/views_rpc.py | 129 ++++++++++++++++++++++ rpcapi.yaml | 2 +- 7 files changed, 151 insertions(+), 129 deletions(-) create mode 100644 ietf/api/urls_rpc.py create mode 100644 ietf/api/views_rpc.py diff --git a/dev/deploy-to-container/settings_local.py b/dev/deploy-to-container/settings_local.py index 60981ba567..2b0eb6a1f1 100644 --- a/dev/deploy-to-container/settings_local.py +++ b/dev/deploy-to-container/settings_local.py @@ -68,3 +68,7 @@ SLIDE_STAGING_PATH = '/test/staging/' DE_GFM_BINARY = '/usr/local/bin/de-gfm' + +APP_API_TOKENS = { + "ietf.api.views_rpc" : ["devtoken"], +} diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index 766e39d1a9..1e78b9472c 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -60,8 +60,5 @@ STATIC_IETF_ORG_INTERNAL = "http://static" APP_API_TOKENS = { - "ietf.api.views.rpc_person" : ["devtoken"], - "ietf.api.views.rpc_draft" : ["devtoken"], - "ietf.api.views.submitted_to_rpc" : ["devtoken"], - "ietf.api.views.create_demo_resources" : ["devtoken"], # remove for production! + "ietf.api.views_rpc" : ["devtoken"], } diff --git a/ietf/api/urls.py b/ietf/api/urls.py index 545842266b..0adc88b9f9 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -60,14 +60,7 @@ url(r'^rfcdiff-latest-json/(?P[Rr][Ff][Cc] [0-9]+?)(\.txt|\.html)?/?$', api_views.rfcdiff_latest_json), # direct authentication url(r'^directauth/?$', api_views.directauth), - url(r'^rpc/person/(?P[0-9]+)$', api_views.rpc_person), - url(r'^rpc/persons/$', api_views.rpc_persons), - url(r'^rpc/doc/submitted_to_rpc/$', api_views.submitted_to_rpc), - url(r'^rpc/doc/drafts/(?P[0-9]+)$', api_views.rpc_draft), - url(r'^rpc/person/create_demo_person/$', api_views.create_demo_person), - url(r'^rpc/doc/create_demo_draft/$', api_views.create_demo_draft), - - + url(r'^rpc/', include('ietf.api.urls_rpc')), ] # Additional (standard) Tastypie endpoints diff --git a/ietf/api/urls_rpc.py b/ietf/api/urls_rpc.py new file mode 100644 index 0000000000..0e3b176710 --- /dev/null +++ b/ietf/api/urls_rpc.py @@ -0,0 +1,14 @@ +# Copyright The IETF Trust 2023, All Rights Reserved + +from ietf.api import views_rpc + +from ietf.utils.urls import url + +urlpatterns = [ + url(r'^doc/create_demo_draft/$', views_rpc.create_demo_draft), + url(r'^doc/drafts/(?P[0-9]+)$', views_rpc.rpc_draft), + url(r'^doc/submitted_to_rpc/$', views_rpc.submitted_to_rpc), + url(r'^person/create_demo_person/$', views_rpc.create_demo_person), + url(r'^person/(?P[0-9]+)$', views_rpc.rpc_person), + url(r'^persons/$', views_rpc.rpc_persons), +] diff --git a/ietf/api/views.py b/ietf/api/views.py index 54d75d4acf..b4c15cc5eb 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -14,13 +14,9 @@ from django.contrib.auth.models import User from django.core.exceptions import ValidationError from django.core.validators import validate_email -from django.db.models import Q from django.http import ( HttpResponse, Http404, - JsonResponse, - HttpResponseNotAllowed, - HttpResponseNotFound, ) from django.shortcuts import render, get_object_or_404 from django.urls import reverse @@ -37,13 +33,10 @@ import debug # pyflakes:ignore import ietf -from ietf.person.factories import PersonFactory # DO NOT MERGE INTO MAIN from ietf.person.models import Person, Email from ietf.api import _api_list from ietf.api.serializer import JsonExportMixin -from ietf.api.ietf_utils import is_valid_token, requires_api_token -from ietf.doc.factories import WgDraftFactory # DO NOT MERGE INTO MAIN -from ietf.doc.models import Document +from ietf.api.ietf_utils import is_valid_token from ietf.doc.utils import fuzzy_find_documents from ietf.ietfauth.views import send_account_creation_email from ietf.ietfauth.utils import role_required @@ -446,112 +439,4 @@ def directauth(request): else: return HttpResponse(status=405) -@csrf_exempt -@requires_api_token -def rpc_person(request, person_id): - person = get_object_or_404(Person, pk=person_id) - return JsonResponse({ - "id": person.id, - "plain_name": person.plain_name(), - }) - -@csrf_exempt -@requires_api_token("ietf.api.views.rpc_person") -def rpc_persons(request): - """ Get a batch of rpc person names""" - if request.method != "POST": - return HttpResponseNotAllowed(["POST"]) - - pks = json.loads(request.body) - response = dict() - for p in Person.objects.filter(pk__in=pks): - response[str(p.pk)] = p.plain_name() - return JsonResponse(response) - -@csrf_exempt -@requires_api_token -def rpc_draft(request, doc_id): - if request.method != "GET": - return HttpResponseNotAllowed(["GET"]) - - try: - d = Document.objects.get(pk=doc_id, type_id="draft") - except Document.DoesNotExist: - return HttpResponseNotFound() - return JsonResponse({ - "id": d.pk, - "name": d.name, - "rev": d.rev, - "stream": d.stream.slug, - "title": d.title, - "pages": d.pages, - "authors": [ - { - "id": p.pk, - "plain_name": p.plain_name(), - } for p in d.documentauthor_set.all() - ] - }) - -@csrf_exempt -@requires_api_token -def submitted_to_rpc(request): - """ Return documents in datatracker that have been submitted to the RPC but are not yet in the queue - - Those queries overreturn - there may be things, particularly not from the IETF stream that are already in the queue. - """ - ietf_docs = Q(states__type_id="draft-iesg",states__slug__in=["ann"]) - irtf_iab_ise_docs = Q(states__type_id__in=["draft-stream-iab","draft-stream-irtf","draft-stream-ise"],states__slug__in=["rfc-edit"]) - #TODO: Need a way to talk about editorial stream docs - docs = Document.objects.filter(type_id="draft").filter(ietf_docs|irtf_iab_ise_docs) - response = {"submitted_to_rpc": []} - for doc in docs: - response["submitted_to_rpc"].append({"name":doc.name, "pk": doc.pk, "stream": doc.stream_id, "submitted": f"{doc.sent_to_rfc_editor_event().time:%Y-%m-%d}"}) #TODO reconcile timezone - return JsonResponse(response) - - -@csrf_exempt -@requires_api_token("ietf.api.views.create_demo_resources") -def create_demo_person(request): - """ Helper for creating rpc demo objects - SHOULD NOT MAKE IT INTO PRODUCTION - - """ - if request.method != "POST": - return HttpResponseNotAllowed(["POST"]) - - request_params = json.loads(request.body) - name = request_params["name"] - person = Person.objects.filter(name=name).first() or PersonFactory(name=name) - return JsonResponse({"user_id":person.user.pk,"person_pk":person.pk}) - - -@csrf_exempt -@requires_api_token("ietf.api.views.create_demo_resources") -def create_demo_draft(request): - """ Helper for creating rpc demo objects - SHOULD NOT MAKE IT INTO PRODUCTION - - """ - if request.method != "POST": - return HttpResponseNotAllowed(["POST"]) - - request_params = json.loads(request.body) - name = request_params.get("name") - rev = request_params.get("rev") - states = request_params.get("states") - stream_id = request_params.get("stream_id", "ietf") - doc = None - if not name: - return HttpResponse(status=400, content="Name is required") - doc = Document.objects.filter(name=name).first() - if not doc: - kwargs = {"name": name, "stream_id": stream_id} - if states: - kwargs["states"] = states - if rev: - kwargs["rev"] = rev - doc = WgDraftFactory(**kwargs) # Yes, things may be a little strange if the stream isn't IETF, but until we nned something different... - event_type = "iesg_approved" if stream_id == "ietf" else "requested_publication" - if not doc.docevent_set.filter(type=event_type).exists(): # Not using get_or_create here on purpose - these are wobbly facades we're creating - doc.docevent_set.create(type=event_type, by_id=1, desc="Sent off to the RPC") - return JsonResponse({ "doc_id":doc.pk, "name":doc.name }) diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py new file mode 100644 index 0000000000..dd17db064a --- /dev/null +++ b/ietf/api/views_rpc.py @@ -0,0 +1,129 @@ +# Copyright The IETF Trust 2023, All Rights Reserved + +import json + +from django.db.models import Q +from django.http import ( + HttpResponse, + JsonResponse, + HttpResponseNotAllowed, + HttpResponseNotFound, +) +from django.shortcuts import get_object_or_404 +from django.views.decorators.csrf import csrf_exempt + +from ietf.api.ietf_utils import requires_api_token +from ietf.doc.factories import WgDraftFactory # DO NOT MERGE INTO MAIN +from ietf.doc.models import Document +from ietf.person.factories import PersonFactory # DO NOT MERGE INTO MAIN +from ietf.person.models import Person + +@csrf_exempt +@requires_api_token +def rpc_person(request, person_id): + person = get_object_or_404(Person, pk=person_id) + return JsonResponse({ + "id": person.id, + "plain_name": person.plain_name(), + }) + +@csrf_exempt +@requires_api_token("ietf.api.views_rpc") +def rpc_persons(request): + """ Get a batch of rpc person names""" + if request.method != "POST": + return HttpResponseNotAllowed(["POST"]) + + pks = json.loads(request.body) + response = dict() + for p in Person.objects.filter(pk__in=pks): + response[str(p.pk)] = p.plain_name() + return JsonResponse(response) + +@csrf_exempt +@requires_api_token("ietf.api.views_rpc") +def rpc_draft(request, doc_id): + if request.method != "GET": + return HttpResponseNotAllowed(["GET"]) + + try: + d = Document.objects.get(pk=doc_id, type_id="draft") + except Document.DoesNotExist: + return HttpResponseNotFound() + return JsonResponse({ + "id": d.pk, + "name": d.name, + "rev": d.rev, + "stream": d.stream.slug, + "title": d.title, + "pages": d.pages, + "authors": [ + { + "id": p.pk, + "plain_name": p.plain_name(), + } for p in d.documentauthor_set.all() + ] + }) + +@csrf_exempt +@requires_api_token("ietf.api.views_rpc") +def submitted_to_rpc(request): + """ Return documents in datatracker that have been submitted to the RPC but are not yet in the queue + + Those queries overreturn - there may be things, particularly not from the IETF stream that are already in the queue. + """ + ietf_docs = Q(states__type_id="draft-iesg",states__slug__in=["ann"]) + irtf_iab_ise_docs = Q(states__type_id__in=["draft-stream-iab","draft-stream-irtf","draft-stream-ise"],states__slug__in=["rfc-edit"]) + #TODO: Need a way to talk about editorial stream docs + docs = Document.objects.filter(type_id="draft").filter(ietf_docs|irtf_iab_ise_docs) + response = {"submitted_to_rpc": []} + for doc in docs: + response["submitted_to_rpc"].append({"name":doc.name, "pk": doc.pk, "stream": doc.stream_id, "submitted": f"{doc.sent_to_rfc_editor_event().time:%Y-%m-%d}"}) #TODO reconcile timezone + + return JsonResponse(response) + + +@csrf_exempt +@requires_api_token("ietf.api.views_rpc") +def create_demo_person(request): + """ Helper for creating rpc demo objects - SHOULD NOT MAKE IT INTO PRODUCTION + + """ + if request.method != "POST": + return HttpResponseNotAllowed(["POST"]) + + request_params = json.loads(request.body) + name = request_params["name"] + person = Person.objects.filter(name=name).first() or PersonFactory(name=name) + return JsonResponse({"user_id":person.user.pk,"person_pk":person.pk}) + + +@csrf_exempt +@requires_api_token("ietf.api.views_rpc") +def create_demo_draft(request): + """ Helper for creating rpc demo objects - SHOULD NOT MAKE IT INTO PRODUCTION + + """ + if request.method != "POST": + return HttpResponseNotAllowed(["POST"]) + + request_params = json.loads(request.body) + name = request_params.get("name") + rev = request_params.get("rev") + states = request_params.get("states") + stream_id = request_params.get("stream_id", "ietf") + doc = None + if not name: + return HttpResponse(status=400, content="Name is required") + doc = Document.objects.filter(name=name).first() + if not doc: + kwargs = {"name": name, "stream_id": stream_id} + if states: + kwargs["states"] = states + if rev: + kwargs["rev"] = rev + doc = WgDraftFactory(**kwargs) # Yes, things may be a little strange if the stream isn't IETF, but until we nned something different... + event_type = "iesg_approved" if stream_id == "ietf" else "requested_publication" + if not doc.docevent_set.filter(type=event_type).exists(): # Not using get_or_create here on purpose - these are wobbly facades we're creating + doc.docevent_set.create(type=event_type, by_id=1, desc="Sent off to the RPC") + return JsonResponse({ "doc_id":doc.pk, "name":doc.name }) diff --git a/rpcapi.yaml b/rpcapi.yaml index e314e4cae8..1f9c0ef909 100644 --- a/rpcapi.yaml +++ b/rpcapi.yaml @@ -4,7 +4,7 @@ info: description: Datatracker RPC API version: 1.0.0 servers: - - url: 'http://localhost:8000/api/rpc/' + - url: 'http://localhost:8000/api/rpc' paths: /person/{personId}: get: From f2b31384111258256e7e3212b1edb7a729805cb3 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 20 Oct 2023 12:33:11 -0500 Subject: [PATCH 19/64] fix: authenticate person api correctly --- ietf/api/views_rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index dd17db064a..6626016016 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -19,7 +19,7 @@ from ietf.person.models import Person @csrf_exempt -@requires_api_token +@requires_api_token("ietf.api.views_rpc") def rpc_person(request, person_id): person = get_object_or_404(Person, pk=person_id) return JsonResponse({ From 6b75cff90d3e6fefe61b48a74bea43bc1f7aa2bb Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 25 Oct 2023 09:44:38 -0300 Subject: [PATCH 20/64] chore: Fix plain_name lookup --- ietf/api/views_rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 6626016016..38d42ba8da 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -60,7 +60,7 @@ def rpc_draft(request, doc_id): "authors": [ { "id": p.pk, - "plain_name": p.plain_name(), + "plain_name": p.person.plain_name(), } for p in d.documentauthor_set.all() ] }) From 3f04ca74fa13ae28aad02618ec8f60fd54edd14b Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 1 Nov 2023 15:50:28 -0300 Subject: [PATCH 21/64] feat: subject_id -> Person api call (#6566) * feat: subject_id -> Person api call * doc: Add error responses to openapi spec --- ietf/api/urls_rpc.py | 13 +++++++------ ietf/api/views_rpc.py | 17 +++++++++++++++++ rpcapi.yaml | 41 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 64 insertions(+), 7 deletions(-) diff --git a/ietf/api/urls_rpc.py b/ietf/api/urls_rpc.py index 0e3b176710..7a0223b414 100644 --- a/ietf/api/urls_rpc.py +++ b/ietf/api/urls_rpc.py @@ -5,10 +5,11 @@ from ietf.utils.urls import url urlpatterns = [ - url(r'^doc/create_demo_draft/$', views_rpc.create_demo_draft), - url(r'^doc/drafts/(?P[0-9]+)$', views_rpc.rpc_draft), - url(r'^doc/submitted_to_rpc/$', views_rpc.submitted_to_rpc), - url(r'^person/create_demo_person/$', views_rpc.create_demo_person), - url(r'^person/(?P[0-9]+)$', views_rpc.rpc_person), - url(r'^persons/$', views_rpc.rpc_persons), + url(r"^doc/create_demo_draft/$", views_rpc.create_demo_draft), + url(r"^doc/drafts/(?P[0-9]+)$", views_rpc.rpc_draft), + url(r"^doc/submitted_to_rpc/$", views_rpc.submitted_to_rpc), + url(r"^person/create_demo_person/$", views_rpc.create_demo_person), + url(r"^person/(?P[0-9]+)$", views_rpc.rpc_person), + url(r"^persons/$", views_rpc.rpc_persons), + url(r"^subject/(?P[0-9]+)/person/$", views_rpc.rpc_subject_person), ] diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 38d42ba8da..b689faa281 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -11,6 +11,7 @@ ) from django.shortcuts import get_object_or_404 from django.views.decorators.csrf import csrf_exempt +from django.contrib.auth.models import User from ietf.api.ietf_utils import requires_api_token from ietf.doc.factories import WgDraftFactory # DO NOT MERGE INTO MAIN @@ -27,6 +28,22 @@ def rpc_person(request, person_id): "plain_name": person.plain_name(), }) +@csrf_exempt +@requires_api_token("ietf.api.views_rpc") +def rpc_subject_person(request, subject_id): + try: + user_id = int(subject_id) + except ValueError: + return JsonResponse({"error": "Invalid subject id"}, status=400) + try: + user = User.objects.get(pk=user_id) + except User.DoesNotExist: + return JsonResponse({"error": "Unknown subject"}, status=404) + if hasattr(user, "person"): # test this way to avoid exception on reverse OneToOneField + return rpc_person(request, person_id=user.person.pk) + return JsonResponse({"error": "Subject has no person"}, status=404) + + @csrf_exempt @requires_api_token("ietf.api.views_rpc") def rpc_persons(request): diff --git a/rpcapi.yaml b/rpcapi.yaml index 1f9c0ef909..66cfa5cbef 100644 --- a/rpcapi.yaml +++ b/rpcapi.yaml @@ -128,7 +128,7 @@ paths: application/json: schema: $ref: '#/components/schemas/SubmittedToQueue' - + /doc/drafts/{docId}: get: operationId: get_draft_by_id @@ -151,6 +151,38 @@ paths: '404': description: Not found + /subject/{subjectId}/person: + get: + operationId: get_subject_person_by_id + summary: Find person for OIDC subject by ID + description: Returns a single person + parameters: + - name: subjectId + in: path + description: subject ID of person to return + required: true + schema: + type: string + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/Person' + '400': + description: No such subject or no person for this subject + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + '404': + description: No such subject or no person for this subject + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + components: schemas: Person: @@ -180,6 +212,13 @@ components: submitted: type: string format: date + + ErrorResponse: + type: object + properties: + error: + type: string + Draft: type: object properties: From e8fe73165544d9b55f28687c04c5a182fdfb3366 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 2 Jan 2024 10:56:10 -0600 Subject: [PATCH 22/64] feat: rpc api drafts by names (#6853) --- ietf/api/urls_rpc.py | 1 + ietf/api/views_rpc.py | 29 +++++++++++++++++++++++++++++ rpcapi.yaml | 23 +++++++++++++++++++++++ 3 files changed, 53 insertions(+) diff --git a/ietf/api/urls_rpc.py b/ietf/api/urls_rpc.py index 7a0223b414..c487170147 100644 --- a/ietf/api/urls_rpc.py +++ b/ietf/api/urls_rpc.py @@ -7,6 +7,7 @@ urlpatterns = [ url(r"^doc/create_demo_draft/$", views_rpc.create_demo_draft), url(r"^doc/drafts/(?P[0-9]+)$", views_rpc.rpc_draft), + url(r"^doc/drafts_by_names/", views_rpc.drafts_by_names), url(r"^doc/submitted_to_rpc/$", views_rpc.submitted_to_rpc), url(r"^person/create_demo_person/$", views_rpc.create_demo_person), url(r"^person/(?P[0-9]+)$", views_rpc.rpc_person), diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index b689faa281..3ec9f69273 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -5,6 +5,7 @@ from django.db.models import Q from django.http import ( HttpResponse, + HttpResponseBadRequest, JsonResponse, HttpResponseNotAllowed, HttpResponseNotFound, @@ -82,6 +83,34 @@ def rpc_draft(request, doc_id): ] }) +@csrf_exempt +@requires_api_token("ietf.api.views_rpc") +def drafts_by_names(request): + if request.method != "POST": + return HttpResponseNotAllowed(["POST"]) + try: + names = json.loads(request.body) + except json.JSONDecodeError: + return HttpResponseBadRequest() + docs = Document.objects.filter(type_id="draft",name__in=names) + response = dict() + for doc in docs: + response[doc.name] = { + "id": doc.pk, + "name": doc.name, + "rev": doc.rev, + "stream": doc.stream.slug, + "title": doc.title, + "pages": doc.pages, + "authors": [ + { + "id": p.pk, + "plain_name": p.person.plain_name(), + } for p in doc.documentauthor_set.all() + ] + } + return JsonResponse(response) + @csrf_exempt @requires_api_token("ietf.api.views_rpc") def submitted_to_rpc(request): diff --git a/rpcapi.yaml b/rpcapi.yaml index 66cfa5cbef..1b6b1dac17 100644 --- a/rpcapi.yaml +++ b/rpcapi.yaml @@ -151,6 +151,29 @@ paths: '404': description: Not found + /doc/drafts_by_names/: + post: + operationId: get_drafts_by_names + summary: Get a batch of drafts by draft names + description: returns a dict of drafts with matching names + requestBody: + required: true + content: + application/json: + schema: + type: array + items: + type: string + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + additionalProperties: + $ref:'#/components/schemas/Draft' + /subject/{subjectId}/person: get: operationId: get_subject_person_by_id From 34a09879e63a110190acef897414d3a4369400af Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Fri, 9 Aug 2024 17:33:56 -0500 Subject: [PATCH 23/64] feat: get stream rfcs were first published into (#7814) * feat: get stream rfcs were first published into * chore: black * fix: deal with the case of no DocHistory having a stream --- ietf/api/urls_rpc.py | 1 + ietf/api/views_rpc.py | 158 +++++++++++++++++++++++++++++------------- rpcapi.yaml | 27 +++++++- 3 files changed, 135 insertions(+), 51 deletions(-) diff --git a/ietf/api/urls_rpc.py b/ietf/api/urls_rpc.py index c487170147..314ff8643f 100644 --- a/ietf/api/urls_rpc.py +++ b/ietf/api/urls_rpc.py @@ -9,6 +9,7 @@ url(r"^doc/drafts/(?P[0-9]+)$", views_rpc.rpc_draft), url(r"^doc/drafts_by_names/", views_rpc.drafts_by_names), url(r"^doc/submitted_to_rpc/$", views_rpc.submitted_to_rpc), + url(r"^doc/rfc/original_stream/$", views_rpc.rfc_original_stream), url(r"^person/create_demo_person/$", views_rpc.create_demo_person), url(r"^person/(?P[0-9]+)$", views_rpc.rpc_person), url(r"^persons/$", views_rpc.rpc_persons), diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 3ec9f69273..dd264da761 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -2,7 +2,7 @@ import json -from django.db.models import Q +from django.db.models import OuterRef, Subquery, Q from django.http import ( HttpResponse, HttpResponseBadRequest, @@ -15,19 +15,23 @@ from django.contrib.auth.models import User from ietf.api.ietf_utils import requires_api_token -from ietf.doc.factories import WgDraftFactory # DO NOT MERGE INTO MAIN -from ietf.doc.models import Document -from ietf.person.factories import PersonFactory # DO NOT MERGE INTO MAIN +from ietf.doc.factories import WgDraftFactory # DO NOT MERGE INTO MAIN +from ietf.doc.models import Document, DocHistory +from ietf.person.factories import PersonFactory # DO NOT MERGE INTO MAIN from ietf.person.models import Person + @csrf_exempt @requires_api_token("ietf.api.views_rpc") def rpc_person(request, person_id): person = get_object_or_404(Person, pk=person_id) - return JsonResponse({ - "id": person.id, - "plain_name": person.plain_name(), - }) + return JsonResponse( + { + "id": person.id, + "plain_name": person.plain_name(), + } + ) + @csrf_exempt @requires_api_token("ietf.api.views_rpc") @@ -36,52 +40,59 @@ def rpc_subject_person(request, subject_id): user_id = int(subject_id) except ValueError: return JsonResponse({"error": "Invalid subject id"}, status=400) - try: + try: user = User.objects.get(pk=user_id) except User.DoesNotExist: return JsonResponse({"error": "Unknown subject"}, status=404) - if hasattr(user, "person"): # test this way to avoid exception on reverse OneToOneField + if hasattr( + user, "person" + ): # test this way to avoid exception on reverse OneToOneField return rpc_person(request, person_id=user.person.pk) - return JsonResponse({"error": "Subject has no person"}, status=404) + return JsonResponse({"error": "Subject has no person"}, status=404) @csrf_exempt @requires_api_token("ietf.api.views_rpc") def rpc_persons(request): - """ Get a batch of rpc person names""" + """Get a batch of rpc person names""" if request.method != "POST": return HttpResponseNotAllowed(["POST"]) - + pks = json.loads(request.body) response = dict() for p in Person.objects.filter(pk__in=pks): response[str(p.pk)] = p.plain_name() return JsonResponse(response) + @csrf_exempt @requires_api_token("ietf.api.views_rpc") def rpc_draft(request, doc_id): if request.method != "GET": return HttpResponseNotAllowed(["GET"]) - + try: d = Document.objects.get(pk=doc_id, type_id="draft") except Document.DoesNotExist: return HttpResponseNotFound() - return JsonResponse({ - "id": d.pk, - "name": d.name, - "rev": d.rev, - "stream": d.stream.slug, - "title": d.title, - "pages": d.pages, - "authors": [ - { - "id": p.pk, - "plain_name": p.person.plain_name(), - } for p in d.documentauthor_set.all() - ] - }) + return JsonResponse( + { + "id": d.pk, + "name": d.name, + "rev": d.rev, + "stream": d.stream.slug, + "title": d.title, + "pages": d.pages, + "authors": [ + { + "id": p.pk, + "plain_name": p.person.plain_name(), + } + for p in d.documentauthor_set.all() + ], + } + ) + @csrf_exempt @requires_api_token("ietf.api.views_rpc") @@ -92,7 +103,7 @@ def drafts_by_names(request): names = json.loads(request.body) except json.JSONDecodeError: return HttpResponseBadRequest() - docs = Document.objects.filter(type_id="draft",name__in=names) + docs = Document.objects.filter(type_id="draft", name__in=names) response = dict() for doc in docs: response[doc.name] = { @@ -106,50 +117,91 @@ def drafts_by_names(request): { "id": p.pk, "plain_name": p.person.plain_name(), - } for p in doc.documentauthor_set.all() - ] + } + for p in doc.documentauthor_set.all() + ], } return JsonResponse(response) + @csrf_exempt @requires_api_token("ietf.api.views_rpc") def submitted_to_rpc(request): - """ Return documents in datatracker that have been submitted to the RPC but are not yet in the queue - + """Return documents in datatracker that have been submitted to the RPC but are not yet in the queue + Those queries overreturn - there may be things, particularly not from the IETF stream that are already in the queue. """ - ietf_docs = Q(states__type_id="draft-iesg",states__slug__in=["ann"]) - irtf_iab_ise_docs = Q(states__type_id__in=["draft-stream-iab","draft-stream-irtf","draft-stream-ise"],states__slug__in=["rfc-edit"]) - #TODO: Need a way to talk about editorial stream docs - docs = Document.objects.filter(type_id="draft").filter(ietf_docs|irtf_iab_ise_docs) + ietf_docs = Q(states__type_id="draft-iesg", states__slug__in=["ann"]) + irtf_iab_ise_docs = Q( + states__type_id__in=[ + "draft-stream-iab", + "draft-stream-irtf", + "draft-stream-ise", + ], + states__slug__in=["rfc-edit"], + ) + # TODO: Need a way to talk about editorial stream docs + docs = Document.objects.filter(type_id="draft").filter( + ietf_docs | irtf_iab_ise_docs + ) response = {"submitted_to_rpc": []} for doc in docs: - response["submitted_to_rpc"].append({"name":doc.name, "pk": doc.pk, "stream": doc.stream_id, "submitted": f"{doc.sent_to_rfc_editor_event().time:%Y-%m-%d}"}) #TODO reconcile timezone + response["submitted_to_rpc"].append( + { + "name": doc.name, + "pk": doc.pk, + "stream": doc.stream_id, + "submitted": f"{doc.sent_to_rfc_editor_event().time:%Y-%m-%d}", + } + ) # TODO reconcile timezone return JsonResponse(response) @csrf_exempt @requires_api_token("ietf.api.views_rpc") -def create_demo_person(request): - """ Helper for creating rpc demo objects - SHOULD NOT MAKE IT INTO PRODUCTION +def rfc_original_stream(request): + """Return the stream that an rfc was first published into for all rfcs""" + rfcs = Document.objects.filter(type="rfc").annotate( + orig_stream_id=Subquery( + DocHistory.objects.filter(doc=OuterRef("pk")) + .exclude(stream__isnull=True) + .order_by("time") + .values_list("stream_id", flat=True)[:1] + ) + ) + response = {"original_stream": []} + for rfc in rfcs: + response["original_stream"].append( + { + "rfc_number": rfc.rfc_number, + "stream": ( + rfc.orig_stream_id + if rfc.orig_stream_id is not None + else rfc.stream_id + ), + } + ) + return JsonResponse(response) - """ + +@csrf_exempt +@requires_api_token("ietf.api.views_rpc") +def create_demo_person(request): + """Helper for creating rpc demo objects - SHOULD NOT MAKE IT INTO PRODUCTION""" if request.method != "POST": return HttpResponseNotAllowed(["POST"]) - + request_params = json.loads(request.body) name = request_params["name"] person = Person.objects.filter(name=name).first() or PersonFactory(name=name) - return JsonResponse({"user_id":person.user.pk,"person_pk":person.pk}) + return JsonResponse({"user_id": person.user.pk, "person_pk": person.pk}) @csrf_exempt @requires_api_token("ietf.api.views_rpc") def create_demo_draft(request): - """ Helper for creating rpc demo objects - SHOULD NOT MAKE IT INTO PRODUCTION - - """ + """Helper for creating rpc demo objects - SHOULD NOT MAKE IT INTO PRODUCTION""" if request.method != "POST": return HttpResponseNotAllowed(["POST"]) @@ -168,8 +220,14 @@ def create_demo_draft(request): kwargs["states"] = states if rev: kwargs["rev"] = rev - doc = WgDraftFactory(**kwargs) # Yes, things may be a little strange if the stream isn't IETF, but until we nned something different... + doc = WgDraftFactory( + **kwargs + ) # Yes, things may be a little strange if the stream isn't IETF, but until we nned something different... event_type = "iesg_approved" if stream_id == "ietf" else "requested_publication" - if not doc.docevent_set.filter(type=event_type).exists(): # Not using get_or_create here on purpose - these are wobbly facades we're creating - doc.docevent_set.create(type=event_type, by_id=1, desc="Sent off to the RPC") - return JsonResponse({ "doc_id":doc.pk, "name":doc.name }) + if not doc.docevent_set.filter( + type=event_type + ).exists(): # Not using get_or_create here on purpose - these are wobbly facades we're creating + doc.docevent_set.create( + type=event_type, by_id=1, desc="Sent off to the RPC" + ) + return JsonResponse({"doc_id": doc.pk, "name": doc.name}) diff --git a/rpcapi.yaml b/rpcapi.yaml index 1b6b1dac17..b2a4215162 100644 --- a/rpcapi.yaml +++ b/rpcapi.yaml @@ -172,7 +172,20 @@ paths: schema: type: object additionalProperties: - $ref:'#/components/schemas/Draft' + $ref:'#/components/schemas/Draft' + + /doc/rfc/original_stream/: + get: + operationId: get_rfc_original_streams + summary: Get the streams rfcs were originally published into + description: returns a list of dicts associating an rfc with its originally published stream + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/OriginalStream' /subject/{subjectId}/person: get: @@ -262,6 +275,18 @@ components: items: $ref: '#/components/schemas/Person' + OriginalStream: + type: object + properties: + original_stream: + type: list + items: + properties: + rfc_number: + type: integer + stream: + type: string + securitySchemes: ApiKeyAuth: type: apiKey From 6ea1e8212c9453ad89f8be3f087bd67475cb6c7c Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Tue, 27 Aug 2024 21:17:55 -0500 Subject: [PATCH 24/64] fix: return "none" from drafts_by_names if a draft has a stream of None (#7863) --- ietf/api/views_rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index dd264da761..028f3d6866 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -110,7 +110,7 @@ def drafts_by_names(request): "id": doc.pk, "name": doc.name, "rev": doc.rev, - "stream": doc.stream.slug, + "stream": doc.stream.slug if doc.stream else "none", "title": doc.title, "pages": doc.pages, "authors": [ From 2ed6a2e20e3d0ebbecfb797df49c352929523c44 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 6 Sep 2024 16:48:17 -0300 Subject: [PATCH 25/64] feat: API changes needed for "real" draft imports (#7877) * style: black * fix: submitted as DateTime, not Date * fix: pk -> id in API * feat: rpc_draft source_format * feat: add shepherd to rpc_draft() api * fix: "unknown" src fmt instead of error * feat: add intended_std_level to api * refactor: blank, not None, for shepherd / std_level * style: black --- ietf/api/views_rpc.py | 28 +++++++++++++++++++++++----- rpcapi.yaml | 20 ++++++++++++++++---- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 028f3d6866..8b86417b5d 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -65,6 +65,20 @@ def rpc_persons(request): return JsonResponse(response) +def _document_source_format(doc): + submission = doc.submission() + if submission is None: + return "unknown" + if ".xml" in submission.file_types: + if submission.xml_version == "3": + return "xml-v3" + else: + return "xml-v2" + elif ".txt" in submission.file_types: + return "txt" + return "unknown" + + @csrf_exempt @requires_api_token("ietf.api.views_rpc") def rpc_draft(request, doc_id): @@ -83,6 +97,7 @@ def rpc_draft(request, doc_id): "stream": d.stream.slug, "title": d.title, "pages": d.pages, + "source_format": _document_source_format(d), "authors": [ { "id": p.pk, @@ -90,10 +105,13 @@ def rpc_draft(request, doc_id): } for p in d.documentauthor_set.all() ], + "shepherd": d.shepherd.formatted_ascii_email() if d.shepherd else "", + "intended_std_level": ( + d.intended_std_level.slug if d.intended_std_level else "" + ), } ) - @csrf_exempt @requires_api_token("ietf.api.views_rpc") def drafts_by_names(request): @@ -113,6 +131,7 @@ def drafts_by_names(request): "stream": doc.stream.slug if doc.stream else "none", "title": doc.title, "pages": doc.pages, + "source_format": _document_source_format(d), "authors": [ { "id": p.pk, @@ -149,12 +168,11 @@ def submitted_to_rpc(request): response["submitted_to_rpc"].append( { "name": doc.name, - "pk": doc.pk, + "id": doc.pk, "stream": doc.stream_id, - "submitted": f"{doc.sent_to_rfc_editor_event().time:%Y-%m-%d}", + "submitted": f"{doc.sent_to_rfc_editor_event().time.isoformat()}", } - ) # TODO reconcile timezone - + ) return JsonResponse(response) diff --git a/rpcapi.yaml b/rpcapi.yaml index b2a4215162..ef92192168 100644 --- a/rpcapi.yaml +++ b/rpcapi.yaml @@ -219,8 +219,8 @@ paths: schema: $ref: '#/components/schemas/ErrorResponse' -components: - schemas: +components: + schemas: Person: type: object properties: @@ -241,13 +241,13 @@ components: properties: name: type: string - pk: + id: type: integer stream: type: string submitted: type: string - format: date + format: date-time ErrorResponse: type: object @@ -270,10 +270,22 @@ components: type: string pages: type: integer + source_format: + type: string + enum: + - unknown + - txt + - xml-v2 + - xml-v3 authors: type: array items: $ref: '#/components/schemas/Person' + shepherd: + type: string + format: email + intended_std_level: + type: string OriginalStream: type: object From 1d3eba705b87824640476bb1935e368a021ffe81 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 9 Sep 2024 13:22:08 -0300 Subject: [PATCH 26/64] fix: typo in drafts_by_names() (#7912) --- ietf/api/views_rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 8b86417b5d..9247cfbe98 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -131,7 +131,7 @@ def drafts_by_names(request): "stream": doc.stream.slug if doc.stream else "none", "title": doc.title, "pages": doc.pages, - "source_format": _document_source_format(d), + "source_format": _document_source_format(doc), "authors": [ { "id": p.pk, From 9cf8f1015cc692b252ee3761f043df21e9d08afd Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Mon, 28 Oct 2024 14:23:33 -0500 Subject: [PATCH 27/64] feat: support for building rfc authors (#7983) * feat: support for building rfc authors * chore: copyrights --- ietf/api/urls_rpc.py | 4 ++- ietf/api/views_rpc.py | 51 ++++++++++++++++++++++++++-- rpcapi.yaml | 78 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 130 insertions(+), 3 deletions(-) diff --git a/ietf/api/urls_rpc.py b/ietf/api/urls_rpc.py index 314ff8643f..85783bbc3f 100644 --- a/ietf/api/urls_rpc.py +++ b/ietf/api/urls_rpc.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2023, All Rights Reserved +# Copyright The IETF Trust 2023-2024, All Rights Reserved from ietf.api import views_rpc @@ -10,7 +10,9 @@ url(r"^doc/drafts_by_names/", views_rpc.drafts_by_names), url(r"^doc/submitted_to_rpc/$", views_rpc.submitted_to_rpc), url(r"^doc/rfc/original_stream/$", views_rpc.rfc_original_stream), + url(r"^doc/rfc/authors/$", views_rpc.rfc_authors), url(r"^person/create_demo_person/$", views_rpc.create_demo_person), + url(r"^person/persons_by_email/$", views_rpc.persons_by_email), url(r"^person/(?P[0-9]+)$", views_rpc.rpc_person), url(r"^persons/$", views_rpc.rpc_persons), url(r"^subject/(?P[0-9]+)/person/$", views_rpc.rpc_subject_person), diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 9247cfbe98..bddc56919c 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -1,5 +1,6 @@ -# Copyright The IETF Trust 2023, All Rights Reserved +# Copyright The IETF Trust 2023-2024, All Rights Reserved +from collections import defaultdict import json from django.db.models import OuterRef, Subquery, Q @@ -18,7 +19,7 @@ from ietf.doc.factories import WgDraftFactory # DO NOT MERGE INTO MAIN from ietf.doc.models import Document, DocHistory from ietf.person.factories import PersonFactory # DO NOT MERGE INTO MAIN -from ietf.person.models import Person +from ietf.person.models import Email, Person @csrf_exempt @@ -203,6 +204,52 @@ def rfc_original_stream(request): return JsonResponse(response) +@csrf_exempt +@requires_api_token("ietf.api.views_rpc") +def persons_by_email(request): + if request.method != "POST": + return HttpResponseNotAllowed(["POST"]) + try: + emails = json.loads(request.body) + except json.JSONDecodeError: + return HttpResponseBadRequest() + response = [] + for email in Email.objects.filter(address__in=emails).exclude(person__isnull=True): + response.append({ + "email": email.address, + "person_pk": email.person.pk, + "name": email.person.name, + "last_name": email.person.last_name(), + "initials": email.person.initials(), + }) + return JsonResponse(response,safe=False) + + +@csrf_exempt +@requires_api_token("ietf.api.views_rpc") +def rfc_authors(request): + """Gather authors of the RFCs with the given numbers""" + if request.method != "POST": + return HttpResponseNotAllowed(["POST"]) + try: + rfc_numbers = json.loads(request.body) + except json.JSONDecodeError: + return HttpResponseBadRequest() + response = [] + for rfc in Document.objects.filter(type="rfc",rfc_number__in=rfc_numbers): + item={"rfc_number": rfc.rfc_number, "authors": []} + for author in rfc.authors(): + item_author=dict() + item_author["person_pk"] = author.pk + item_author["name"] = author.name + item_author["last_name"] = author.last_name() + item_author["initials"] = author.initials() + item_author["email_addresses"] = [address.lower() for address in author.email_set.values_list("address", flat=True)] + item["authors"].append(item_author) + response.append(item) + return JsonResponse(response, safe=False) + + @csrf_exempt @requires_api_token("ietf.api.views_rpc") def create_demo_person(request): diff --git a/rpcapi.yaml b/rpcapi.yaml index ef92192168..dd1acc31eb 100644 --- a/rpcapi.yaml +++ b/rpcapi.yaml @@ -28,6 +28,41 @@ paths: '404': description: Not found + /person/persons_by_email/: + post: + operationId: persons_by_email + summary: Get a batch of persons by email addresses + description: returns a list of objects with the email address and related person information + requestBody: + required: true + content: + application/json: + schema: + type: array + items: + type: string + responses: + '200': + description: OK + content: + application/json: + schema: + type: array + items: + type: object + properties: + email: + type: string + person_pk: + type: integer + name: + type: string + last_name: + type: string + initials: + type: string + + /persons/: post: operationId: get_persons @@ -174,6 +209,49 @@ paths: additionalProperties: $ref:'#/components/schemas/Draft' + /doc/rfc/authors/: + post: + operationId: get_rfc_authors + summary: Gather authors of the RFCs with the given numbers + description: returns a dict mapping rfc numbers to objects describing authors + requestBody: + required: true + content: + application/json: + schema: + type: array + items: + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + type: array + items: + type: object + properties: + rfc_number: + type: integer + authors: + type: array + items: + type: object + properties: + person_pk: + type: integer + name: + type: string + last_name: + type: string + initials: + type: string + email_addresses: + type: array + items: + type: string + /doc/rfc/original_stream/: get: operationId: get_rfc_original_streams From 1ab36cd3fd34fcdb4cec8ee41833e3aa128cdf01 Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Wed, 30 Oct 2024 17:19:37 -0500 Subject: [PATCH 28/64] feat: api to gather draft authors (#8126) * feat: api to gather draft authors * chore: black --- ietf/api/urls_rpc.py | 1 + ietf/api/views_rpc.py | 60 +++++++++++++++++++++++++++++++++---------- rpcapi.yaml | 43 +++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 13 deletions(-) diff --git a/ietf/api/urls_rpc.py b/ietf/api/urls_rpc.py index 85783bbc3f..f0ff26caed 100644 --- a/ietf/api/urls_rpc.py +++ b/ietf/api/urls_rpc.py @@ -11,6 +11,7 @@ url(r"^doc/submitted_to_rpc/$", views_rpc.submitted_to_rpc), url(r"^doc/rfc/original_stream/$", views_rpc.rfc_original_stream), url(r"^doc/rfc/authors/$", views_rpc.rfc_authors), + url(r"^doc/draft/authors/$", views_rpc.draft_authors), url(r"^person/create_demo_person/$", views_rpc.create_demo_person), url(r"^person/persons_by_email/$", views_rpc.persons_by_email), url(r"^person/(?P[0-9]+)$", views_rpc.rpc_person), diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index bddc56919c..1d7e12df9f 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -79,7 +79,7 @@ def _document_source_format(doc): return "txt" return "unknown" - + @csrf_exempt @requires_api_token("ietf.api.views_rpc") def rpc_draft(request, doc_id): @@ -113,6 +113,7 @@ def rpc_draft(request, doc_id): } ) + @csrf_exempt @requires_api_token("ietf.api.views_rpc") def drafts_by_names(request): @@ -215,14 +216,16 @@ def persons_by_email(request): return HttpResponseBadRequest() response = [] for email in Email.objects.filter(address__in=emails).exclude(person__isnull=True): - response.append({ - "email": email.address, - "person_pk": email.person.pk, - "name": email.person.name, - "last_name": email.person.last_name(), - "initials": email.person.initials(), - }) - return JsonResponse(response,safe=False) + response.append( + { + "email": email.address, + "person_pk": email.person.pk, + "name": email.person.name, + "last_name": email.person.last_name(), + "initials": email.person.initials(), + } + ) + return JsonResponse(response, safe=False) @csrf_exempt @@ -236,15 +239,46 @@ def rfc_authors(request): except json.JSONDecodeError: return HttpResponseBadRequest() response = [] - for rfc in Document.objects.filter(type="rfc",rfc_number__in=rfc_numbers): - item={"rfc_number": rfc.rfc_number, "authors": []} + for rfc in Document.objects.filter(type="rfc", rfc_number__in=rfc_numbers): + item = {"rfc_number": rfc.rfc_number, "authors": []} for author in rfc.authors(): - item_author=dict() + item_author = dict() + item_author["person_pk"] = author.pk + item_author["name"] = author.name + item_author["last_name"] = author.last_name() + item_author["initials"] = author.initials() + item_author["email_addresses"] = [ + address.lower() + for address in author.email_set.values_list("address", flat=True) + ] + item["authors"].append(item_author) + response.append(item) + return JsonResponse(response, safe=False) + + +@csrf_exempt +@requires_api_token("ietf.api.views_rpc") +def draft_authors(request): + """Gather authors of the RFCs with the given numbers""" + if request.method != "POST": + return HttpResponseNotAllowed(["POST"]) + try: + draft_names = json.loads(request.body) + except json.JSONDecodeError: + return HttpResponseBadRequest() + response = [] + for draft in Document.objects.filter(type="draft", name__in=draft_names): + item = {"draft_name": draft.name, "authors": []} + for author in draft.authors(): + item_author = dict() item_author["person_pk"] = author.pk item_author["name"] = author.name item_author["last_name"] = author.last_name() item_author["initials"] = author.initials() - item_author["email_addresses"] = [address.lower() for address in author.email_set.values_list("address", flat=True)] + item_author["email_addresses"] = [ + address.lower() + for address in author.email_set.values_list("address", flat=True) + ] item["authors"].append(item_author) response.append(item) return JsonResponse(response, safe=False) diff --git a/rpcapi.yaml b/rpcapi.yaml index dd1acc31eb..f0c90f173d 100644 --- a/rpcapi.yaml +++ b/rpcapi.yaml @@ -252,6 +252,49 @@ paths: items: type: string + /doc/draft/authors/: + post: + operationId: get_draft_authors + summary: Gather authors of the drafts with the given names + description: returns a dict mapping draft names to objects describing authors + requestBody: + required: true + content: + application/json: + schema: + type: array + items: + type: string + responses: + '200': + description: OK + content: + application/json: + schema: + type: array + items: + type: object + properties: + draft_name: + type: string + authors: + type: array + items: + type: object + properties: + person_pk: + type: integer + name: + type: string + last_name: + type: string + initials: + type: string + email_addresses: + type: array + items: + type: string + /doc/rfc/original_stream/: get: operationId: get_rfc_original_streams From ebc27104e300bbe2da7b6111595aff3938b3e990 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 21 Nov 2024 18:06:37 -0400 Subject: [PATCH 29/64] ci: tag feature branch release images --- .github/workflows/build.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f1044223dc..46cb150e05 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -256,6 +256,10 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Add feature-latest tag + if: ${{ startsWith(github.ref_name, 'feat/') }} + run: echo "FEATURE_LATEST_TAG=$(echo $GITHUB_REF_NAME | tr / -)" >> $GITHUB_ENV + - name: Build Images uses: docker/build-push-action@v6 env: @@ -265,7 +269,9 @@ jobs: file: dev/build/Dockerfile platforms: ${{ github.event.inputs.skiparm == 'true' && 'linux/amd64' || 'linux/amd64,linux/arm64' }} push: true - tags: ghcr.io/ietf-tools/datatracker:${{ env.PKG_VERSION }} + tags: | + ghcr.io/ietf-tools/datatracker:${{ env.PKG_VERSION }} + ${{ env.FEATURE_LATEST_TAG && format('ghcr.io/ietf-tools/datatracker:{0}-latest', env.FEATURE_LATEST_TAG) || null }} cache-from: type=gha cache-to: type=gha,mode=max From eba817522372b772a029b2c39466e9b110e4c991 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 21 Nov 2024 19:24:47 -0400 Subject: [PATCH 30/64] chore: fix git nonsense --- .github/workflows/build-base-app.yml | 132 +++++----- .github/workflows/tests.yml | 374 +++++++++++++-------------- 2 files changed, 253 insertions(+), 253 deletions(-) diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml index 609b641ef9..c8f66a22b7 100644 --- a/.github/workflows/build-base-app.yml +++ b/.github/workflows/build-base-app.yml @@ -1,66 +1,66 @@ -name: Build Base App Docker Image - -on: - push: - branches: - - 'main' - paths: - - 'docker/base.Dockerfile' - - 'requirements.txt' - - workflow_dispatch: - -jobs: - publish: - runs-on: ubuntu-latest - permissions: - contents: write - packages: write - - steps: - - uses: actions/checkout@v4 - with: - token: ${{ secrets.GH_COMMON_TOKEN }} - - - name: Set Version - run: | - printf -v CURDATE '%(%Y%m%dT%H%M)T' -1 - echo "IMGVERSION=$CURDATE" >> $GITHUB_ENV - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Docker Build & Push - uses: docker/build-push-action@v6 - env: - DOCKER_BUILD_NO_SUMMARY: true - with: - context: . - file: docker/base.Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - tags: | - ghcr.io/ietf-tools/datatracker-app-base:${{ env.IMGVERSION }} - ghcr.io/ietf-tools/datatracker-app-base:latest - - - name: Update version references - run: | - sed -i "1s/.*/FROM ghcr.io\/ietf-tools\/datatracker-app-base:${{ env.IMGVERSION }}/" dev/build/Dockerfile - echo "${{ env.IMGVERSION }}" > dev/build/TARGET_BASE - - - name: Commit CHANGELOG.md - uses: stefanzweifel/git-auto-commit-action@v5 - with: - branch: main - commit_message: 'ci: update base image target version to ${{ env.IMGVERSION }}' - file_pattern: dev/build/Dockerfile dev/build/TARGET_BASE +name: Build Base App Docker Image + +on: + push: + branches: + - 'main' + paths: + - 'docker/base.Dockerfile' + - 'requirements.txt' + + workflow_dispatch: + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: write + packages: write + + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.GH_COMMON_TOKEN }} + + - name: Set Version + run: | + printf -v CURDATE '%(%Y%m%dT%H%M)T' -1 + echo "IMGVERSION=$CURDATE" >> $GITHUB_ENV + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker Build & Push + uses: docker/build-push-action@v6 + env: + DOCKER_BUILD_NO_SUMMARY: true + with: + context: . + file: docker/base.Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ghcr.io/ietf-tools/datatracker-app-base:${{ env.IMGVERSION }} + ghcr.io/ietf-tools/datatracker-app-base:latest + + - name: Update version references + run: | + sed -i "1s/.*/FROM ghcr.io\/ietf-tools\/datatracker-app-base:${{ env.IMGVERSION }}/" dev/build/Dockerfile + echo "${{ env.IMGVERSION }}" > dev/build/TARGET_BASE + + - name: Commit CHANGELOG.md + uses: stefanzweifel/git-auto-commit-action@v5 + with: + branch: main + commit_message: 'ci: update base image target version to ${{ env.IMGVERSION }}' + file_pattern: dev/build/Dockerfile dev/build/TARGET_BASE diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8eb6adc953..5457415f59 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,187 +1,187 @@ -name: Reusable Tests Workflow - -on: - workflow_call: - inputs: - ignoreLowerCoverage: - description: 'Ignore Lower Coverage' - default: false - required: true - type: boolean - skipSelenium: - description: 'Skip Selenium Tests' - default: false - required: false - type: boolean - targetBaseVersion: - description: 'Target Base Image Version' - default: latest - required: false - type: string - -jobs: - tests-python: - name: Python Tests - runs-on: ubuntu-latest - container: ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.targetBaseVersion }} - - services: - db: - image: ghcr.io/ietf-tools/datatracker-db:latest - - steps: - - uses: actions/checkout@v4 - - - name: Prepare for tests - run: | - chmod +x ./dev/tests/prepare.sh - sh ./dev/tests/prepare.sh - - - name: Ensure DB is ready - run: | - /usr/local/bin/wait-for db:5432 -- echo "DB ready" - - - name: Run all tests - shell: bash - run: | - echo "Running checks..." - ./ietf/manage.py check - ./ietf/manage.py migrate --fake-initial - echo "Validating migrations..." - if ! ( ietf/manage.py makemigrations --dry-run --check --verbosity 3 ) ; then - echo "Model changes without migrations found." - exit 1 - fi - if [[ "x${{ inputs.skipSelenium }}" == "xtrue" ]]; then - echo "Disable selenium tests..." - rm /usr/bin/geckodriver - fi - echo "Running tests..." - if [[ "x${{ inputs.ignoreLowerCoverage }}" == "xtrue" ]]; then - echo "Lower coverage failures will be ignored." - HOME=/root ./ietf/manage.py test -v2 --validate-html-harder --settings=settings_test --ignore-lower-coverage - else - HOME=/root ./ietf/manage.py test -v2 --validate-html-harder --settings=settings_test - fi - coverage xml - - - name: Upload geckodriver.log - uses: actions/upload-artifact@v4 - if: ${{ failure() }} - with: - name: geckodriverlog - path: geckodriver.log - - - name: Upload Coverage Results to Codecov - uses: codecov/codecov-action@v5 - with: - disable_search: true - files: coverage.xml - token: ${{ secrets.CODECOV_TOKEN }} - - - name: Convert Coverage Results - if: ${{ always() }} - run: | - mv latest-coverage.json coverage.json - - - name: Upload Coverage Results as Build Artifact - uses: actions/upload-artifact@v4 - if: ${{ always() }} - with: - name: coverage - path: coverage.json - - tests-playwright: - name: Playwright Tests - runs-on: macos-latest - strategy: - fail-fast: false - matrix: - project: [chromium, firefox] - - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-node@v4 - with: - node-version: '18' - - - name: Run all tests - run: | - echo "Installing dependencies..." - yarn - echo "Installing Playwright..." - cd playwright - mkdir test-results - npm ci - npx playwright install --with-deps ${{ matrix.project }} - echo "Running tests..." - npx playwright test --project=${{ matrix.project }} - - - name: Upload Report - uses: actions/upload-artifact@v4 - if: ${{ always() }} - continue-on-error: true - with: - name: playwright-results-${{ matrix.project }} - path: playwright/test-results/ - if-no-files-found: ignore - - tests-playwright-legacy: - name: Playwright Legacy Tests - runs-on: ubuntu-latest - container: ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.targetBaseVersion }} - strategy: - fail-fast: false - matrix: - project: [chromium, firefox] - - services: - db: - image: ghcr.io/ietf-tools/datatracker-db:latest - - steps: - - uses: actions/checkout@v4 - - - name: Prepare for tests - run: | - chmod +x ./dev/tests/prepare.sh - sh ./dev/tests/prepare.sh - - - name: Ensure DB is ready - run: | - /usr/local/bin/wait-for db:5432 -- echo "DB ready" - - - name: Start Datatracker - run: | - echo "Running checks..." - ./ietf/manage.py check - ./ietf/manage.py migrate --fake-initial - echo "Starting datatracker..." - ./ietf/manage.py runserver 0.0.0.0:8000 --settings=settings_local & - echo "Waiting for datatracker to be ready..." - /usr/local/bin/wait-for localhost:8000 -- echo "Datatracker ready" - - - name: Run all tests - env: - # Required to get firefox to run as root: - HOME: "" - run: | - echo "Installing dependencies..." - yarn - echo "Installing Playwright..." - cd playwright - mkdir test-results - npm ci - npx playwright install --with-deps ${{ matrix.project }} - echo "Running tests..." - npx playwright test --project=${{ matrix.project }} -c playwright-legacy.config.js - - - name: Upload Report - uses: actions/upload-artifact@v4 - if: ${{ always() }} - continue-on-error: true - with: - name: playwright-legacy-results-${{ matrix.project }} - path: playwright/test-results/ - if-no-files-found: ignore +name: Reusable Tests Workflow + +on: + workflow_call: + inputs: + ignoreLowerCoverage: + description: 'Ignore Lower Coverage' + default: false + required: true + type: boolean + skipSelenium: + description: 'Skip Selenium Tests' + default: false + required: false + type: boolean + targetBaseVersion: + description: 'Target Base Image Version' + default: latest + required: false + type: string + +jobs: + tests-python: + name: Python Tests + runs-on: ubuntu-latest + container: ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.targetBaseVersion }} + + services: + db: + image: ghcr.io/ietf-tools/datatracker-db:latest + + steps: + - uses: actions/checkout@v4 + + - name: Prepare for tests + run: | + chmod +x ./dev/tests/prepare.sh + sh ./dev/tests/prepare.sh + + - name: Ensure DB is ready + run: | + /usr/local/bin/wait-for db:5432 -- echo "DB ready" + + - name: Run all tests + shell: bash + run: | + echo "Running checks..." + ./ietf/manage.py check + ./ietf/manage.py migrate --fake-initial + echo "Validating migrations..." + if ! ( ietf/manage.py makemigrations --dry-run --check --verbosity 3 ) ; then + echo "Model changes without migrations found." + exit 1 + fi + if [[ "x${{ inputs.skipSelenium }}" == "xtrue" ]]; then + echo "Disable selenium tests..." + rm /usr/bin/geckodriver + fi + echo "Running tests..." + if [[ "x${{ inputs.ignoreLowerCoverage }}" == "xtrue" ]]; then + echo "Lower coverage failures will be ignored." + HOME=/root ./ietf/manage.py test -v2 --validate-html-harder --settings=settings_test --ignore-lower-coverage + else + HOME=/root ./ietf/manage.py test -v2 --validate-html-harder --settings=settings_test + fi + coverage xml + + - name: Upload geckodriver.log + uses: actions/upload-artifact@v4 + if: ${{ failure() }} + with: + name: geckodriverlog + path: geckodriver.log + + - name: Upload Coverage Results to Codecov + uses: codecov/codecov-action@v5 + with: + disable_search: true + files: coverage.xml + token: ${{ secrets.CODECOV_TOKEN }} + + - name: Convert Coverage Results + if: ${{ always() }} + run: | + mv latest-coverage.json coverage.json + + - name: Upload Coverage Results as Build Artifact + uses: actions/upload-artifact@v4 + if: ${{ always() }} + with: + name: coverage + path: coverage.json + + tests-playwright: + name: Playwright Tests + runs-on: macos-latest + strategy: + fail-fast: false + matrix: + project: [chromium, firefox] + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: '18' + + - name: Run all tests + run: | + echo "Installing dependencies..." + yarn + echo "Installing Playwright..." + cd playwright + mkdir test-results + npm ci + npx playwright install --with-deps ${{ matrix.project }} + echo "Running tests..." + npx playwright test --project=${{ matrix.project }} + + - name: Upload Report + uses: actions/upload-artifact@v4 + if: ${{ always() }} + continue-on-error: true + with: + name: playwright-results-${{ matrix.project }} + path: playwright/test-results/ + if-no-files-found: ignore + + tests-playwright-legacy: + name: Playwright Legacy Tests + runs-on: ubuntu-latest + container: ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.targetBaseVersion }} + strategy: + fail-fast: false + matrix: + project: [chromium, firefox] + + services: + db: + image: ghcr.io/ietf-tools/datatracker-db:latest + + steps: + - uses: actions/checkout@v4 + + - name: Prepare for tests + run: | + chmod +x ./dev/tests/prepare.sh + sh ./dev/tests/prepare.sh + + - name: Ensure DB is ready + run: | + /usr/local/bin/wait-for db:5432 -- echo "DB ready" + + - name: Start Datatracker + run: | + echo "Running checks..." + ./ietf/manage.py check + ./ietf/manage.py migrate --fake-initial + echo "Starting datatracker..." + ./ietf/manage.py runserver 0.0.0.0:8000 --settings=settings_local & + echo "Waiting for datatracker to be ready..." + /usr/local/bin/wait-for localhost:8000 -- echo "Datatracker ready" + + - name: Run all tests + env: + # Required to get firefox to run as root: + HOME: "" + run: | + echo "Installing dependencies..." + yarn + echo "Installing Playwright..." + cd playwright + mkdir test-results + npm ci + npx playwright install --with-deps ${{ matrix.project }} + echo "Running tests..." + npx playwright test --project=${{ matrix.project }} -c playwright-legacy.config.js + + - name: Upload Report + uses: actions/upload-artifact@v4 + if: ${{ always() }} + continue-on-error: true + with: + name: playwright-legacy-results-${{ matrix.project }} + path: playwright/test-results/ + if-no-files-found: ignore From a90cbadcacc98eb6f46b7e29c5493e067874268d Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 9 Dec 2024 12:34:43 -0400 Subject: [PATCH 31/64] feat: API for RFC metadata fetch/filtering (#8291) * feat: RFC API for rfceditor website (WIP) * feat: pagination + ordered RFC index * feat: filter by publication date * feat: stream + stream filtering * feat: DOI * feat: group/area for RFCs * feat: group/area filtering * feat: result sorting * refactor: send rfc number, not name * feat: search rfc title/abstract * style: Black * feat: add 'status' field * feat: filter by 'status' * style: remove redundant parentheses * feat: add updated_by/obsoleted_by fields --- ietf/api/urls.py | 17 +++-- ietf/doc/api.py | 88 ++++++++++++++++++++++ ietf/doc/serializers.py | 153 ++++++++++++++++++++++++++++++++++++++ ietf/group/models.py | 3 + ietf/group/serializers.py | 11 +++ ietf/name/serializers.py | 11 +++ ietf/settings.py | 2 + requirements.txt | 1 + 8 files changed, 280 insertions(+), 6 deletions(-) create mode 100644 ietf/doc/api.py create mode 100644 ietf/doc/serializers.py create mode 100644 ietf/group/serializers.py create mode 100644 ietf/name/serializers.py diff --git a/ietf/api/urls.py b/ietf/api/urls.py index eeca2bdbd2..caf18b48e1 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -1,26 +1,30 @@ # Copyright The IETF Trust 2017-2024, All Rights Reserved +from drf_spectacular.views import SpectacularAPIView + from django.conf import settings -from django.urls import include +from django.urls import include, path from django.views.generic import TemplateView from ietf import api -from ietf.doc import views_ballot +from ietf.doc import views_ballot, api as doc_api from ietf.meeting import views as meeting_views from ietf.submit import views as submit_views from ietf.utils.urls import url from . import views as api_views +from .routers import PrefixedSimpleRouter # DRF API routing - disabled until we plan to use it -# from drf_spectacular.views import SpectacularAPIView -# from django.urls import path # from ietf.person import api as person_api -# from .routers import PrefixedSimpleRouter # core_router = PrefixedSimpleRouter(name_prefix="ietf.api.core_api") # core api router # core_router.register("email", person_api.EmailViewSet) # core_router.register("person", person_api.PersonViewSet) +# todo more general name for this API? +red_router = PrefixedSimpleRouter(name_prefix="ietf.api.red_api") # red api router +red_router.register("doc", doc_api.RfcViewSet) + api.autodiscover() urlpatterns = [ @@ -32,7 +36,8 @@ url(r'^v2/person/person', api_views.ApiV2PersonExportView.as_view()), # --- DRF API --- # path("core/", include(core_router.urls)), - # path("schema/", SpectacularAPIView.as_view()), + path("red/", include(red_router.urls)), + path("schema/", SpectacularAPIView.as_view()), # # --- Custom API endpoints, sorted alphabetically --- # Email alias information for drafts diff --git a/ietf/doc/api.py b/ietf/doc/api.py new file mode 100644 index 0000000000..9102dc028f --- /dev/null +++ b/ietf/doc/api.py @@ -0,0 +1,88 @@ +# Copyright The IETF Trust 2024, All Rights Reserved +"""Doc API implementations""" +from django.db.models import OuterRef, Subquery, Prefetch +from django.db.models.functions import TruncDate +from django_filters import rest_framework as filters +from rest_framework import filters as drf_filters +from rest_framework.mixins import ListModelMixin, RetrieveModelMixin +from rest_framework.pagination import LimitOffsetPagination +from rest_framework.viewsets import GenericViewSet + +from ietf.group.models import Group +from ietf.name.models import StreamName +from ietf.utils.timezone import RPC_TZINFO +from .models import Document, DocEvent, RelatedDocument +from .serializers import RfcMetadataSerializer, RfcStatus + + +class RfcLimitOffsetPagination(LimitOffsetPagination): + default_limit = 10 + max_limit = 500 + + +class RfcFilter(filters.FilterSet): + published = filters.DateFromToRangeFilter() + stream = filters.ModelMultipleChoiceFilter( + queryset=StreamName.objects.filter(used=True) + ) + group = filters.ModelMultipleChoiceFilter( + queryset=Group.objects.wgs(), + field_name="group__acronym", + to_field_name="acronym", + ) + area = filters.ModelMultipleChoiceFilter( + queryset=Group.objects.areas(), + field_name="group__parent__acronym", + to_field_name="acronym", + ) + status = filters.MultipleChoiceFilter( + choices=[(slug, slug) for slug in RfcStatus.status_slugs], + method=RfcStatus.filter, + ) + sort = filters.OrderingFilter( + fields=( + ("rfc_number", "number"), # ?sort=number / ?sort=-number + ("published", "published"), # ?sort=published / ?sort=-published + ), + ) + + +class RfcViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): + permission_classes = [] + queryset = ( + Document.objects.filter(type_id="rfc") + .annotate( + published_datetime=Subquery( + DocEvent.objects.filter( + doc_id=OuterRef("pk"), + type="published_rfc", + ) + .order_by("-time") + .values("time")[:1] + ), + ) + .annotate(published=TruncDate("published_datetime", tzinfo=RPC_TZINFO)) + .order_by("-rfc_number") + .prefetch_related( + Prefetch( + "targets_related", # relationship to follow + queryset=RelatedDocument.objects.filter( + source__type_id="rfc", relationship_id="obs" + ), + to_attr="obsoleted_by", # attr to add to queryset instances + ), + Prefetch( + "targets_related", # relationship to follow + queryset=RelatedDocument.objects.filter( + source__type_id="rfc", relationship_id="updates" + ), + to_attr="updated_by", # attr to add to queryset instances + ), + ) + ) # default ordering - RfcFilter may override + + serializer_class = RfcMetadataSerializer + pagination_class = RfcLimitOffsetPagination + filter_backends = [filters.DjangoFilterBackend, drf_filters.SearchFilter] + filterset_class = RfcFilter + search_fields = ["title", "abstract"] diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py new file mode 100644 index 0000000000..fee2e4be4b --- /dev/null +++ b/ietf/doc/serializers.py @@ -0,0 +1,153 @@ +# Copyright The IETF Trust 2024, All Rights Reserved +"""django-rest-framework serializers""" +from dataclasses import dataclass +from typing import Literal, ClassVar + +from drf_spectacular.utils import extend_schema_field +from rest_framework import serializers, fields + +from ietf.group.serializers import GroupSerializer +from ietf.name.serializers import StreamNameSerializer +from .models import Document, DocumentAuthor, RelatedDocument + + +class RfcAuthorSerializer(serializers.ModelSerializer): + """Serializer for a DocumentAuthor in a response""" + + name = fields.CharField(source="person.plain_name") + email = fields.EmailField(source="email.address", required=False) + + class Meta: + model = DocumentAuthor + fields = ["person", "name", "email", "affiliation", "country"] + + +@dataclass +class DocIdentifier: + type: Literal["doi", "issn"] + value: str + + +class DocIdentifierSerializer(serializers.Serializer): + type = serializers.ChoiceField(choices=["doi", "issn"]) + value = serializers.CharField() + + +# This should become "type RfcStatusSlugT ..." when we drop pre-py3.12 support +# It should be "RfcStatusSlugT: TypeAlias ..." when we drop py3.9 support +RfcStatusSlugT = Literal[ + "standard", "bcp", "informational", "experimental", "historic", "unknown" +] + + +@dataclass +class RfcStatus: + """Helper to extract the 'Status' from an RFC document for serialization""" + + slug: RfcStatusSlugT + + # Names that aren't just the slug itself. ClassVar annotation prevents dataclass from treating this as a field. + fancy_names: ClassVar[dict[RfcStatusSlugT, str]] = { + "standard": "standards track", + "bcp": "best current practice", + } + + # ClassVar annotation prevents dataclass from treating this as a field + stdlevelname_slug_map: ClassVar[dict[str, RfcStatusSlugT]] = { + "bcp": "bcp", + "ds": "standard", # ds is obsolete + "exp": "experimental", + "hist": "historic", + "inf": "informational", + "std": "standard", + "ps": "standard", + "unkn": "unknown", + } + + # ClassVar annotation prevents dataclass from treating this as a field + status_slugs: ClassVar[list[RfcStatusSlugT]] = sorted( + set(stdlevelname_slug_map.values()) + ) + + @property + def name(self): + return RfcStatus.fancy_names.get(self.slug, self.slug) + + @classmethod + def from_document(cls, doc: Document): + """Decide the status that applies to a document""" + return cls( + slug=(cls.stdlevelname_slug_map.get(doc.std_level.slug, "unknown")), + ) + + @classmethod + def filter(cls, queryset, name, value: list[RfcStatusSlugT]): + """Filter a queryset by status + + This is basically the inverse of the from_document() method. Given a status name, filter + the queryset to those in that status. The queryset should be a Document queryset. + """ + interesting_slugs = [ + stdlevelname_slug + for stdlevelname_slug, status_slug in cls.stdlevelname_slug_map.items() + if status_slug in value + ] + if len(interesting_slugs) == 0: + return queryset.none() + return queryset.filter(std_level__slug__in=interesting_slugs) + + +class RfcStatusSerializer(serializers.Serializer): + """Status serializer for a Document instance""" + + slug = serializers.ChoiceField(choices=RfcStatus.status_slugs) + name = serializers.CharField() + + def to_representation(self, instance: Document): + return super().to_representation(instance=RfcStatus.from_document(instance)) + + +class RelatedRfcSerializer(serializers.Serializer): + id = serializers.IntegerField(source="source.id") + number = serializers.IntegerField(source="source.rfc_number") + title = serializers.CharField(source="source.title") + + +class RfcMetadataSerializer(serializers.ModelSerializer): + number = serializers.IntegerField(source="rfc_number") + published = serializers.DateField() + status = RfcStatusSerializer(source="*") + authors = RfcAuthorSerializer(many=True, source="documentauthor_set") + group = GroupSerializer() + area = GroupSerializer(source="group.area", required=False) + stream = StreamNameSerializer() + identifiers = fields.SerializerMethodField() + obsoleted_by = RelatedRfcSerializer(many=True, read_only=True) + updated_by = RelatedRfcSerializer(many=True, read_only=True) + + class Meta: + model = Document + fields = [ + "id", + "number", + "title", + "published", + "status", + "pages", + "authors", + "group", + "area", + "stream", + "identifiers", + "obsoleted_by", + "updated_by", + ] + + @extend_schema_field(DocIdentifierSerializer) + def get_identifiers(self, doc: Document): + identifiers = [] + if doc.rfc_number: + identifiers.append( + DocIdentifier(type="doi", value=f"10.17487/RFC{doc.rfc_number:04d}") + ) + return DocIdentifierSerializer(instance=identifiers, many=True).data diff --git a/ietf/group/models.py b/ietf/group/models.py index 52549e8cc1..5e0983579e 100644 --- a/ietf/group/models.py +++ b/ietf/group/models.py @@ -112,6 +112,9 @@ def active_wgs(self): def closed_wgs(self): return self.wgs().exclude(state__in=Group.ACTIVE_STATE_IDS) + def areas(self): + return self.get_queryset().filter(type="area") + def with_meetings(self): return self.get_queryset().filter(type__features__has_meetings=True) diff --git a/ietf/group/serializers.py b/ietf/group/serializers.py new file mode 100644 index 0000000000..b123e091e3 --- /dev/null +++ b/ietf/group/serializers.py @@ -0,0 +1,11 @@ +# Copyright The IETF Trust 2024, All Rights Reserved +"""django-rest-framework serializers""" +from rest_framework import serializers + +from .models import Group + + +class GroupSerializer(serializers.ModelSerializer): + class Meta: + model = Group + fields = ["acronym", "name"] diff --git a/ietf/name/serializers.py b/ietf/name/serializers.py new file mode 100644 index 0000000000..a764f56051 --- /dev/null +++ b/ietf/name/serializers.py @@ -0,0 +1,11 @@ +# Copyright The IETF Trust 2024, All Rights Reserved +"""django-rest-framework serializers""" +from rest_framework import serializers + +from .models import StreamName + + +class StreamNameSerializer(serializers.ModelSerializer): + class Meta: + model = StreamName + fields = ["slug", "name", "desc"] diff --git a/ietf/settings.py b/ietf/settings.py index 6990037585..0788ab8c93 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -19,6 +19,7 @@ warnings.filterwarnings("ignore", module="tastypie", message="The django.utils.datetime_safe module is deprecated.") warnings.filterwarnings("ignore", module="oidc_provider", message="The django.utils.timezone.utc alias is deprecated.") warnings.filterwarnings("ignore", message="The USE_DEPRECATED_PYTZ setting,") # https://github.com/ietf-tools/datatracker/issues/5635 +warnings.filterwarnings("ignore", message="The is_dst argument to make_aware\\(\\)") # caused by django-filters when USE_DEPRECATED_PYTZ is true warnings.filterwarnings("ignore", message="The USE_L10N setting is deprecated.") # https://github.com/ietf-tools/datatracker/issues/5648 warnings.filterwarnings("ignore", message="django.contrib.auth.hashers.CryptPasswordHasher is deprecated.") # https://github.com/ietf-tools/datatracker/issues/5663 warnings.filterwarnings("ignore", message="'urllib3\\[secure\\]' extra is deprecated") @@ -454,6 +455,7 @@ def skip_unreadable_post(record): 'django_celery_beat', 'corsheaders', 'django_markup', + 'django_filters', 'oidc_provider', 'drf_spectacular', 'drf_standardized_errors', diff --git a/requirements.txt b/requirements.txt index f974113d8f..a78d2f2ec9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,6 +16,7 @@ django-celery-beat>=2.3.0 django-csp>=3.7 django-cors-headers>=3.11.0 django-debug-toolbar>=3.2.4 +django-filter>=24.3 django-markup>=1.5 # Limited use - need to reconcile against direct use of markdown django-oidc-provider>=0.8.1 # 0.8 dropped Django 2 support django-referrer-policy>=1.0 From ba3bad5421c3a02bc493cfa3ad64b26e3d5e8170 Mon Sep 17 00:00:00 2001 From: Matthew Holloway Date: Thu, 12 Dec 2024 14:16:56 +1300 Subject: [PATCH 32/64] feat: add 'abstract' to rpc api --- ietf/doc/serializers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py index fee2e4be4b..9ffa2efd54 100644 --- a/ietf/doc/serializers.py +++ b/ietf/doc/serializers.py @@ -141,6 +141,7 @@ class Meta: "identifiers", "obsoleted_by", "updated_by", + "abstract", ] @extend_schema_field(DocIdentifierSerializer) From 8eae9786a92f427842d41d1e5bf2af5759e9f61c Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 12 Dec 2024 21:51:00 -0400 Subject: [PATCH 33/64] chore: fix unused/duplicate imports --- ietf/api/ietf_utils.py | 3 --- ietf/doc/serializers.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/ietf/api/ietf_utils.py b/ietf/api/ietf_utils.py index 7816fe5634..50767a5afd 100644 --- a/ietf/api/ietf_utils.py +++ b/ietf/api/ietf_utils.py @@ -5,9 +5,6 @@ from functools import wraps from typing import Callable, Optional, Union -from functools import wraps -from typing import Callable, Optional, Union - from django.conf import settings from django.http import HttpResponseForbidden diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py index 9ffa2efd54..4a4cda9f2d 100644 --- a/ietf/doc/serializers.py +++ b/ietf/doc/serializers.py @@ -8,7 +8,7 @@ from ietf.group.serializers import GroupSerializer from ietf.name.serializers import StreamNameSerializer -from .models import Document, DocumentAuthor, RelatedDocument +from .models import Document, DocumentAuthor class RfcAuthorSerializer(serializers.ModelSerializer): From e8969286c9627c7dba095bf40541c2631b601d1c Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 12 Dec 2024 21:53:34 -0400 Subject: [PATCH 34/64] chore: fix mypy lint --- ietf/doc/api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ietf/doc/api.py b/ietf/doc/api.py index 9102dc028f..663ee15ef7 100644 --- a/ietf/doc/api.py +++ b/ietf/doc/api.py @@ -6,6 +6,7 @@ from rest_framework import filters as drf_filters from rest_framework.mixins import ListModelMixin, RetrieveModelMixin from rest_framework.pagination import LimitOffsetPagination +from rest_framework.permissions import BasePermission from rest_framework.viewsets import GenericViewSet from ietf.group.models import Group @@ -48,7 +49,7 @@ class RfcFilter(filters.FilterSet): class RfcViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): - permission_classes = [] + permission_classes: list[BasePermission] = [] queryset = ( Document.objects.filter(type_id="rfc") .annotate( From d245312e956baaf9cce88d9227845580e4b65c78 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 12 Dec 2024 22:38:07 -0400 Subject: [PATCH 35/64] chore: unused import --- ietf/api/views_rpc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 1d7e12df9f..65983ff213 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -1,6 +1,5 @@ # Copyright The IETF Trust 2023-2024, All Rights Reserved -from collections import defaultdict import json from django.db.models import OuterRef, Subquery, Q From d60dba14c8cff6bc86d6f89533cfe6d718ca5573 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 19 Dec 2024 16:55:14 -0400 Subject: [PATCH 36/64] feat: retrieve single rfc, including text (#8346) * feat: retrieve single rfc Use RFC number instead of doc PK as id * feat: include text in single-rfc response * chore: drop doc id from api response --- ietf/doc/api.py | 12 ++++++++---- ietf/doc/serializers.py | 13 ++++++++++++- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/ietf/doc/api.py b/ietf/doc/api.py index 663ee15ef7..671d1686df 100644 --- a/ietf/doc/api.py +++ b/ietf/doc/api.py @@ -13,7 +13,7 @@ from ietf.name.models import StreamName from ietf.utils.timezone import RPC_TZINFO from .models import Document, DocEvent, RelatedDocument -from .serializers import RfcMetadataSerializer, RfcStatus +from .serializers import RfcMetadataSerializer, RfcStatus, RfcSerializer class RfcLimitOffsetPagination(LimitOffsetPagination): @@ -50,8 +50,9 @@ class RfcFilter(filters.FilterSet): class RfcViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): permission_classes: list[BasePermission] = [] + lookup_field = "rfc_number" queryset = ( - Document.objects.filter(type_id="rfc") + Document.objects.filter(type_id="rfc", rfc_number__isnull=False) .annotate( published_datetime=Subquery( DocEvent.objects.filter( @@ -81,9 +82,12 @@ class RfcViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): ), ) ) # default ordering - RfcFilter may override - - serializer_class = RfcMetadataSerializer pagination_class = RfcLimitOffsetPagination filter_backends = [filters.DjangoFilterBackend, drf_filters.SearchFilter] filterset_class = RfcFilter search_fields = ["title", "abstract"] + + def get_serializer_class(self): + if self.action == "retrieve": + return RfcSerializer + return RfcMetadataSerializer diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py index 4a4cda9f2d..d5f7bddd05 100644 --- a/ietf/doc/serializers.py +++ b/ietf/doc/serializers.py @@ -114,6 +114,8 @@ class RelatedRfcSerializer(serializers.Serializer): class RfcMetadataSerializer(serializers.ModelSerializer): + """Serialize metadata of an RFC""" + number = serializers.IntegerField(source="rfc_number") published = serializers.DateField() status = RfcStatusSerializer(source="*") @@ -128,7 +130,6 @@ class RfcMetadataSerializer(serializers.ModelSerializer): class Meta: model = Document fields = [ - "id", "number", "title", "published", @@ -152,3 +153,13 @@ def get_identifiers(self, doc: Document): DocIdentifier(type="doi", value=f"10.17487/RFC{doc.rfc_number:04d}") ) return DocIdentifierSerializer(instance=identifiers, many=True).data + + +class RfcSerializer(RfcMetadataSerializer): + """Serialize an RFC, including its metadata and text content if available""" + + text = serializers.CharField(allow_null=True) + + class Meta: + model = RfcMetadataSerializer.Meta.model + fields = RfcMetadataSerializer.Meta.fields + ["text"] From c48e6e765db269578ebedd9bcd639c4105338a4f Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 15 Jan 2025 19:14:50 -0400 Subject: [PATCH 37/64] fix: many=True for identifiers (#8425) --- ietf/doc/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py index d5f7bddd05..da2054ee60 100644 --- a/ietf/doc/serializers.py +++ b/ietf/doc/serializers.py @@ -145,7 +145,7 @@ class Meta: "abstract", ] - @extend_schema_field(DocIdentifierSerializer) + @extend_schema_field(DocIdentifierSerializer(many=True)) def get_identifiers(self, doc: Document): identifiers = [] if doc.rfc_number: From 151c9360567e67b8e82da24fb95f5621aca8d05c Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 31 Jan 2025 17:34:05 -0400 Subject: [PATCH 38/64] feat: add more rfc api fields (many stubs) --- ietf/doc/api.py | 55 +++++++++++++++++++++++++++++++---------- ietf/doc/serializers.py | 36 ++++++++++++++++++++++++--- 2 files changed, 75 insertions(+), 16 deletions(-) diff --git a/ietf/doc/api.py b/ietf/doc/api.py index 671d1686df..2fecdfa239 100644 --- a/ietf/doc/api.py +++ b/ietf/doc/api.py @@ -1,6 +1,6 @@ # Copyright The IETF Trust 2024, All Rights Reserved """Doc API implementations""" -from django.db.models import OuterRef, Subquery, Prefetch +from django.db.models import OuterRef, Subquery, Prefetch, Value, JSONField from django.db.models.functions import TruncDate from django_filters import rest_framework as filters from rest_framework import filters as drf_filters @@ -48,6 +48,27 @@ class RfcFilter(filters.FilterSet): ) +class PrefetchRelatedDocument(Prefetch): + """Prefetch via a RelatedDocument + + Prefetches following RelatedDocument relationships to other docs. By default, includes + those for which the current RFC is the `source`. If `reverse` is True, includes those + for which it is the `target` instead. Defaults to only "rfc" documents. + """ + + def __init__(self, to_attr, relationship_id, reverse=False, doc_type_id="rfc"): + super().__init__( + lookup="targets_related" if reverse else "relateddocument_set", + queryset=RelatedDocument.objects.filter( + **{ + "relationship_id": relationship_id, + f"{'source' if reverse else 'target'}__type_id": doc_type_id, + } + ), + to_attr=to_attr, + ) + + class RfcViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): permission_classes: list[BasePermission] = [] lookup_field = "rfc_number" @@ -66,21 +87,29 @@ class RfcViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): .annotate(published=TruncDate("published_datetime", tzinfo=RPC_TZINFO)) .order_by("-rfc_number") .prefetch_related( - Prefetch( - "targets_related", # relationship to follow - queryset=RelatedDocument.objects.filter( - source__type_id="rfc", relationship_id="obs" - ), - to_attr="obsoleted_by", # attr to add to queryset instances + PrefetchRelatedDocument( + to_attr="drafts", + relationship_id="became_rfc", + doc_type_id="draft", + reverse=True, + ), + PrefetchRelatedDocument(to_attr="obsoletes", relationship_id="obs"), + PrefetchRelatedDocument( + to_attr="obsoleted_by", relationship_id="obs", reverse=True ), - Prefetch( - "targets_related", # relationship to follow - queryset=RelatedDocument.objects.filter( - source__type_id="rfc", relationship_id="updates" - ), - to_attr="updated_by", # attr to add to queryset instances + PrefetchRelatedDocument(to_attr="updates", relationship_id="updates"), + PrefetchRelatedDocument( + to_attr="updated_by", relationship_id="updates", reverse=True ), ) + .annotate( + # TODO implement these fake fields for real + is_also=Value([], output_field=JSONField()), + see_also=Value([], output_field=JSONField()), + formats=Value(["txt", "xml"], output_field=JSONField()), + keywords=Value(["keyword"], output_field=JSONField()), + errata=Value([], output_field=JSONField()), + ) ) # default ordering - RfcFilter may override pagination_class = RfcLimitOffsetPagination filter_backends = [filters.DjangoFilterBackend, drf_filters.SearchFilter] diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py index da2054ee60..355022a15e 100644 --- a/ietf/doc/serializers.py +++ b/ietf/doc/serializers.py @@ -66,7 +66,8 @@ class RfcStatus: # ClassVar annotation prevents dataclass from treating this as a field status_slugs: ClassVar[list[RfcStatusSlugT]] = sorted( - set(stdlevelname_slug_map.values()) + # TODO implement "not-issued" RFCs + set(stdlevelname_slug_map.values()) | {"not-issued"} ) @property @@ -107,7 +108,19 @@ def to_representation(self, instance: Document): return super().to_representation(instance=RfcStatus.from_document(instance)) +class RelatedDraftSerializer(serializers.ModelSerializer): + class Meta: + model = Document + fields = ["id", "name", "title"] + + class RelatedRfcSerializer(serializers.Serializer): + id = serializers.IntegerField(source="target.id") + number = serializers.IntegerField(source="target.rfc_number") + title = serializers.CharField(source="target.title") + + +class ReverseRelatedRfcSerializer(serializers.Serializer): id = serializers.IntegerField(source="source.id") number = serializers.IntegerField(source="source.rfc_number") title = serializers.CharField(source="source.title") @@ -115,6 +128,7 @@ class RelatedRfcSerializer(serializers.Serializer): class RfcMetadataSerializer(serializers.ModelSerializer): """Serialize metadata of an RFC""" + RFC_FORMATS = ("xml", "txt", "html", "htmlized", "pdf") number = serializers.IntegerField(source="rfc_number") published = serializers.DateField() @@ -124,8 +138,16 @@ class RfcMetadataSerializer(serializers.ModelSerializer): area = GroupSerializer(source="group.area", required=False) stream = StreamNameSerializer() identifiers = fields.SerializerMethodField() - obsoleted_by = RelatedRfcSerializer(many=True, read_only=True) - updated_by = RelatedRfcSerializer(many=True, read_only=True) + draft = RelatedDraftSerializer(source="came_from_draft", read_only=True) # todo prefetch this + obsoletes = RelatedRfcSerializer(many=True, read_only=True) + obsoleted_by = ReverseRelatedRfcSerializer(many=True, read_only=True) + updates = RelatedRfcSerializer(many=True, read_only=True) + updated_by = ReverseRelatedRfcSerializer(many=True, read_only=True) + is_also = serializers.ListField(child=serializers.CharField(), read_only=True) + see_also = serializers.ListField(child=serializers.CharField(), read_only=True) + formats = fields.MultipleChoiceField(choices=RFC_FORMATS) + keywords = serializers.ListField(child=serializers.CharField(), read_only=True) + errata = serializers.ListField(child=serializers.CharField(), read_only=True) class Meta: model = Document @@ -140,9 +162,17 @@ class Meta: "area", "stream", "identifiers", + "obsoletes", "obsoleted_by", + "updates", "updated_by", + "is_also", + "see_also", + "draft", "abstract", + "formats", + "keywords", + "errata", ] @extend_schema_field(DocIdentifierSerializer(many=True)) From f88e9f959eb20bc6fc5aff654cd6a16a78d3cd84 Mon Sep 17 00:00:00 2001 From: Matthew Holloway Date: Fri, 21 Feb 2025 15:11:52 +1300 Subject: [PATCH 39/64] chore: adding postscript (ps) to rfc meta serializer (#8560) --- ietf/doc/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py index 355022a15e..5e52a76d1a 100644 --- a/ietf/doc/serializers.py +++ b/ietf/doc/serializers.py @@ -128,7 +128,7 @@ class ReverseRelatedRfcSerializer(serializers.Serializer): class RfcMetadataSerializer(serializers.ModelSerializer): """Serialize metadata of an RFC""" - RFC_FORMATS = ("xml", "txt", "html", "htmlized", "pdf") + RFC_FORMATS = ("xml", "txt", "html", "htmlized", "pdf", "ps") number = serializers.IntegerField(source="rfc_number") published = serializers.DateField() From 76ebc1065cd078e960ae2d4240fb1cc6c98a3787 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 22 May 2025 12:47:06 -0300 Subject: [PATCH 40/64] fix: acknowledge not-issued in RfcStatusSlugT --- ietf/doc/serializers.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py index 5e52a76d1a..ff590e8700 100644 --- a/ietf/doc/serializers.py +++ b/ietf/doc/serializers.py @@ -36,7 +36,13 @@ class DocIdentifierSerializer(serializers.Serializer): # This should become "type RfcStatusSlugT ..." when we drop pre-py3.12 support # It should be "RfcStatusSlugT: TypeAlias ..." when we drop py3.9 support RfcStatusSlugT = Literal[ - "standard", "bcp", "informational", "experimental", "historic", "unknown" + "standard", + "bcp", + "informational", + "experimental", + "historic", + "unknown", + "not-issued", ] From 1f810efc853d9b17346fcf8f3a0261d42543a0d2 Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Fri, 23 May 2025 21:08:19 +1200 Subject: [PATCH 41/64] feat: Add API call to get references --- ietf/api/urls_rpc.py | 1 + ietf/api/views_rpc.py | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/ietf/api/urls_rpc.py b/ietf/api/urls_rpc.py index f0ff26caed..63b0b47042 100644 --- a/ietf/api/urls_rpc.py +++ b/ietf/api/urls_rpc.py @@ -7,6 +7,7 @@ urlpatterns = [ url(r"^doc/create_demo_draft/$", views_rpc.create_demo_draft), url(r"^doc/drafts/(?P[0-9]+)$", views_rpc.rpc_draft), + url(r"^doc/drafts/(?P[0-9]+)/references", views_rpc.rpc_draft_refs), url(r"^doc/drafts_by_names/", views_rpc.drafts_by_names), url(r"^doc/submitted_to_rpc/$", views_rpc.submitted_to_rpc), url(r"^doc/rfc/original_stream/$", views_rpc.rfc_original_stream), diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 65983ff213..935845b2e5 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -113,6 +113,33 @@ def rpc_draft(request, doc_id): ) +@csrf_exempt +#@requires_api_token("ietf.api.views_rpc") +def rpc_draft_refs(request, doc_id): + """Return norminative references""" + if request.method != "GET": + return HttpResponseNotAllowed(["GET"]) + + try: + d = Document.objects.get(pk=doc_id, type_id="draft") + except Document.DoesNotExist: + return HttpResponseNotFound() + + references = d.references() + norminative_references = [] + + for r in references: + if r.relationship.name == "normatively references": + norminative_references.append( + { + "id": r.target.id, + "name": r.target.name, + } + ) + + return JsonResponse({"references": norminative_references}) + + @csrf_exempt @requires_api_token("ietf.api.views_rpc") def drafts_by_names(request): From d37a982be1eca4837417361dc710e8015a24298e Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Fri, 23 May 2025 23:06:59 +1200 Subject: [PATCH 42/64] fix: Filter drafts --- ietf/api/views_rpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 935845b2e5..d58b2d2878 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -129,7 +129,7 @@ def rpc_draft_refs(request, doc_id): norminative_references = [] for r in references: - if r.relationship.name == "normatively references": + if r.relationship.name == "normatively references" and r.target.type_id == "draft": norminative_references.append( { "id": r.target.id, From 69ef75dd3f1db980ee008b9bcaaded9928c9f7e5 Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Wed, 28 May 2025 12:04:17 +1200 Subject: [PATCH 43/64] chore: Optimize the data query --- ietf/api/views_rpc.py | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index d58b2d2878..0cd70ac67e 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -16,7 +16,7 @@ from ietf.api.ietf_utils import requires_api_token from ietf.doc.factories import WgDraftFactory # DO NOT MERGE INTO MAIN -from ietf.doc.models import Document, DocHistory +from ietf.doc.models import Document, DocHistory, RelatedDocument from ietf.person.factories import PersonFactory # DO NOT MERGE INTO MAIN from ietf.person.models import Email, Person @@ -114,30 +114,22 @@ def rpc_draft(request, doc_id): @csrf_exempt -#@requires_api_token("ietf.api.views_rpc") +@requires_api_token("ietf.api.views_rpc") def rpc_draft_refs(request, doc_id): """Return norminative references""" if request.method != "GET": return HttpResponseNotAllowed(["GET"]) - try: - d = Document.objects.get(pk=doc_id, type_id="draft") - except Document.DoesNotExist: - return HttpResponseNotFound() - - references = d.references() - norminative_references = [] - - for r in references: - if r.relationship.name == "normatively references" and r.target.type_id == "draft": - norminative_references.append( - { - "id": r.target.id, - "name": r.target.name, - } - ) - - return JsonResponse({"references": norminative_references}) + return JsonResponse( + dict( + references=[ + dict(id=t[0], name=t[1]) + for t in RelatedDocument.objects.filter( + source_id=doc_id, target__type_id="draft", relationship_id="refnorm" + ).values_list("target_id", "target__name") + ] + ) + ) @csrf_exempt From b568be9f92d315cc021b9c5b316d6ef27b380584 Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Wed, 28 May 2025 12:05:07 +1200 Subject: [PATCH 44/64] test: Test for norminative references API call --- ietf/api/tests_views_rpc.py | 69 +++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 ietf/api/tests_views_rpc.py diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py new file mode 100644 index 0000000000..caa442f594 --- /dev/null +++ b/ietf/api/tests_views_rpc.py @@ -0,0 +1,69 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +# -*- coding: utf-8 -*- + +from django.test.utils import override_settings +from django.urls import reverse as urlreverse + +from ietf.doc.factories import IndividualDraftFactory +from ietf.doc.models import RelatedDocument +from ietf.utils.test_utils import TestCase, reload_db_objects + + +class RpcApiTests(TestCase): + @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + def test_api_refs(self): + # non-existent draft + url = urlreverse("ietf.api.views_rpc.rpc_draft_refs", kwargs={"doc_id": 999999}) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + jsondata = r.json() + refs = jsondata["references"] + self.assertEqual(refs, []) + + # draft without any norminative references + draft = IndividualDraftFactory() + draft = reload_db_objects(draft) + url = urlreverse( + "ietf.api.views_rpc.rpc_draft_refs", kwargs={"doc_id": draft.id} + ) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + jsondata = r.json() + refs = jsondata["references"] + self.assertEqual(refs, []) + + # draft without any norminative references but with an informative reference + draft_foo = IndividualDraftFactory() + draft_foo = reload_db_objects(draft_foo) + RelatedDocument.objects.create( + source=draft, target=draft_foo, relationship_id="refinfo" + ) + url = urlreverse( + "ietf.api.views_rpc.rpc_draft_refs", kwargs={"doc_id": draft.id} + ) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + jsondata = r.json() + refs = jsondata["references"] + self.assertEqual(refs, []) + + # draft with a norminative reference + draft_bar = IndividualDraftFactory() + draft_bar = reload_db_objects(draft_bar) + RelatedDocument.objects.create( + source=draft, target=draft_bar, relationship_id="refnorm" + ) + url = urlreverse( + "ietf.api.views_rpc.rpc_draft_refs", kwargs={"doc_id": draft.id} + ) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + jsondata = r.json() + refs = jsondata["references"] + self.assertEqual(len(refs), 1) + self.assertEqual(refs[0]["id"], draft_bar.id) + self.assertEqual(refs[0]["name"], draft_bar.name) From 28c85ffde571fa6b5d92c1668b8294795ae220de Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Wed, 28 May 2025 12:09:34 +1200 Subject: [PATCH 45/64] chore: Fix typos in tests --- ietf/api/tests_views_rpc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index caa442f594..8d7a14a425 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -21,7 +21,7 @@ def test_api_refs(self): refs = jsondata["references"] self.assertEqual(refs, []) - # draft without any norminative references + # draft without any nominative references draft = IndividualDraftFactory() draft = reload_db_objects(draft) url = urlreverse( @@ -34,7 +34,7 @@ def test_api_refs(self): refs = jsondata["references"] self.assertEqual(refs, []) - # draft without any norminative references but with an informative reference + # draft without any nominative references but with an informative reference draft_foo = IndividualDraftFactory() draft_foo = reload_db_objects(draft_foo) RelatedDocument.objects.create( @@ -50,7 +50,7 @@ def test_api_refs(self): refs = jsondata["references"] self.assertEqual(refs, []) - # draft with a norminative reference + # draft with a nominative reference draft_bar = IndividualDraftFactory() draft_bar = reload_db_objects(draft_bar) RelatedDocument.objects.create( From 95aecccfcaa9aac4aaa4d2d0ea0ae81de26cbc56 Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Thu, 29 May 2025 08:50:59 +1200 Subject: [PATCH 46/64] chore: Fix typos --- ietf/api/tests_views_rpc.py | 6 +++--- ietf/api/views_rpc.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py index 8d7a14a425..37e4416b67 100644 --- a/ietf/api/tests_views_rpc.py +++ b/ietf/api/tests_views_rpc.py @@ -21,7 +21,7 @@ def test_api_refs(self): refs = jsondata["references"] self.assertEqual(refs, []) - # draft without any nominative references + # draft without any normative references draft = IndividualDraftFactory() draft = reload_db_objects(draft) url = urlreverse( @@ -34,7 +34,7 @@ def test_api_refs(self): refs = jsondata["references"] self.assertEqual(refs, []) - # draft without any nominative references but with an informative reference + # draft without any normative references but with an informative reference draft_foo = IndividualDraftFactory() draft_foo = reload_db_objects(draft_foo) RelatedDocument.objects.create( @@ -50,7 +50,7 @@ def test_api_refs(self): refs = jsondata["references"] self.assertEqual(refs, []) - # draft with a nominative reference + # draft with a normative reference draft_bar = IndividualDraftFactory() draft_bar = reload_db_objects(draft_bar) RelatedDocument.objects.create( diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 0cd70ac67e..cfc17b437c 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -116,7 +116,7 @@ def rpc_draft(request, doc_id): @csrf_exempt @requires_api_token("ietf.api.views_rpc") def rpc_draft_refs(request, doc_id): - """Return norminative references""" + """Return normative references""" if request.method != "GET": return HttpResponseNotAllowed(["GET"]) From 9f718a3aefdcef0de905ba127513a0bf7260feff Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Fri, 30 May 2025 03:14:02 +1200 Subject: [PATCH 47/64] refactor: Separate demo logic (#8937) * refactor: Separate demo logic * chore: Skip tests --- ietf/api/urls_rpc.py | 14 +++++++-- ietf/api/views_rpc.py | 51 -------------------------------- ietf/api/views_rpc_demo.py | 60 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 54 deletions(-) create mode 100644 ietf/api/views_rpc_demo.py diff --git a/ietf/api/urls_rpc.py b/ietf/api/urls_rpc.py index 63b0b47042..aef075cb47 100644 --- a/ietf/api/urls_rpc.py +++ b/ietf/api/urls_rpc.py @@ -1,11 +1,11 @@ # Copyright The IETF Trust 2023-2024, All Rights Reserved -from ietf.api import views_rpc +from django.conf import settings +from ietf.api import views_rpc, views_rpc_demo from ietf.utils.urls import url urlpatterns = [ - url(r"^doc/create_demo_draft/$", views_rpc.create_demo_draft), url(r"^doc/drafts/(?P[0-9]+)$", views_rpc.rpc_draft), url(r"^doc/drafts/(?P[0-9]+)/references", views_rpc.rpc_draft_refs), url(r"^doc/drafts_by_names/", views_rpc.drafts_by_names), @@ -13,9 +13,17 @@ url(r"^doc/rfc/original_stream/$", views_rpc.rfc_original_stream), url(r"^doc/rfc/authors/$", views_rpc.rfc_authors), url(r"^doc/draft/authors/$", views_rpc.draft_authors), - url(r"^person/create_demo_person/$", views_rpc.create_demo_person), url(r"^person/persons_by_email/$", views_rpc.persons_by_email), url(r"^person/(?P[0-9]+)$", views_rpc.rpc_person), url(r"^persons/$", views_rpc.rpc_persons), url(r"^subject/(?P[0-9]+)/person/$", views_rpc.rpc_subject_person), ] + +if settings.SERVER_MODE not in {"production", "test"}: + # for non production demos + urlpatterns.append( + url(r"^doc/create_demo_draft/$", views_rpc_demo.create_demo_draft) + ) + urlpatterns.append( + url(r"^person/create_demo_person/$", views_rpc_demo.create_demo_person) + ) diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index cfc17b437c..81eee78f1f 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -4,7 +4,6 @@ from django.db.models import OuterRef, Subquery, Q from django.http import ( - HttpResponse, HttpResponseBadRequest, JsonResponse, HttpResponseNotAllowed, @@ -15,9 +14,7 @@ from django.contrib.auth.models import User from ietf.api.ietf_utils import requires_api_token -from ietf.doc.factories import WgDraftFactory # DO NOT MERGE INTO MAIN from ietf.doc.models import Document, DocHistory, RelatedDocument -from ietf.person.factories import PersonFactory # DO NOT MERGE INTO MAIN from ietf.person.models import Email, Person @@ -300,51 +297,3 @@ def draft_authors(request): item["authors"].append(item_author) response.append(item) return JsonResponse(response, safe=False) - - -@csrf_exempt -@requires_api_token("ietf.api.views_rpc") -def create_demo_person(request): - """Helper for creating rpc demo objects - SHOULD NOT MAKE IT INTO PRODUCTION""" - if request.method != "POST": - return HttpResponseNotAllowed(["POST"]) - - request_params = json.loads(request.body) - name = request_params["name"] - person = Person.objects.filter(name=name).first() or PersonFactory(name=name) - return JsonResponse({"user_id": person.user.pk, "person_pk": person.pk}) - - -@csrf_exempt -@requires_api_token("ietf.api.views_rpc") -def create_demo_draft(request): - """Helper for creating rpc demo objects - SHOULD NOT MAKE IT INTO PRODUCTION""" - if request.method != "POST": - return HttpResponseNotAllowed(["POST"]) - - request_params = json.loads(request.body) - name = request_params.get("name") - rev = request_params.get("rev") - states = request_params.get("states") - stream_id = request_params.get("stream_id", "ietf") - doc = None - if not name: - return HttpResponse(status=400, content="Name is required") - doc = Document.objects.filter(name=name).first() - if not doc: - kwargs = {"name": name, "stream_id": stream_id} - if states: - kwargs["states"] = states - if rev: - kwargs["rev"] = rev - doc = WgDraftFactory( - **kwargs - ) # Yes, things may be a little strange if the stream isn't IETF, but until we nned something different... - event_type = "iesg_approved" if stream_id == "ietf" else "requested_publication" - if not doc.docevent_set.filter( - type=event_type - ).exists(): # Not using get_or_create here on purpose - these are wobbly facades we're creating - doc.docevent_set.create( - type=event_type, by_id=1, desc="Sent off to the RPC" - ) - return JsonResponse({"doc_id": doc.pk, "name": doc.name}) diff --git a/ietf/api/views_rpc_demo.py b/ietf/api/views_rpc_demo.py new file mode 100644 index 0000000000..51dff32bf0 --- /dev/null +++ b/ietf/api/views_rpc_demo.py @@ -0,0 +1,60 @@ +# Copyright The IETF Trust 2023-2024, All Rights Reserved + +import json + +from django.views.decorators.csrf import csrf_exempt +from django.http import HttpResponse, HttpResponseNotAllowed, JsonResponse + +from ietf.api.ietf_utils import requires_api_token +from ietf.doc.factories import WgDraftFactory +from ietf.doc.models import Document +from ietf.person.factories import PersonFactory +from ietf.person.models import Person + + +@csrf_exempt +@requires_api_token("ietf.api.views_rpc") +def create_demo_person(request): + """Helper for creating rpc demo objects - SHOULD NOT MAKE IT INTO PRODUCTION""" + if request.method != "POST": + return HttpResponseNotAllowed(["POST"]) + + request_params = json.loads(request.body) + name = request_params["name"] + person = Person.objects.filter(name=name).first() or PersonFactory(name=name) + return JsonResponse({"user_id": person.user.pk, "person_pk": person.pk}) + + +@csrf_exempt +@requires_api_token("ietf.api.views_rpc") +def create_demo_draft(request): + """Helper for creating rpc demo objects - SHOULD NOT MAKE IT INTO PRODUCTION""" + if request.method != "POST": + return HttpResponseNotAllowed(["POST"]) + + request_params = json.loads(request.body) + name = request_params.get("name") + rev = request_params.get("rev") + states = request_params.get("states") + stream_id = request_params.get("stream_id", "ietf") + doc = None + if not name: + return HttpResponse(status=400, content="Name is required") + doc = Document.objects.filter(name=name).first() + if not doc: + kwargs = {"name": name, "stream_id": stream_id} + if states: + kwargs["states"] = states + if rev: + kwargs["rev"] = rev + doc = WgDraftFactory( + **kwargs + ) # Yes, things may be a little strange if the stream isn't IETF, but until we nned something different... + event_type = "iesg_approved" if stream_id == "ietf" else "requested_publication" + if not doc.docevent_set.filter( + type=event_type + ).exists(): # Not using get_or_create here on purpose - these are wobbly facades we're creating + doc.docevent_set.create( + type=event_type, by_id=1, desc="Sent off to the RPC" + ) + return JsonResponse({"doc_id": doc.pk, "name": doc.name}) From 01aa99d07d046526557bdbac1dd0ec9a3d96551c Mon Sep 17 00:00:00 2001 From: Robert Sparks Date: Thu, 29 May 2025 14:42:52 -0500 Subject: [PATCH 48/64] fix: trailing slashes for all rpc api endpoints (#8940) --- ietf/api/urls_rpc.py | 8 ++++---- rpcapi.yaml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ietf/api/urls_rpc.py b/ietf/api/urls_rpc.py index aef075cb47..cfdb279307 100644 --- a/ietf/api/urls_rpc.py +++ b/ietf/api/urls_rpc.py @@ -6,15 +6,15 @@ from ietf.utils.urls import url urlpatterns = [ - url(r"^doc/drafts/(?P[0-9]+)$", views_rpc.rpc_draft), - url(r"^doc/drafts/(?P[0-9]+)/references", views_rpc.rpc_draft_refs), - url(r"^doc/drafts_by_names/", views_rpc.drafts_by_names), + url(r"^doc/drafts/(?P[0-9]+)/$", views_rpc.rpc_draft), + url(r"^doc/drafts/(?P[0-9]+)/references/$", views_rpc.rpc_draft_refs), + url(r"^doc/drafts_by_names/$", views_rpc.drafts_by_names), url(r"^doc/submitted_to_rpc/$", views_rpc.submitted_to_rpc), url(r"^doc/rfc/original_stream/$", views_rpc.rfc_original_stream), url(r"^doc/rfc/authors/$", views_rpc.rfc_authors), url(r"^doc/draft/authors/$", views_rpc.draft_authors), url(r"^person/persons_by_email/$", views_rpc.persons_by_email), - url(r"^person/(?P[0-9]+)$", views_rpc.rpc_person), + url(r"^person/(?P[0-9]+)/$", views_rpc.rpc_person), url(r"^persons/$", views_rpc.rpc_persons), url(r"^subject/(?P[0-9]+)/person/$", views_rpc.rpc_subject_person), ] diff --git a/rpcapi.yaml b/rpcapi.yaml index f0c90f173d..c51f76d79b 100644 --- a/rpcapi.yaml +++ b/rpcapi.yaml @@ -6,7 +6,7 @@ info: servers: - url: 'http://localhost:8000/api/rpc' paths: - /person/{personId}: + /person/{personId}/: get: operationId: get_person_by_id summary: Find person by ID @@ -151,7 +151,7 @@ paths: name: type: string - /doc/submitted_to_rpc: + /doc/submitted_to_rpc/: get: operationId: submitted_to_rpc summary: List documents ready to enter the RFC Editor Queue @@ -164,7 +164,7 @@ paths: schema: $ref: '#/components/schemas/SubmittedToQueue' - /doc/drafts/{docId}: + /doc/drafts/{docId}/: get: operationId: get_draft_by_id summary: Get a draft @@ -308,7 +308,7 @@ paths: schema: $ref: '#/components/schemas/OriginalStream' - /subject/{subjectId}/person: + /subject/{subjectId}/person/: get: operationId: get_subject_person_by_id summary: Find person for OIDC subject by ID From f3b860092183039f4ed0a86d89f20318babc1c61 Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Fri, 30 May 2025 16:01:39 +1200 Subject: [PATCH 49/64] chore: Add RPC references API call to OpenAPI spec (#8941) * chore: Remove line noise * chore: Add RPC references API call to OpenAPI spec * chore: Update rpcapi.yaml Co-authored-by: Robert Sparks --------- Co-authored-by: Robert Sparks --- rpcapi.yaml | 42 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/rpcapi.yaml b/rpcapi.yaml index c51f76d79b..8bd79141b6 100644 --- a/rpcapi.yaml +++ b/rpcapi.yaml @@ -86,7 +86,6 @@ paths: additionalProperties: type: string - /person/create_demo_person/: post: operationId: create_demo_person @@ -163,7 +162,7 @@ paths: application/json: schema: $ref: '#/components/schemas/SubmittedToQueue' - + /doc/drafts/{docId}/: get: operationId: get_draft_by_id @@ -186,6 +185,26 @@ paths: '404': description: Not found + /doc/drafts/{docId}/references/: + get: + operationId: get_draft_references + summary: Get normative references to Internet-Drafts + description: Returns the id and name of each normatively referenced Internet-Draft for the given docId + parameters: + - name: docId + in: path + description: ID of draft + required: true + schema: + type: integer + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/References' + /doc/drafts_by_names/: post: operationId: get_drafts_by_names @@ -306,7 +325,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/OriginalStream' + $ref: '#/components/schemas/OriginalStream' /subject/{subjectId}/person/: get: @@ -369,13 +388,13 @@ components: submitted: type: string format: date-time - + ErrorResponse: type: object properties: error: type: string - + Draft: type: object properties: @@ -408,6 +427,19 @@ components: intended_std_level: type: string + References: + type: object + properties: + references: + type: array + items: + type: object + properties: + id: + type: integer + name: + type: string + OriginalStream: type: object properties: From e266247718672409a233e2de9bb038d711d7359b Mon Sep 17 00:00:00 2001 From: Kesara Rathnayake Date: Sat, 31 May 2025 02:27:13 +1200 Subject: [PATCH 50/64] fix: Fix OpenAPI spec errors (#8943) * chore: Remove more line noise * fix: Fix OpenAPI spec errors --- rpcapi.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/rpcapi.yaml b/rpcapi.yaml index 8bd79141b6..0e3ffbafcf 100644 --- a/rpcapi.yaml +++ b/rpcapi.yaml @@ -226,7 +226,7 @@ paths: schema: type: object additionalProperties: - $ref:'#/components/schemas/Draft' + $ref: '#/components/schemas/Draft' /doc/rfc/authors/: post: @@ -379,13 +379,13 @@ components: items: type: object properties: - name: + name: type: string - id: + id: type: integer - stream: + stream: type: string - submitted: + submitted: type: string format: date-time @@ -397,7 +397,7 @@ components: Draft: type: object - properties: + properties: id: type: integer name: @@ -444,7 +444,7 @@ components: type: object properties: original_stream: - type: list + type: array items: properties: rfc_number: @@ -452,7 +452,7 @@ components: stream: type: string - securitySchemes: + securitySchemes: ApiKeyAuth: type: apiKey in: header From daf7b6220f6b9e149fc30f8ba33167cec64db2f5 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 17 Jun 2025 10:18:06 -0300 Subject: [PATCH 51/64] feat: include picture URL in rpc_person API (#9009) --- ietf/api/views_rpc.py | 1 + rpcapi.yaml | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index 81eee78f1f..f80374a662 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -26,6 +26,7 @@ def rpc_person(request, person_id): { "id": person.id, "plain_name": person.plain_name(), + "picture": person.cdn_photo_url() or None, } ) diff --git a/rpcapi.yaml b/rpcapi.yaml index 0e3ffbafcf..ef2e5559a0 100644 --- a/rpcapi.yaml +++ b/rpcapi.yaml @@ -370,6 +370,11 @@ components: plain_name: type: string example: John Doe + picture: + type: string + example: https://cdn.example.com/avatars/some-photo.png + format: uri + nullable: true SubmittedToQueue: type: object From 38fda6362d0e1dae06f0e5945fcc25d0e44f80b2 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 26 Jun 2025 22:45:13 -0300 Subject: [PATCH 52/64] feat: person search endpoint (#9062) * feat: person search endpoint * refactor: address review comments * improved naming of operation/components in API schema * reused Person schema component * added serializers_rpc.py * chore: RpcPersonSerializer -> PersonSerializer Better matches the hand-written schema. * fix: search for entire term, not word-by-word * fix: only look at name/plain in search Including ascii / ascii_short might be useful eventually, but since we only show plain_name in the response it can cause confusing results. By the same reasoning we could remove email__address as well, but that's useful and I expect we'll include email addresses in our response soon anyway. --- ietf/api/serializers_rpc.py | 13 +++++++++ ietf/api/urls_rpc.py | 1 + ietf/api/views_rpc.py | 47 ++++++++++++++++++++++++++++++- ietf/person/models.py | 2 +- rpcapi.yaml | 55 +++++++++++++++++++++++++++++++++++++ 5 files changed, 116 insertions(+), 2 deletions(-) create mode 100644 ietf/api/serializers_rpc.py diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py new file mode 100644 index 0000000000..2793690b06 --- /dev/null +++ b/ietf/api/serializers_rpc.py @@ -0,0 +1,13 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +from rest_framework import serializers + +from ietf.person.models import Person + + +class PersonSerializer(serializers.ModelSerializer): + picture = serializers.URLField(source="cdn_photo_url", read_only=True) + + class Meta: + model = Person + fields = ["id", "plain_name", "picture"] + read_only_fields = ["id", "plain_name", "picture"] diff --git a/ietf/api/urls_rpc.py b/ietf/api/urls_rpc.py index cfdb279307..b69755f9dc 100644 --- a/ietf/api/urls_rpc.py +++ b/ietf/api/urls_rpc.py @@ -16,6 +16,7 @@ url(r"^person/persons_by_email/$", views_rpc.persons_by_email), url(r"^person/(?P[0-9]+)/$", views_rpc.rpc_person), url(r"^persons/$", views_rpc.rpc_persons), + url(r"^persons/search/", views_rpc.RpcPersonSearch.as_view()), url(r"^subject/(?P[0-9]+)/person/$", views_rpc.rpc_subject_person), ] diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index f80374a662..b4ea0c240c 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2023-2024, All Rights Reserved +# Copyright The IETF Trust 2023-2025, All Rights Reserved import json @@ -12,8 +12,14 @@ from django.shortcuts import get_object_or_404 from django.views.decorators.csrf import csrf_exempt from django.contrib.auth.models import User +from drf_spectacular.utils import extend_schema_view, extend_schema +from rest_framework import generics +from rest_framework.fields import CharField +from rest_framework.filters import SearchFilter +from rest_framework.pagination import LimitOffsetPagination from ietf.api.ietf_utils import requires_api_token +from ietf.api.serializers_rpc import PersonSerializer from ietf.doc.models import Document, DocHistory, RelatedDocument from ietf.person.models import Email, Person @@ -63,6 +69,45 @@ def rpc_persons(request): return JsonResponse(response) +class RpcLimitOffsetPagination(LimitOffsetPagination): + default_limit = 10 + max_limit = 100 + + +class SingleTermSearchFilter(SearchFilter): + """SearchFilter backend that does not split terms + + The default SearchFilter treats comma or whitespace-separated terms as individual + search terms. This backend instead searches for the exact term. + """ + + def get_search_terms(self, request): + value = request.query_params.get(self.search_param, '') + field = CharField(trim_whitespace=False, allow_blank=True) + cleaned_value = field.run_validation(value) + return [cleaned_value] + + +@extend_schema_view( + get=extend_schema( + operation_id="search_person", + description="Get a list of persons, matching by partial name or email", + ), +) +class RpcPersonSearch(generics.ListAPIView): + # n.b. the OpenAPI schema for this can be generated by running + # ietf/manage.py spectacular --file spectacular.yaml + # and extracting / touching up the rpc_person_search_list operation + api_key_endpoint = "ietf.api.views_rpc" + queryset = Person.objects.all() + serializer_class = PersonSerializer + pagination_class = RpcLimitOffsetPagination + + # Searchable on all name-like fields or email addresses + filter_backends = [SingleTermSearchFilter] + search_fields = ["name", "plain", "email__address"] + + def _document_source_format(doc): submission = doc.submission() if submission is None: diff --git a/ietf/person/models.py b/ietf/person/models.py index f703cf0ad8..2be3c1cc82 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -88,7 +88,7 @@ def short(self): else: prefix, first, middle, last, suffix = self.ascii_parts() return (first and first[0]+"." or "")+(middle or "")+" "+last+(suffix and " "+suffix or "") - def plain_name(self): + def plain_name(self) -> str: if not hasattr(self, '_cached_plain_name'): if self.plain: self._cached_plain_name = self.plain diff --git a/rpcapi.yaml b/rpcapi.yaml index ef2e5559a0..6e6e316ca0 100644 --- a/rpcapi.yaml +++ b/rpcapi.yaml @@ -85,6 +85,37 @@ paths: type: object additionalProperties: type: string + + /persons/search/: + get: + operationId: search_person + description: Get a list of persons, matching by partial name or email + parameters: + - name: limit + required: false + in: query + description: Number of results to return per page. + schema: + type: integer + - name: offset + required: false + in: query + description: The initial index from which to return the results. + schema: + type: integer + - name: search + required: false + in: query + description: A search term. + schema: + type: string + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/PaginatedPersonList' /person/create_demo_person/: post: @@ -457,6 +488,30 @@ components: stream: type: string + PaginatedPersonList: + type: object + required: + - count + - results + properties: + count: + type: integer + example: 123 + next: + type: string + nullable: true + format: uri + example: https://api.example.org/accounts/?offset=400&limit=100 + previous: + type: string + nullable: true + format: uri + example: https://api.example.org/accounts/?offset=200&limit=100 + results: + type: array + items: + $ref: '#/components/schemas/Person' + securitySchemes: ApiKeyAuth: type: apiKey From 70f093dfbe9303177293954f9b8aabda61cb9f66 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Tue, 8 Jul 2025 15:54:22 -0300 Subject: [PATCH 53/64] refactor: reimplement purple API in django-rest-framework (#9097) * refactor: rpc_person -> PersonViewSet * refactor: rpc_subject_person -> SubjectPersonView * refactor: rpc_persons -> RpcPersonsView * refactor: move get_persons into PersonViewSet Changes the interface to return a list of Persons instead of a map from ID to name. * refactor: rpc_draft -> DraftViewSet * refactor: drafts_by_names -> DraftsByNameView * refactor: submitted_to_rpc -> DraftViewSet * refactor: rfc_original_stream -> RfcViewSet * refactor: rpc demo APIs -> viewset * refactor: get_draft_refs -> DraftViewSet * refactor: persons_by_email -> PersonViewSet * refactor: rfc_authors -> RfcViewSet * refactor: draft_authors -> DraftViewSet * refactor: avoid \x00 in regex validator Gets turned into a literal nul somewhere in the process of generating a schema and building a Python client for purple. This has the same effect but avoids the nul. * fix: missing arg on references() action * style: ruff, remove unused imports * style: ruff ruff * chore: remove rpcapi.yaml * refactor: move API to /api/purple Side effect is that the purple API client is named PurpleApi instead of RpcApi. * fix: get_draft_authors returns DraftWithAuthors * fix: distinguish CharField flavors * fix: no serializer validators for draft name/title This prevents at least one existing draft from being looked up. * fix: get_draft_authors works with str, not int * Revert "refactor: avoid \x00 in regex validator" This reverts commit 63f40cf2 * Revert "Revert "refactor: avoid \x00 in regex validator"" (#9111) This reverts commit d8656f470045c21542824d7b9b9be41bdcb8866d. --- ietf/api/serializers_rpc.py | 158 ++++++ ietf/api/urls.py | 2 +- ietf/api/urls_rpc.py | 43 +- ietf/api/views_rpc.py | 505 ++++++++--------- ietf/api/views_rpc_demo.py | 140 +++-- ...r_dochistory_title_alter_document_title.py | 41 ++ ietf/doc/models.py | 14 +- ietf/utils/validators.py | 5 +- rpcapi.yaml | 522 ------------------ 9 files changed, 548 insertions(+), 882 deletions(-) create mode 100644 ietf/doc/migrations/0026_alter_dochistory_title_alter_document_title.py delete mode 100644 rpcapi.yaml diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 2793690b06..dcf1aad864 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -1,6 +1,12 @@ # Copyright The IETF Trust 2025, All Rights Reserved +import datetime +from typing import Literal, Optional + +from drf_spectacular.types import OpenApiTypes +from drf_spectacular.utils import extend_schema_field from rest_framework import serializers +from ietf.doc.models import DocumentAuthor, Document, RelatedDocument from ietf.person.models import Person @@ -11,3 +17,155 @@ class Meta: model = Person fields = ["id", "plain_name", "picture"] read_only_fields = ["id", "plain_name", "picture"] + + +class EmailPersonSerializer(serializers.Serializer): + email = serializers.EmailField(source="address") + person_pk = serializers.IntegerField(source="person.pk") + name = serializers.CharField(source="person.name") + last_name = serializers.CharField(source="person.last_name") + initials = serializers.CharField(source="person.initials") + + +class LowerCaseEmailField(serializers.EmailField): + def to_representation(self, value): + return super().to_representation(value).lower() + + +class AuthorPersonSerializer(serializers.ModelSerializer): + person_pk = serializers.IntegerField(source="pk", read_only=True) + last_name = serializers.CharField() + initials = serializers.CharField() + email_addresses = serializers.ListField( + source="email_set.all", child=LowerCaseEmailField() + ) + + class Meta: + model = Person + fields = ["person_pk", "name", "last_name", "initials", "email_addresses"] + + +class RfcWithAuthorsSerializer(serializers.ModelSerializer): + authors = AuthorPersonSerializer(many=True) + + class Meta: + model = Document + fields = ["rfc_number", "authors"] + + +class DraftWithAuthorsSerializer(serializers.ModelSerializer): + draft_name = serializers.CharField(source="name") + authors = AuthorPersonSerializer(many=True) + + class Meta: + model = Document + fields = ["draft_name", "authors"] + + +class DocumentAuthorSerializer(serializers.ModelSerializer): + """Serializer for a Person in a response""" + + plain_name = serializers.SerializerMethodField() + + class Meta: + model = DocumentAuthor + fields = ["person", "plain_name"] + + def get_plain_name(self, document_author: DocumentAuthor) -> str: + return document_author.person.plain_name() + + +class FullDraftSerializer(serializers.ModelSerializer): + # Redefine these fields so they don't pick up the regex validator patterns. + # There seem to be some non-compliant drafts in the system! If this serializer + # is used for a writeable view, the validation will need to be added back. + name = serializers.CharField(max_length=255) + title = serializers.CharField(max_length=255) + + # Other fields we need to add / adjust + source_format = serializers.SerializerMethodField() + authors = DocumentAuthorSerializer(many=True, source="documentauthor_set") + shepherd = serializers.SerializerMethodField() + + class Meta: + model = Document + fields = [ + "id", + "name", + "rev", + "stream", + "title", + "pages", + "source_format", + "authors", + "shepherd", + "intended_std_level", + ] + + def get_source_format( + self, doc: Document + ) -> Literal["unknown", "xml-v2", "xml-v3", "txt"]: + submission = doc.submission() + if submission is None: + return "unknown" + if ".xml" in submission.file_types: + if submission.xml_version == "3": + return "xml-v3" + else: + return "xml-v2" + elif ".txt" in submission.file_types: + return "txt" + return "unknown" + + @extend_schema_field(OpenApiTypes.EMAIL) + def get_shepherd(self, doc: Document) -> str: + if doc.shepherd: + return doc.shepherd.formatted_ascii_email() + return "" + + +class DraftSerializer(FullDraftSerializer): + class Meta: + model = Document + fields = [ + "id", + "name", + "rev", + "stream", + "title", + "pages", + "source_format", + "authors", + ] + + +class SubmittedToQueueSerializer(FullDraftSerializer): + submitted = serializers.SerializerMethodField() + + class Meta: + model = Document + fields = [ + "id", + "name", + "stream", + "submitted", + ] + + def get_submitted(self, doc) -> Optional[datetime.datetime]: + event = doc.sent_to_rfc_editor_event() + return None if event is None else event.time + + +class OriginalStreamSerializer(serializers.ModelSerializer): + stream = serializers.CharField(read_only=True, source="orig_stream_id") + + class Meta: + model = Document + fields = ["rfc_number", "stream"] + + +class ReferenceSerializer(serializers.ModelSerializer): + class Meta: + model = Document + fields = ["id", "name"] + read_only_fields = ["id", "name"] diff --git a/ietf/api/urls.py b/ietf/api/urls.py index abf359c0b6..f75a1ceb67 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -36,6 +36,7 @@ url(r'^v2/person/person', api_views.ApiV2PersonExportView.as_view()), # --- DRF API --- # path("core/", include(core_router.urls)), + path("purple/", include("ietf.api.urls_rpc")), path("red/", include(red_router.urls)), path("schema/", SpectacularAPIView.as_view()), # @@ -97,7 +98,6 @@ url(r'^rfcdiff-latest-json/(?P[Rr][Ff][Cc] [0-9]+?)(\.txt|\.html)?/?$', api_views.rfcdiff_latest_json), # direct authentication url(r'^directauth/?$', api_views.directauth), - url(r'^rpc/', include('ietf.api.urls_rpc')), ] # Additional (standard) Tastypie endpoints diff --git a/ietf/api/urls_rpc.py b/ietf/api/urls_rpc.py index b69755f9dc..925a091cb9 100644 --- a/ietf/api/urls_rpc.py +++ b/ietf/api/urls_rpc.py @@ -1,30 +1,33 @@ -# Copyright The IETF Trust 2023-2024, All Rights Reserved +# Copyright The IETF Trust 2023-2025, All Rights Reserved + +from rest_framework import routers from django.conf import settings +from django.urls import include, path from ietf.api import views_rpc, views_rpc_demo from ietf.utils.urls import url +router = routers.DefaultRouter() +router.register(r"draft", views_rpc.DraftViewSet, basename="draft") +router.register(r"person", views_rpc.PersonViewSet) +router.register(r"rfc", views_rpc.RfcViewSet, basename="rfc") + +if settings.SERVER_MODE not in {"production", "test"}: + # for non production demos + router.register(r"demo", views_rpc_demo.DemoViewSet, basename="demo") + + urlpatterns = [ - url(r"^doc/drafts/(?P[0-9]+)/$", views_rpc.rpc_draft), - url(r"^doc/drafts/(?P[0-9]+)/references/$", views_rpc.rpc_draft_refs), - url(r"^doc/drafts_by_names/$", views_rpc.drafts_by_names), - url(r"^doc/submitted_to_rpc/$", views_rpc.submitted_to_rpc), - url(r"^doc/rfc/original_stream/$", views_rpc.rfc_original_stream), - url(r"^doc/rfc/authors/$", views_rpc.rfc_authors), - url(r"^doc/draft/authors/$", views_rpc.draft_authors), - url(r"^person/persons_by_email/$", views_rpc.persons_by_email), - url(r"^person/(?P[0-9]+)/$", views_rpc.rpc_person), - url(r"^persons/$", views_rpc.rpc_persons), + url(r"^doc/drafts_by_names/", views_rpc.DraftsByNamesView.as_view()), url(r"^persons/search/", views_rpc.RpcPersonSearch.as_view()), - url(r"^subject/(?P[0-9]+)/person/$", views_rpc.rpc_subject_person), + path(r"subject//person/", views_rpc.SubjectPersonView.as_view()), ] -if settings.SERVER_MODE not in {"production", "test"}: - # for non production demos - urlpatterns.append( - url(r"^doc/create_demo_draft/$", views_rpc_demo.create_demo_draft) - ) - urlpatterns.append( - url(r"^person/create_demo_person/$", views_rpc_demo.create_demo_person) - ) +# add routers at the end so individual routes can steal parts of their address +# space (specifically, ^person/ routes so far) +urlpatterns.extend( + [ + path("", include(router.urls)), + ] +) diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py index b4ea0c240c..f3ca254597 100644 --- a/ietf/api/views_rpc.py +++ b/ietf/api/views_rpc.py @@ -1,72 +1,108 @@ # Copyright The IETF Trust 2023-2025, All Rights Reserved -import json - -from django.db.models import OuterRef, Subquery, Q -from django.http import ( - HttpResponseBadRequest, - JsonResponse, - HttpResponseNotAllowed, - HttpResponseNotFound, -) -from django.shortcuts import get_object_or_404 -from django.views.decorators.csrf import csrf_exempt -from django.contrib.auth.models import User +from drf_spectacular.utils import OpenApiParameter +from rest_framework import serializers, viewsets, mixins +from rest_framework.decorators import action +from rest_framework.views import APIView +from rest_framework.response import Response + +from django.db.models import CharField as ModelCharField, OuterRef, Subquery, Q +from django.db.models.functions import Coalesce +from django.http import Http404 from drf_spectacular.utils import extend_schema_view, extend_schema from rest_framework import generics -from rest_framework.fields import CharField +from rest_framework.fields import CharField as DrfCharField from rest_framework.filters import SearchFilter from rest_framework.pagination import LimitOffsetPagination -from ietf.api.ietf_utils import requires_api_token -from ietf.api.serializers_rpc import PersonSerializer -from ietf.doc.models import Document, DocHistory, RelatedDocument +from ietf.api.serializers_rpc import ( + PersonSerializer, + FullDraftSerializer, + DraftSerializer, + SubmittedToQueueSerializer, + OriginalStreamSerializer, + ReferenceSerializer, + EmailPersonSerializer, + RfcWithAuthorsSerializer, + DraftWithAuthorsSerializer, +) +from ietf.doc.models import Document, DocHistory from ietf.person.models import Email, Person -@csrf_exempt -@requires_api_token("ietf.api.views_rpc") -def rpc_person(request, person_id): - person = get_object_or_404(Person, pk=person_id) - return JsonResponse( - { - "id": person.id, - "plain_name": person.plain_name(), - "picture": person.cdn_photo_url() or None, - } +@extend_schema_view( + retrieve=extend_schema( + operation_id="get_person_by_id", + summary="Find person by ID", + description="Returns a single person", + ), +) +class PersonViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet): + queryset = Person.objects.all() + serializer_class = PersonSerializer + api_key_endpoint = "ietf.api.views_rpc" + lookup_url_kwarg = "person_id" + + @extend_schema( + operation_id="get_persons", + summary="Get a batch of persons", + description="Returns a list of persons matching requested ids. Omits any that are missing.", + request=list[int], + responses=PersonSerializer(many=True), + ) + @action(detail=False, methods=["post"]) + def batch(self, request): + """Get a batch of rpc person names""" + pks = request.data + return Response( + self.get_serializer(Person.objects.filter(pk__in=pks), many=True).data + ) + + @extend_schema( + operation_id="persons_by_email", + summary="Get a batch of persons by email addresses", + description=( + "Returns a list of persons matching requested ids. " + "Omits any that are missing." + ), + request=list[str], + responses=EmailPersonSerializer(many=True), ) + @action(detail=False, methods=["post"], serializer_class=EmailPersonSerializer) + def batch_by_email(self, request): + emails = Email.objects.filter(address__in=request.data, person__isnull=False) + serializer = self.get_serializer(emails, many=True) + return Response(serializer.data) + +class SubjectPersonView(APIView): + api_key_endpoint = "ietf.api.views_rpc" -@csrf_exempt -@requires_api_token("ietf.api.views_rpc") -def rpc_subject_person(request, subject_id): - try: - user_id = int(subject_id) - except ValueError: - return JsonResponse({"error": "Invalid subject id"}, status=400) - try: - user = User.objects.get(pk=user_id) - except User.DoesNotExist: - return JsonResponse({"error": "Unknown subject"}, status=404) - if hasattr( - user, "person" - ): # test this way to avoid exception on reverse OneToOneField - return rpc_person(request, person_id=user.person.pk) - return JsonResponse({"error": "Subject has no person"}, status=404) - - -@csrf_exempt -@requires_api_token("ietf.api.views_rpc") -def rpc_persons(request): - """Get a batch of rpc person names""" - if request.method != "POST": - return HttpResponseNotAllowed(["POST"]) - - pks = json.loads(request.body) - response = dict() - for p in Person.objects.filter(pk__in=pks): - response[str(p.pk)] = p.plain_name() - return JsonResponse(response) + @extend_schema( + operation_id="get_subject_person_by_id", + summary="Find person for OIDC subject by ID", + description="Returns a single person", + responses=PersonSerializer, + parameters=[ + OpenApiParameter( + name="subject_id", + type=str, + description="subject ID of person to return", + location="path", + ), + ], + ) + def get(self, request, subject_id: str): + try: + user_id = int(subject_id) + except ValueError: + raise serializers.ValidationError( + {"subject_id": "This field must be an integer value."} + ) + person = Person.objects.filter(user__pk=user_id).first() + if person: + return Response(PersonSerializer(person).data) + raise Http404 class RpcLimitOffsetPagination(LimitOffsetPagination): @@ -76,14 +112,14 @@ class RpcLimitOffsetPagination(LimitOffsetPagination): class SingleTermSearchFilter(SearchFilter): """SearchFilter backend that does not split terms - + The default SearchFilter treats comma or whitespace-separated terms as individual search terms. This backend instead searches for the exact term. """ def get_search_terms(self, request): - value = request.query_params.get(self.search_param, '') - field = CharField(trim_whitespace=False, allow_blank=True) + value = request.query_params.get(self.search_param, "") + field = DrfCharField(trim_whitespace=False, allow_blank=True) cleaned_value = field.run_validation(value) return [cleaned_value] @@ -108,238 +144,139 @@ class RpcPersonSearch(generics.ListAPIView): search_fields = ["name", "plain", "email__address"] -def _document_source_format(doc): - submission = doc.submission() - if submission is None: - return "unknown" - if ".xml" in submission.file_types: - if submission.xml_version == "3": - return "xml-v3" - else: - return "xml-v2" - elif ".txt" in submission.file_types: - return "txt" - return "unknown" - - -@csrf_exempt -@requires_api_token("ietf.api.views_rpc") -def rpc_draft(request, doc_id): - if request.method != "GET": - return HttpResponseNotAllowed(["GET"]) - - try: - d = Document.objects.get(pk=doc_id, type_id="draft") - except Document.DoesNotExist: - return HttpResponseNotFound() - return JsonResponse( - { - "id": d.pk, - "name": d.name, - "rev": d.rev, - "stream": d.stream.slug, - "title": d.title, - "pages": d.pages, - "source_format": _document_source_format(d), - "authors": [ - { - "id": p.pk, - "plain_name": p.person.plain_name(), - } - for p in d.documentauthor_set.all() +@extend_schema_view( + retrieve=extend_schema( + operation_id="get_draft_by_id", + summary="Get a draft", + description="Returns the draft for the requested ID", + ), + submitted_to_rpc=extend_schema( + operation_id="submitted_to_rpc", + summary="List documents ready to enter the RFC Editor Queue", + description="List documents ready to enter the RFC Editor Queue", + responses=SubmittedToQueueSerializer(many=True), + ), +) +class DraftViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet): + queryset = Document.objects.filter(type_id="draft") + serializer_class = FullDraftSerializer + api_key_endpoint = "ietf.api.views_rpc" + lookup_url_kwarg = "doc_id" + + @action(detail=False, serializer_class=SubmittedToQueueSerializer) + def submitted_to_rpc(self, request): + """Return documents in datatracker that have been submitted to the RPC but are not yet in the queue + + Those queries overreturn - there may be things, particularly not from the IETF stream that are already in the queue. + """ + ietf_docs = Q(states__type_id="draft-iesg", states__slug__in=["ann"]) + irtf_iab_ise_docs = Q( + states__type_id__in=[ + "draft-stream-iab", + "draft-stream-irtf", + "draft-stream-ise", ], - "shepherd": d.shepherd.formatted_ascii_email() if d.shepherd else "", - "intended_std_level": ( - d.intended_std_level.slug if d.intended_std_level else "" - ), - } + states__slug__in=["rfc-edit"], + ) + # TODO: Need a way to talk about editorial stream docs + docs = ( + self.get_queryset() + .filter(type_id="draft") + .filter(ietf_docs | irtf_iab_ise_docs) + ) + serializer = self.get_serializer(docs, many=True) + return Response(serializer.data) + + @extend_schema( + operation_id="get_draft_references", + summary="Get normative references to I-Ds", + description=( + "Returns the id and name of each normatively " + "referenced Internet-Draft for the given docId" + ), + responses=ReferenceSerializer(many=True), ) - - -@csrf_exempt -@requires_api_token("ietf.api.views_rpc") -def rpc_draft_refs(request, doc_id): - """Return normative references""" - if request.method != "GET": - return HttpResponseNotAllowed(["GET"]) - - return JsonResponse( - dict( - references=[ - dict(id=t[0], name=t[1]) - for t in RelatedDocument.objects.filter( - source_id=doc_id, target__type_id="draft", relationship_id="refnorm" - ).values_list("target_id", "target__name") - ] + @action(detail=True, serializer_class=ReferenceSerializer) + def references(self, request, doc_id=None): + doc = self.get_object() + serializer = self.get_serializer( + [ + reference + for reference in doc.related_that_doc("refnorm") + if reference.type_id == "draft" + ], + many=True, ) + return Response(serializer.data) + + @extend_schema( + operation_id="get_draft_authors", + summary="Gather authors of the drafts with the given names", + description="returns a list mapping draft names to objects describing authors", + request=list[str], + responses=DraftWithAuthorsSerializer(many=True), ) + @action(detail=False, methods=["post"], serializer_class=DraftWithAuthorsSerializer) + def authors(self, request): + drafts = self.get_queryset().filter(name__in=request.data) + serializer = self.get_serializer(drafts, many=True) + return Response(serializer.data) -@csrf_exempt -@requires_api_token("ietf.api.views_rpc") -def drafts_by_names(request): - if request.method != "POST": - return HttpResponseNotAllowed(["POST"]) - try: - names = json.loads(request.body) - except json.JSONDecodeError: - return HttpResponseBadRequest() - docs = Document.objects.filter(type_id="draft", name__in=names) - response = dict() - for doc in docs: - response[doc.name] = { - "id": doc.pk, - "name": doc.name, - "rev": doc.rev, - "stream": doc.stream.slug if doc.stream else "none", - "title": doc.title, - "pages": doc.pages, - "source_format": _document_source_format(doc), - "authors": [ - { - "id": p.pk, - "plain_name": p.person.plain_name(), - } - for p in doc.documentauthor_set.all() - ], - } - return JsonResponse(response) +@extend_schema_view( + rfc_original_stream=extend_schema( + operation_id="get_rfc_original_streams", + summary="Get the streams RFCs were originally published into", + description="returns a list of dicts associating an RFC with its originally published stream", + responses=OriginalStreamSerializer(many=True), + ) +) +class RfcViewSet(viewsets.GenericViewSet): + queryset = Document.objects.filter(type_id="rfc") + api_key_endpoint = "ietf.api.views_rpc" + + @action(detail=False, serializer_class=OriginalStreamSerializer) + def rfc_original_stream(self, request): + rfcs = self.get_queryset().annotate( + orig_stream_id=Coalesce( + Subquery( + DocHistory.objects.filter(doc=OuterRef("pk")) + .exclude(stream__isnull=True) + .order_by("time") + .values_list("stream_id", flat=True)[:1] + ), + "stream_id", + output_field=ModelCharField(), + ), + ) + serializer = self.get_serializer(rfcs, many=True) + return Response(serializer.data) + + @extend_schema( + operation_id="get_rfc_authors", + summary="Gather authors of the RFCs with the given numbers", + description="returns a list mapping rfc numbers to objects describing authors", + request=list[int], + responses=RfcWithAuthorsSerializer(many=True), + ) + @action(detail=False, methods=["post"], serializer_class=RfcWithAuthorsSerializer) + def authors(self, request): + rfcs = self.get_queryset().filter(rfc_number__in=request.data) + serializer = self.get_serializer(rfcs, many=True) + return Response(serializer.data) -@csrf_exempt -@requires_api_token("ietf.api.views_rpc") -def submitted_to_rpc(request): - """Return documents in datatracker that have been submitted to the RPC but are not yet in the queue +class DraftsByNamesView(APIView): + api_key_endpoint = "ietf.api.views_rpc" - Those queries overreturn - there may be things, particularly not from the IETF stream that are already in the queue. - """ - ietf_docs = Q(states__type_id="draft-iesg", states__slug__in=["ann"]) - irtf_iab_ise_docs = Q( - states__type_id__in=[ - "draft-stream-iab", - "draft-stream-irtf", - "draft-stream-ise", - ], - states__slug__in=["rfc-edit"], - ) - # TODO: Need a way to talk about editorial stream docs - docs = Document.objects.filter(type_id="draft").filter( - ietf_docs | irtf_iab_ise_docs + @extend_schema( + operation_id="get_drafts_by_names", + summary="Get a batch of drafts by draft names", + description="returns a list of drafts with matching names", + request=list[str], + responses=DraftSerializer(many=True), ) - response = {"submitted_to_rpc": []} - for doc in docs: - response["submitted_to_rpc"].append( - { - "name": doc.name, - "id": doc.pk, - "stream": doc.stream_id, - "submitted": f"{doc.sent_to_rfc_editor_event().time.isoformat()}", - } - ) - return JsonResponse(response) - - -@csrf_exempt -@requires_api_token("ietf.api.views_rpc") -def rfc_original_stream(request): - """Return the stream that an rfc was first published into for all rfcs""" - rfcs = Document.objects.filter(type="rfc").annotate( - orig_stream_id=Subquery( - DocHistory.objects.filter(doc=OuterRef("pk")) - .exclude(stream__isnull=True) - .order_by("time") - .values_list("stream_id", flat=True)[:1] - ) - ) - response = {"original_stream": []} - for rfc in rfcs: - response["original_stream"].append( - { - "rfc_number": rfc.rfc_number, - "stream": ( - rfc.orig_stream_id - if rfc.orig_stream_id is not None - else rfc.stream_id - ), - } - ) - return JsonResponse(response) - - -@csrf_exempt -@requires_api_token("ietf.api.views_rpc") -def persons_by_email(request): - if request.method != "POST": - return HttpResponseNotAllowed(["POST"]) - try: - emails = json.loads(request.body) - except json.JSONDecodeError: - return HttpResponseBadRequest() - response = [] - for email in Email.objects.filter(address__in=emails).exclude(person__isnull=True): - response.append( - { - "email": email.address, - "person_pk": email.person.pk, - "name": email.person.name, - "last_name": email.person.last_name(), - "initials": email.person.initials(), - } - ) - return JsonResponse(response, safe=False) - - -@csrf_exempt -@requires_api_token("ietf.api.views_rpc") -def rfc_authors(request): - """Gather authors of the RFCs with the given numbers""" - if request.method != "POST": - return HttpResponseNotAllowed(["POST"]) - try: - rfc_numbers = json.loads(request.body) - except json.JSONDecodeError: - return HttpResponseBadRequest() - response = [] - for rfc in Document.objects.filter(type="rfc", rfc_number__in=rfc_numbers): - item = {"rfc_number": rfc.rfc_number, "authors": []} - for author in rfc.authors(): - item_author = dict() - item_author["person_pk"] = author.pk - item_author["name"] = author.name - item_author["last_name"] = author.last_name() - item_author["initials"] = author.initials() - item_author["email_addresses"] = [ - address.lower() - for address in author.email_set.values_list("address", flat=True) - ] - item["authors"].append(item_author) - response.append(item) - return JsonResponse(response, safe=False) - - -@csrf_exempt -@requires_api_token("ietf.api.views_rpc") -def draft_authors(request): - """Gather authors of the RFCs with the given numbers""" - if request.method != "POST": - return HttpResponseNotAllowed(["POST"]) - try: - draft_names = json.loads(request.body) - except json.JSONDecodeError: - return HttpResponseBadRequest() - response = [] - for draft in Document.objects.filter(type="draft", name__in=draft_names): - item = {"draft_name": draft.name, "authors": []} - for author in draft.authors(): - item_author = dict() - item_author["person_pk"] = author.pk - item_author["name"] = author.name - item_author["last_name"] = author.last_name() - item_author["initials"] = author.initials() - item_author["email_addresses"] = [ - address.lower() - for address in author.email_set.values_list("address", flat=True) - ] - item["authors"].append(item_author) - response.append(item) - return JsonResponse(response, safe=False) + def post(self, request): + names = request.data + docs = Document.objects.filter(type_id="draft", name__in=names) + return Response(DraftSerializer(docs, many=True).data) diff --git a/ietf/api/views_rpc_demo.py b/ietf/api/views_rpc_demo.py index 51dff32bf0..ad969b0832 100644 --- a/ietf/api/views_rpc_demo.py +++ b/ietf/api/views_rpc_demo.py @@ -1,60 +1,98 @@ -# Copyright The IETF Trust 2023-2024, All Rights Reserved +# Copyright The IETF Trust 2023-2025, All Rights Reserved -import json +from drf_spectacular.utils import extend_schema_view, extend_schema +from rest_framework import serializers, viewsets +from rest_framework.decorators import action +from rest_framework.response import Response -from django.views.decorators.csrf import csrf_exempt -from django.http import HttpResponse, HttpResponseNotAllowed, JsonResponse - -from ietf.api.ietf_utils import requires_api_token from ietf.doc.factories import WgDraftFactory from ietf.doc.models import Document from ietf.person.factories import PersonFactory from ietf.person.models import Person -@csrf_exempt -@requires_api_token("ietf.api.views_rpc") -def create_demo_person(request): - """Helper for creating rpc demo objects - SHOULD NOT MAKE IT INTO PRODUCTION""" - if request.method != "POST": - return HttpResponseNotAllowed(["POST"]) - - request_params = json.loads(request.body) - name = request_params["name"] - person = Person.objects.filter(name=name).first() or PersonFactory(name=name) - return JsonResponse({"user_id": person.user.pk, "person_pk": person.pk}) - - -@csrf_exempt -@requires_api_token("ietf.api.views_rpc") -def create_demo_draft(request): - """Helper for creating rpc demo objects - SHOULD NOT MAKE IT INTO PRODUCTION""" - if request.method != "POST": - return HttpResponseNotAllowed(["POST"]) - - request_params = json.loads(request.body) - name = request_params.get("name") - rev = request_params.get("rev") - states = request_params.get("states") - stream_id = request_params.get("stream_id", "ietf") - doc = None - if not name: - return HttpResponse(status=400, content="Name is required") - doc = Document.objects.filter(name=name).first() - if not doc: - kwargs = {"name": name, "stream_id": stream_id} - if states: - kwargs["states"] = states - if rev: - kwargs["rev"] = rev - doc = WgDraftFactory( - **kwargs - ) # Yes, things may be a little strange if the stream isn't IETF, but until we nned something different... - event_type = "iesg_approved" if stream_id == "ietf" else "requested_publication" - if not doc.docevent_set.filter( - type=event_type - ).exists(): # Not using get_or_create here on purpose - these are wobbly facades we're creating - doc.docevent_set.create( - type=event_type, by_id=1, desc="Sent off to the RPC" +class DemoPersonCreateSerializer(serializers.Serializer): + name = serializers.CharField(max_length=255) + + +class DemoPersonSerializer(serializers.ModelSerializer): + person_pk = serializers.IntegerField(source="pk") + + class Meta: + model = Person + fields = ["user_id", "person_pk"] + + +class DemoDraftCreateSerializer(serializers.Serializer): + name = serializers.CharField(max_length=255, required=True) + stream_id = serializers.CharField(default="ietf") + rev = serializers.CharField(default=None) + states = serializers.DictField(child=serializers.CharField(), default=None) + + +class DemoDraftSerializer(serializers.ModelSerializer): + doc_id = serializers.IntegerField(source="pk") + + class Meta: + model = Document + fields = ["doc_id", "name"] + + +@extend_schema_view( + create_demo_person=extend_schema( + operation_id="create_demo_person", + summary="Build a datatracker Person for RPC demo purposes", + description="returns a datatracker User id for a person created with the given name", + request=DemoPersonCreateSerializer, + responses=DemoPersonSerializer, + ), + create_demo_draft=extend_schema( + operation_id="create_demo_draft", + summary="Build a datatracker WG draft for RPC demo purposes", + description="returns a datatracker document id for a draft created with the provided name and states. " + "The arguments, if present, are passed directly to the WgDraftFactory", + request=DemoDraftCreateSerializer, + responses=DemoDraftSerializer, + ), +) +class DemoViewSet(viewsets.ViewSet): + """SHOULD NOT MAKE IT INTO PRODUCTION""" + + api_key_endpoint = "ietf.api.views_rpc" + + @action(detail=False, methods=["post"]) + def create_demo_person(self, request): + """Helper for creating rpc demo objects - SHOULD NOT MAKE IT INTO PRODUCTION""" + request_params = DemoPersonCreateSerializer(request.data) + name = request_params.data["name"] + person = Person.objects.filter(name=name).first() or PersonFactory(name=name) + return Response(DemoPersonSerializer(person).data) + + @action(detail=False, methods=["post"]) + def create_demo_draft(self, request): + """Helper for creating rpc demo objects - SHOULD NOT MAKE IT INTO PRODUCTION""" + request_params = DemoDraftCreateSerializer(request.data) + name = request_params.data["name"] + rev = request_params.data["rev"] + stream_id = request_params.data["stream_id"] + states = request_params.data["states"] + doc = Document.objects.filter(name=name).first() + if not doc: + kwargs = {"name": name, "stream_id": stream_id} + if states: + kwargs["states"] = states + if rev: + kwargs["rev"] = rev + doc = WgDraftFactory( + **kwargs + ) # Yes, things may be a little strange if the stream isn't IETF, but until we need something different... + event_type = ( + "iesg_approved" if stream_id == "ietf" else "requested_publication" ) - return JsonResponse({"doc_id": doc.pk, "name": doc.name}) + if not doc.docevent_set.filter( + type=event_type + ).exists(): # Not using get_or_create here on purpose - these are wobbly facades we're creating + doc.docevent_set.create( + type=event_type, by_id=1, desc="Sent off to the RPC" + ) + return Response(DemoDraftSerializer(doc).data) diff --git a/ietf/doc/migrations/0026_alter_dochistory_title_alter_document_title.py b/ietf/doc/migrations/0026_alter_dochistory_title_alter_document_title.py new file mode 100644 index 0000000000..5f5140a278 --- /dev/null +++ b/ietf/doc/migrations/0026_alter_dochistory_title_alter_document_title.py @@ -0,0 +1,41 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0025_storedobject_storedobject_unique_name_per_store"), + ] + + operations = [ + migrations.AlterField( + model_name="dochistory", + name="title", + field=models.CharField( + max_length=255, + validators=[ + django.core.validators.ProhibitNullCharactersValidator, + django.core.validators.RegexValidator( + message="Please enter a string without control characters.", + regex="^[^\x01-\x1f]*$", + ), + ], + ), + ), + migrations.AlterField( + model_name="document", + name="title", + field=models.CharField( + max_length=255, + validators=[ + django.core.validators.ProhibitNullCharactersValidator, + django.core.validators.RegexValidator( + message="Please enter a string without control characters.", + regex="^[^\x01-\x1f]*$", + ), + ], + ), + ), + ] diff --git a/ietf/doc/models.py b/ietf/doc/models.py index bdf250ebcb..5c5468fb49 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -20,7 +20,11 @@ from django.core import checks from django.core.files.base import File from django.core.cache import caches -from django.core.validators import URLValidator, RegexValidator +from django.core.validators import ( + URLValidator, + RegexValidator, + ProhibitNullCharactersValidator, +) from django.urls import reverse as urlreverse from django.contrib.contenttypes.models import ContentType from django.conf import settings @@ -107,7 +111,13 @@ class DocumentInfo(models.Model): time = models.DateTimeField(default=timezone.now) # should probably have auto_now=True type = ForeignKey(DocTypeName, blank=True, null=True) # Draft, Agenda, Minutes, Charter, Discuss, Guideline, Email, Review, Issue, Wiki, External ... - title = models.CharField(max_length=255, validators=[validate_no_control_chars, ]) + title = models.CharField( + max_length=255, + validators=[ + ProhibitNullCharactersValidator, + validate_no_control_chars, + ], + ) states = models.ManyToManyField(State, blank=True) # plain state (Active/Expired/...), IESG state, stream state tags = models.ManyToManyField(DocTagName, blank=True) # Revised ID Needed, ExternalParty, AD Followup, ... diff --git a/ietf/utils/validators.py b/ietf/utils/validators.py index 92a20f5a26..a99de72724 100644 --- a/ietf/utils/validators.py +++ b/ietf/utils/validators.py @@ -33,8 +33,9 @@ # Note that this is an instantiation of the regex validator, _not_ the # regex-string validator defined right below validate_no_control_chars = RegexValidator( - regex="^[^\x00-\x1f]*$", - message="Please enter a string without control characters." ) + regex="^[^\x01-\x1f]*$", + message="Please enter a string without control characters.", +) @deconstructible diff --git a/rpcapi.yaml b/rpcapi.yaml deleted file mode 100644 index 6e6e316ca0..0000000000 --- a/rpcapi.yaml +++ /dev/null @@ -1,522 +0,0 @@ -openapi: 3.0.3 -info: - title: Datatracker RPC API - description: Datatracker RPC API - version: 1.0.0 -servers: - - url: 'http://localhost:8000/api/rpc' -paths: - /person/{personId}/: - get: - operationId: get_person_by_id - summary: Find person by ID - description: Returns a single person - parameters: - - name: personId - in: path - description: ID of person to return - required: true - schema: - type: integer - responses: - '200': - description: OK - content: - application/json: - schema: - $ref: '#/components/schemas/Person' - '404': - description: Not found - - /person/persons_by_email/: - post: - operationId: persons_by_email - summary: Get a batch of persons by email addresses - description: returns a list of objects with the email address and related person information - requestBody: - required: true - content: - application/json: - schema: - type: array - items: - type: string - responses: - '200': - description: OK - content: - application/json: - schema: - type: array - items: - type: object - properties: - email: - type: string - person_pk: - type: integer - name: - type: string - last_name: - type: string - initials: - type: string - - - /persons/: - post: - operationId: get_persons - summary: Get a batch of persons - description: returns a dict of person pks to person names - requestBody: - required: true - content: - application/json: - schema: - type: array - items: - type: integer - responses: - '200': - description: OK - content: - application/json: - schema: - type: object - additionalProperties: - type: string - - /persons/search/: - get: - operationId: search_person - description: Get a list of persons, matching by partial name or email - parameters: - - name: limit - required: false - in: query - description: Number of results to return per page. - schema: - type: integer - - name: offset - required: false - in: query - description: The initial index from which to return the results. - schema: - type: integer - - name: search - required: false - in: query - description: A search term. - schema: - type: string - responses: - '200': - description: OK - content: - application/json: - schema: - $ref: '#/components/schemas/PaginatedPersonList' - - /person/create_demo_person/: - post: - operationId: create_demo_person - summary: Build a datatracker Person for RPC demo purposes - description: returns a datatracker User id for a person created with the given name - requestBody: - required: true - content: - application/json: - schema: - type: object - properties: - name: - type: string - responses: - '200': - description: OK - content: - application/json: - schema: - type: object - properties: - user_id: - type: integer - person_pk: - type: integer - - /doc/create_demo_draft/: - post: - operationId: create_demo_draft - summary: Build a datatracker WG draft for RPC demo purposes - description: returns a datatracker document id for a draft created with the provided name and states. The arguments, if present, are passed directly to the WgDraftFactory - requestBody: - required: true - content: - application/json: - schema: - type: object - properties: - name: - type: string - stream_id: - type: string - rev: - type: string - states: - type: array - items: - type: array - items: - type: string - responses: - '200': - description: OK - content: - application/json: - schema: - type: object - properties: - doc_id: - type: integer - name: - type: string - - /doc/submitted_to_rpc/: - get: - operationId: submitted_to_rpc - summary: List documents ready to enter the RFC Editor Queue - description: List documents ready to enter the RFC Editor Queue - responses: - '200': - description: OK - content: - application/json: - schema: - $ref: '#/components/schemas/SubmittedToQueue' - - /doc/drafts/{docId}/: - get: - operationId: get_draft_by_id - summary: Get a draft - description: Returns the draft for the requested ID - parameters: - - name: docId - in: path - description: ID of draft to retrieve - required: true - schema: - type: integer - responses: - '200': - description: OK - content: - application/json: - schema: - $ref: '#/components/schemas/Draft' - '404': - description: Not found - - /doc/drafts/{docId}/references/: - get: - operationId: get_draft_references - summary: Get normative references to Internet-Drafts - description: Returns the id and name of each normatively referenced Internet-Draft for the given docId - parameters: - - name: docId - in: path - description: ID of draft - required: true - schema: - type: integer - responses: - '200': - description: OK - content: - application/json: - schema: - $ref: '#/components/schemas/References' - - /doc/drafts_by_names/: - post: - operationId: get_drafts_by_names - summary: Get a batch of drafts by draft names - description: returns a dict of drafts with matching names - requestBody: - required: true - content: - application/json: - schema: - type: array - items: - type: string - responses: - '200': - description: OK - content: - application/json: - schema: - type: object - additionalProperties: - $ref: '#/components/schemas/Draft' - - /doc/rfc/authors/: - post: - operationId: get_rfc_authors - summary: Gather authors of the RFCs with the given numbers - description: returns a dict mapping rfc numbers to objects describing authors - requestBody: - required: true - content: - application/json: - schema: - type: array - items: - type: integer - responses: - '200': - description: OK - content: - application/json: - schema: - type: array - items: - type: object - properties: - rfc_number: - type: integer - authors: - type: array - items: - type: object - properties: - person_pk: - type: integer - name: - type: string - last_name: - type: string - initials: - type: string - email_addresses: - type: array - items: - type: string - - /doc/draft/authors/: - post: - operationId: get_draft_authors - summary: Gather authors of the drafts with the given names - description: returns a dict mapping draft names to objects describing authors - requestBody: - required: true - content: - application/json: - schema: - type: array - items: - type: string - responses: - '200': - description: OK - content: - application/json: - schema: - type: array - items: - type: object - properties: - draft_name: - type: string - authors: - type: array - items: - type: object - properties: - person_pk: - type: integer - name: - type: string - last_name: - type: string - initials: - type: string - email_addresses: - type: array - items: - type: string - - /doc/rfc/original_stream/: - get: - operationId: get_rfc_original_streams - summary: Get the streams rfcs were originally published into - description: returns a list of dicts associating an rfc with its originally published stream - responses: - '200': - description: OK - content: - application/json: - schema: - $ref: '#/components/schemas/OriginalStream' - - /subject/{subjectId}/person/: - get: - operationId: get_subject_person_by_id - summary: Find person for OIDC subject by ID - description: Returns a single person - parameters: - - name: subjectId - in: path - description: subject ID of person to return - required: true - schema: - type: string - responses: - '200': - description: OK - content: - application/json: - schema: - $ref: '#/components/schemas/Person' - '400': - description: No such subject or no person for this subject - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - '404': - description: No such subject or no person for this subject - content: - application/json: - schema: - $ref: '#/components/schemas/ErrorResponse' - -components: - schemas: - Person: - type: object - properties: - id: - type: integer - example: 1234 - plain_name: - type: string - example: John Doe - picture: - type: string - example: https://cdn.example.com/avatars/some-photo.png - format: uri - nullable: true - - SubmittedToQueue: - type: object - properties: - submitted_to_rpc: - type: array - items: - type: object - properties: - name: - type: string - id: - type: integer - stream: - type: string - submitted: - type: string - format: date-time - - ErrorResponse: - type: object - properties: - error: - type: string - - Draft: - type: object - properties: - id: - type: integer - name: - type: string - rev: - type: string - stream: - type: string - title: - type: string - pages: - type: integer - source_format: - type: string - enum: - - unknown - - txt - - xml-v2 - - xml-v3 - authors: - type: array - items: - $ref: '#/components/schemas/Person' - shepherd: - type: string - format: email - intended_std_level: - type: string - - References: - type: object - properties: - references: - type: array - items: - type: object - properties: - id: - type: integer - name: - type: string - - OriginalStream: - type: object - properties: - original_stream: - type: array - items: - properties: - rfc_number: - type: integer - stream: - type: string - - PaginatedPersonList: - type: object - required: - - count - - results - properties: - count: - type: integer - example: 123 - next: - type: string - nullable: true - format: uri - example: https://api.example.org/accounts/?offset=400&limit=100 - previous: - type: string - nullable: true - format: uri - example: https://api.example.org/accounts/?offset=200&limit=100 - results: - type: array - items: - $ref: '#/components/schemas/Person' - - securitySchemes: - ApiKeyAuth: - type: apiKey - in: header - name: X-API-KEY - -security: - - ApiKeyAuth: [] From 84da6319f34e7663761c6289d7bd19a114231300 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Wed, 9 Jul 2025 00:07:43 -0300 Subject: [PATCH 54/64] ci: only migrate blobdb if it is configured --- dev/build/migration-start.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dev/build/migration-start.sh b/dev/build/migration-start.sh index 901026e53b..578daf5cef 100644 --- a/dev/build/migration-start.sh +++ b/dev/build/migration-start.sh @@ -3,7 +3,11 @@ echo "Running Datatracker migrations..." ./ietf/manage.py migrate --settings=settings_local -echo "Running Blobdb migrations ..." -./ietf/manage.py migrate --settings=settings_local --database=blobdb +# Check whether the blobdb database exists - inspectdb will return a false +# status if not. +if ./ietf/manage.py inspectdb --database blobdb > /dev/null 2>&1; then + echo "Running Blobdb migrations ..." + ./ietf/manage.py migrate --settings=settings_local --database=blobdb +fi echo "Done!" From 8bca0e1af5872244b3c5970008470fffac8329ba Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Sat, 12 Jul 2025 10:06:04 -0300 Subject: [PATCH 55/64] feat: add email/url to purple person API (#9127) --- ietf/api/serializers_rpc.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index dcf1aad864..5cd7777a1a 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -2,21 +2,33 @@ import datetime from typing import Literal, Optional +from django.urls import reverse as urlreverse from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import extend_schema_field from rest_framework import serializers -from ietf.doc.models import DocumentAuthor, Document, RelatedDocument +from ietf.doc.models import DocumentAuthor, Document from ietf.person.models import Person class PersonSerializer(serializers.ModelSerializer): + email = serializers.EmailField(read_only=True) picture = serializers.URLField(source="cdn_photo_url", read_only=True) + url = serializers.SerializerMethodField( + help_text="relative URL for datatracker person page" + ) class Meta: model = Person - fields = ["id", "plain_name", "picture"] - read_only_fields = ["id", "plain_name", "picture"] + fields = ["id", "plain_name", "email", "picture", "url"] + read_only_fields = ["id", "plain_name", "email", "picture", "url"] + + @extend_schema_field(OpenApiTypes.URI) + def get_url(self, object: Person): + return urlreverse( + "ietf.person.views.profile", + kwargs={"email_or_name": object.email_address() or object.name}, + ) class EmailPersonSerializer(serializers.Serializer): From b34bb044059cfc04ce0faaca38801fe4b0b1de5e Mon Sep 17 00:00:00 2001 From: Rudi Matz Date: Thu, 11 Sep 2025 16:48:35 -0400 Subject: [PATCH 56/64] feat: expose consensus in submission api --- ietf/api/serializers_rpc.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index 5cd7777a1a..ae18b672a1 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -8,6 +8,7 @@ from rest_framework import serializers from ietf.doc.models import DocumentAuthor, Document +from ietf.doc.utils import default_consensus from ietf.person.models import Person @@ -153,6 +154,7 @@ class Meta: class SubmittedToQueueSerializer(FullDraftSerializer): submitted = serializers.SerializerMethodField() + consensus = serializers.SerializerMethodField() class Meta: model = Document @@ -161,11 +163,15 @@ class Meta: "name", "stream", "submitted", + "consensus", ] def get_submitted(self, doc) -> Optional[datetime.datetime]: event = doc.sent_to_rfc_editor_event() return None if event is None else event.time + + def get_consensus(self, doc) -> Optional[bool]: + return default_consensus(doc) class OriginalStreamSerializer(serializers.ModelSerializer): From 7bc1d63c493b3fa92060e57c4583a2a46110c2c6 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Fri, 19 Sep 2025 12:16:19 -0300 Subject: [PATCH 57/64] feat: subseries api for red (#9556) * refactor: central def of subseries doc types * feat: subseries doc API * refactor: optimize queries via prefetch Reduced 4500 to 18 queries * chore: remove debug * fix: fix serialization of draft field * refactor: clean up prefetch a bit * feat: filter by subseries type * fix: restore max_limit for RFC pagination --- ietf/api/urls.py | 1 + ietf/doc/api.py | 116 ++++++++++++++++++++++++++++++---------- ietf/doc/models.py | 11 ++++ ietf/doc/serializers.py | 56 ++++++++++++++++--- 4 files changed, 150 insertions(+), 34 deletions(-) diff --git a/ietf/api/urls.py b/ietf/api/urls.py index a88ea5d662..4e4cf4be68 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -24,6 +24,7 @@ # todo more general name for this API? red_router = PrefixedSimpleRouter(name_prefix="ietf.api.red_api") # red api router red_router.register("doc", doc_api.RfcViewSet) +red_router.register("subseries", doc_api.SubseriesViewSet, basename="subseries") api.autodiscover() diff --git a/ietf/doc/api.py b/ietf/doc/api.py index 2fecdfa239..81e9df4902 100644 --- a/ietf/doc/api.py +++ b/ietf/doc/api.py @@ -1,6 +1,7 @@ # Copyright The IETF Trust 2024, All Rights Reserved """Doc API implementations""" -from django.db.models import OuterRef, Subquery, Prefetch, Value, JSONField + +from django.db.models import OuterRef, Subquery, Prefetch, Value, JSONField, QuerySet from django.db.models.functions import TruncDate from django_filters import rest_framework as filters from rest_framework import filters as drf_filters @@ -10,10 +11,16 @@ from rest_framework.viewsets import GenericViewSet from ietf.group.models import Group -from ietf.name.models import StreamName +from ietf.name.models import StreamName, DocTypeName from ietf.utils.timezone import RPC_TZINFO -from .models import Document, DocEvent, RelatedDocument -from .serializers import RfcMetadataSerializer, RfcStatus, RfcSerializer +from .models import Document, DocEvent, RelatedDocument, DocumentAuthor, \ + SUBSERIES_DOC_TYPE_IDS +from .serializers import ( + RfcMetadataSerializer, + RfcStatus, + RfcSerializer, + SubseriesDocSerializer, +) class RfcLimitOffsetPagination(LimitOffsetPagination): @@ -55,38 +62,37 @@ class PrefetchRelatedDocument(Prefetch): those for which the current RFC is the `source`. If `reverse` is True, includes those for which it is the `target` instead. Defaults to only "rfc" documents. """ + @staticmethod + def _get_queryset(relationship_id, reverse, doc_type_id): + """Get queryset to use for the prefetch""" + return RelatedDocument.objects.filter( + **{ + "relationship_id": relationship_id, + f"{'source' if reverse else 'target'}__type_id": doc_type_id, + } + ).select_related("source" if reverse else "target") def __init__(self, to_attr, relationship_id, reverse=False, doc_type_id="rfc"): super().__init__( lookup="targets_related" if reverse else "relateddocument_set", - queryset=RelatedDocument.objects.filter( - **{ - "relationship_id": relationship_id, - f"{'source' if reverse else 'target'}__type_id": doc_type_id, - } - ), + queryset=self._get_queryset(relationship_id, reverse, doc_type_id), to_attr=to_attr, ) -class RfcViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): - permission_classes: list[BasePermission] = [] - lookup_field = "rfc_number" - queryset = ( - Document.objects.filter(type_id="rfc", rfc_number__isnull=False) - .annotate( - published_datetime=Subquery( - DocEvent.objects.filter( - doc_id=OuterRef("pk"), - type="published_rfc", - ) - .order_by("-time") - .values("time")[:1] - ), - ) - .annotate(published=TruncDate("published_datetime", tzinfo=RPC_TZINFO)) - .order_by("-rfc_number") +def augment_rfc_queryset(queryset: QuerySet[Document]): + return ( + queryset + .select_related("std_level", "stream") .prefetch_related( + Prefetch( + "group", + Group.objects.select_related("parent"), + ), + Prefetch( + "documentauthor_set", + DocumentAuthor.objects.select_related("email", "person"), + ), PrefetchRelatedDocument( to_attr="drafts", relationship_id="became_rfc", @@ -102,6 +108,17 @@ class RfcViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): to_attr="updated_by", relationship_id="updates", reverse=True ), ) + .annotate( + published_datetime=Subquery( + DocEvent.objects.filter( + doc_id=OuterRef("pk"), + type="published_rfc", + ) + .order_by("-time") + .values("time")[:1] + ), + ) + .annotate(published=TruncDate("published_datetime", tzinfo=RPC_TZINFO)) .annotate( # TODO implement these fake fields for real is_also=Value([], output_field=JSONField()), @@ -110,7 +127,16 @@ class RfcViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): keywords=Value(["keyword"], output_field=JSONField()), errata=Value([], output_field=JSONField()), ) - ) # default ordering - RfcFilter may override + ) + + +class RfcViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): + permission_classes: list[BasePermission] = [] + lookup_field = "rfc_number" + queryset = augment_rfc_queryset( + Document.objects.filter(type_id="rfc", rfc_number__isnull=False) + ).order_by("-rfc_number") + pagination_class = RfcLimitOffsetPagination filter_backends = [filters.DjangoFilterBackend, drf_filters.SearchFilter] filterset_class = RfcFilter @@ -120,3 +146,37 @@ def get_serializer_class(self): if self.action == "retrieve": return RfcSerializer return RfcMetadataSerializer + + +class PrefetchSubseriesContents(Prefetch): + def __init__(self, to_attr): + super().__init__( + lookup="relateddocument_set", + queryset=RelatedDocument.objects.filter( + relationship_id="contains", + target__type_id="rfc", + ).prefetch_related( + Prefetch( + "target", + queryset=augment_rfc_queryset(Document.objects.all()), + ) + ), + to_attr=to_attr, + ) + + +class SubseriesFilter(filters.FilterSet): + type = filters.ModelMultipleChoiceFilter( + queryset=DocTypeName.objects.filter(pk__in=SUBSERIES_DOC_TYPE_IDS) + ) + + +class SubseriesViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): + permission_classes: list[BasePermission] = [] + lookup_field = "name" + serializer_class = SubseriesDocSerializer + queryset = Document.objects.subseries_docs().prefetch_related( + PrefetchSubseriesContents(to_attr="contents") + ) + filter_backends = [filters.DjangoFilterBackend] + filterset_class = SubseriesFilter diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 52a42e845c..20f500d449 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -937,7 +937,18 @@ def role_for_doc(self): 'invalid' ) + +SUBSERIES_DOC_TYPE_IDS = ("bcp", "fyi", "std") + + +class DocumentQuerySet(models.QuerySet): + def subseries_docs(self): + return self.filter(type_id__in=SUBSERIES_DOC_TYPE_IDS) + + class Document(StorableMixin, DocumentInfo): + objects = DocumentQuerySet.as_manager() + name = models.CharField(max_length=255, validators=[validate_docname,], unique=True) # immutable action_holders = models.ManyToManyField(Person, through=DocumentActionHolder, blank=True) diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py index ff590e8700..4e130966f7 100644 --- a/ietf/doc/serializers.py +++ b/ietf/doc/serializers.py @@ -3,12 +3,13 @@ from dataclasses import dataclass from typing import Literal, ClassVar +from django.db.models.manager import BaseManager from drf_spectacular.utils import extend_schema_field from rest_framework import serializers, fields from ietf.group.serializers import GroupSerializer from ietf.name.serializers import StreamNameSerializer -from .models import Document, DocumentAuthor +from .models import Document, DocumentAuthor, RelatedDocument class RfcAuthorSerializer(serializers.ModelSerializer): @@ -114,10 +115,10 @@ def to_representation(self, instance: Document): return super().to_representation(instance=RfcStatus.from_document(instance)) -class RelatedDraftSerializer(serializers.ModelSerializer): - class Meta: - model = Document - fields = ["id", "name", "title"] +class RelatedDraftSerializer(serializers.Serializer): + id = serializers.IntegerField(source="source.id") + name = serializers.CharField(source="source.name") + title = serializers.CharField(source="source.title") class RelatedRfcSerializer(serializers.Serializer): @@ -144,7 +145,7 @@ class RfcMetadataSerializer(serializers.ModelSerializer): area = GroupSerializer(source="group.area", required=False) stream = StreamNameSerializer() identifiers = fields.SerializerMethodField() - draft = RelatedDraftSerializer(source="came_from_draft", read_only=True) # todo prefetch this + draft = serializers.SerializerMethodField() obsoletes = RelatedRfcSerializer(many=True, read_only=True) obsoleted_by = ReverseRelatedRfcSerializer(many=True, read_only=True) updates = RelatedRfcSerializer(many=True, read_only=True) @@ -190,6 +191,14 @@ def get_identifiers(self, doc: Document): ) return DocIdentifierSerializer(instance=identifiers, many=True).data + @extend_schema_field(RelatedDraftSerializer) + def get_draft(self, object): + try: + related_doc = object.drafts[0] + except IndexError: + return None + return RelatedDraftSerializer(related_doc).data + class RfcSerializer(RfcMetadataSerializer): """Serialize an RFC, including its metadata and text content if available""" @@ -199,3 +208,38 @@ class RfcSerializer(RfcMetadataSerializer): class Meta: model = RfcMetadataSerializer.Meta.model fields = RfcMetadataSerializer.Meta.fields + ["text"] + + +class SubseriesContentListSerializer(serializers.ListSerializer): + """ListSerializer that gets its object from item.target""" + + def to_representation(self, data): + """ + List of object instances -> List of dicts of primitive datatypes. + """ + # Dealing with nested relationships, data can be a Manager, + # so, first get a queryset from the Manager if needed + iterable = ( + data.all() if isinstance(data, BaseManager) else data + ) + # Serialize item.target instead of item itself + return [self.child.to_representation(item.target) for item in iterable] + + +class SubseriesContentSerializer(RfcMetadataSerializer): + """Serialize RFC contained in a subseries doc""" + class Meta(RfcMetadataSerializer.Meta): + list_serializer_class = SubseriesContentListSerializer + + +class SubseriesDocSerializer(serializers.ModelSerializer): + """Serialize a subseries document (e.g., a BCP or STD)""" + contents = SubseriesContentSerializer(many=True) + + class Meta: + model = Document + fields = [ + "name", + "type", + "contents", + ] From cb34712c93e8361f77495c9ef271d90aa188063b Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 22 Sep 2025 12:33:19 -0300 Subject: [PATCH 58/64] feat: add subseries+stub titlepage_name to rfc serializer (#9569) * feat: add subseries to RfcMetadataSerializer * feat: titlepage_name for RfcAuthorSerializer Always blank for now * chore: update copyrights * refactor: use py3.12 typing syntax --- ietf/doc/api.py | 35 ++++++++++++++++++++++++----------- ietf/doc/serializers.py | 37 +++++++++++++++++++++++++------------ 2 files changed, 49 insertions(+), 23 deletions(-) diff --git a/ietf/doc/api.py b/ietf/doc/api.py index 81e9df4902..90ab877bc7 100644 --- a/ietf/doc/api.py +++ b/ietf/doc/api.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2024, All Rights Reserved +# Copyright The IETF Trust 2024-2025, All Rights Reserved """Doc API implementations""" from django.db.models import OuterRef, Subquery, Prefetch, Value, JSONField, QuerySet @@ -13,8 +13,13 @@ from ietf.group.models import Group from ietf.name.models import StreamName, DocTypeName from ietf.utils.timezone import RPC_TZINFO -from .models import Document, DocEvent, RelatedDocument, DocumentAuthor, \ - SUBSERIES_DOC_TYPE_IDS +from .models import ( + Document, + DocEvent, + RelatedDocument, + DocumentAuthor, + SUBSERIES_DOC_TYPE_IDS, +) from .serializers import ( RfcMetadataSerializer, RfcStatus, @@ -62,28 +67,31 @@ class PrefetchRelatedDocument(Prefetch): those for which the current RFC is the `source`. If `reverse` is True, includes those for which it is the `target` instead. Defaults to only "rfc" documents. """ + @staticmethod - def _get_queryset(relationship_id, reverse, doc_type_id): + def _get_queryset(relationship_id, reverse, doc_type_ids): """Get queryset to use for the prefetch""" + if isinstance(doc_type_ids, str): + doc_type_ids = (doc_type_ids,) + return RelatedDocument.objects.filter( **{ "relationship_id": relationship_id, - f"{'source' if reverse else 'target'}__type_id": doc_type_id, + f"{'source' if reverse else 'target'}__type_id__in": doc_type_ids, } ).select_related("source" if reverse else "target") - def __init__(self, to_attr, relationship_id, reverse=False, doc_type_id="rfc"): + def __init__(self, to_attr, relationship_id, reverse=False, doc_type_ids="rfc"): super().__init__( lookup="targets_related" if reverse else "relateddocument_set", - queryset=self._get_queryset(relationship_id, reverse, doc_type_id), + queryset=self._get_queryset(relationship_id, reverse, doc_type_ids), to_attr=to_attr, ) def augment_rfc_queryset(queryset: QuerySet[Document]): return ( - queryset - .select_related("std_level", "stream") + queryset.select_related("std_level", "stream") .prefetch_related( Prefetch( "group", @@ -96,7 +104,7 @@ def augment_rfc_queryset(queryset: QuerySet[Document]): PrefetchRelatedDocument( to_attr="drafts", relationship_id="became_rfc", - doc_type_id="draft", + doc_type_ids="draft", reverse=True, ), PrefetchRelatedDocument(to_attr="obsoletes", relationship_id="obs"), @@ -107,6 +115,12 @@ def augment_rfc_queryset(queryset: QuerySet[Document]): PrefetchRelatedDocument( to_attr="updated_by", relationship_id="updates", reverse=True ), + PrefetchRelatedDocument( + to_attr="subseries", + relationship_id="contains", + reverse=True, + doc_type_ids=SUBSERIES_DOC_TYPE_IDS, + ), ) .annotate( published_datetime=Subquery( @@ -121,7 +135,6 @@ def augment_rfc_queryset(queryset: QuerySet[Document]): .annotate(published=TruncDate("published_datetime", tzinfo=RPC_TZINFO)) .annotate( # TODO implement these fake fields for real - is_also=Value([], output_field=JSONField()), see_also=Value([], output_field=JSONField()), formats=Value(["txt", "xml"], output_field=JSONField()), keywords=Value(["keyword"], output_field=JSONField()), diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py index 4e130966f7..e95ba19356 100644 --- a/ietf/doc/serializers.py +++ b/ietf/doc/serializers.py @@ -1,5 +1,6 @@ -# Copyright The IETF Trust 2024, All Rights Reserved +# Copyright The IETF Trust 2024-2025, All Rights Reserved """django-rest-framework serializers""" + from dataclasses import dataclass from typing import Literal, ClassVar @@ -9,18 +10,26 @@ from ietf.group.serializers import GroupSerializer from ietf.name.serializers import StreamNameSerializer -from .models import Document, DocumentAuthor, RelatedDocument +from .models import Document, DocumentAuthor class RfcAuthorSerializer(serializers.ModelSerializer): """Serializer for a DocumentAuthor in a response""" name = fields.CharField(source="person.plain_name") + titlepage_name = fields.CharField(default="") email = fields.EmailField(source="email.address", required=False) class Meta: model = DocumentAuthor - fields = ["person", "name", "email", "affiliation", "country"] + fields = [ + "person", + "name", + "titlepage_name", + "email", + "affiliation", + "country", + ] @dataclass @@ -34,9 +43,7 @@ class DocIdentifierSerializer(serializers.Serializer): value = serializers.CharField() -# This should become "type RfcStatusSlugT ..." when we drop pre-py3.12 support -# It should be "RfcStatusSlugT: TypeAlias ..." when we drop py3.9 support -RfcStatusSlugT = Literal[ +type RfcStatusSlugT = Literal[ "standard", "bcp", "informational", @@ -133,8 +140,14 @@ class ReverseRelatedRfcSerializer(serializers.Serializer): title = serializers.CharField(source="source.title") +class ContainingSubseriesSerializer(serializers.Serializer): + name = serializers.CharField(source="source.name") + type = serializers.CharField(source="source.type_id") + + class RfcMetadataSerializer(serializers.ModelSerializer): """Serialize metadata of an RFC""" + RFC_FORMATS = ("xml", "txt", "html", "htmlized", "pdf", "ps") number = serializers.IntegerField(source="rfc_number") @@ -150,7 +163,7 @@ class RfcMetadataSerializer(serializers.ModelSerializer): obsoleted_by = ReverseRelatedRfcSerializer(many=True, read_only=True) updates = RelatedRfcSerializer(many=True, read_only=True) updated_by = ReverseRelatedRfcSerializer(many=True, read_only=True) - is_also = serializers.ListField(child=serializers.CharField(), read_only=True) + subseries = ContainingSubseriesSerializer(many=True, read_only=True) see_also = serializers.ListField(child=serializers.CharField(), read_only=True) formats = fields.MultipleChoiceField(choices=RFC_FORMATS) keywords = serializers.ListField(child=serializers.CharField(), read_only=True) @@ -173,7 +186,7 @@ class Meta: "obsoleted_by", "updates", "updated_by", - "is_also", + "subseries", "see_also", "draft", "abstract", @@ -212,28 +225,28 @@ class Meta: class SubseriesContentListSerializer(serializers.ListSerializer): """ListSerializer that gets its object from item.target""" - + def to_representation(self, data): """ List of object instances -> List of dicts of primitive datatypes. """ # Dealing with nested relationships, data can be a Manager, # so, first get a queryset from the Manager if needed - iterable = ( - data.all() if isinstance(data, BaseManager) else data - ) + iterable = data.all() if isinstance(data, BaseManager) else data # Serialize item.target instead of item itself return [self.child.to_representation(item.target) for item in iterable] class SubseriesContentSerializer(RfcMetadataSerializer): """Serialize RFC contained in a subseries doc""" + class Meta(RfcMetadataSerializer.Meta): list_serializer_class = SubseriesContentListSerializer class SubseriesDocSerializer(serializers.ModelSerializer): """Serialize a subseries document (e.g., a BCP or STD)""" + contents = SubseriesContentSerializer(many=True) class Meta: From 983c7944dac4efe1b604f43f7a76ab6229f024f3 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Mon, 22 Sep 2025 13:10:40 -0300 Subject: [PATCH 59/64] fix: renumber migrations --- ...e.py => 0027_alter_dochistory_title_alter_document_title.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename ietf/doc/migrations/{0026_alter_dochistory_title_alter_document_title.py => 0027_alter_dochistory_title_alter_document_title.py} (94%) diff --git a/ietf/doc/migrations/0026_alter_dochistory_title_alter_document_title.py b/ietf/doc/migrations/0027_alter_dochistory_title_alter_document_title.py similarity index 94% rename from ietf/doc/migrations/0026_alter_dochistory_title_alter_document_title.py rename to ietf/doc/migrations/0027_alter_dochistory_title_alter_document_title.py index 5f5140a278..79349e0e08 100644 --- a/ietf/doc/migrations/0026_alter_dochistory_title_alter_document_title.py +++ b/ietf/doc/migrations/0027_alter_dochistory_title_alter_document_title.py @@ -6,7 +6,7 @@ class Migration(migrations.Migration): dependencies = [ - ("doc", "0025_storedobject_storedobject_unique_name_per_store"), + ("doc", "0026_change_wg_state_descriptions"), ] operations = [ From 4f5da10047a006d21281d197da51f01258a004bf Mon Sep 17 00:00:00 2001 From: Rudi Matz Date: Wed, 24 Sep 2025 11:44:36 -0400 Subject: [PATCH 60/64] feat: add consensus on FullDraftSerializer --- ietf/api/serializers_rpc.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py index ae18b672a1..5ce588a976 100644 --- a/ietf/api/serializers_rpc.py +++ b/ietf/api/serializers_rpc.py @@ -99,6 +99,7 @@ class FullDraftSerializer(serializers.ModelSerializer): source_format = serializers.SerializerMethodField() authors = DocumentAuthorSerializer(many=True, source="documentauthor_set") shepherd = serializers.SerializerMethodField() + consensus = serializers.SerializerMethodField() class Meta: model = Document @@ -113,8 +114,12 @@ class Meta: "authors", "shepherd", "intended_std_level", + "consensus", ] + def get_consensus(self, doc: Document) -> Optional[bool]: + return default_consensus(doc) + def get_source_format( self, doc: Document ) -> Literal["unknown", "xml-v2", "xml-v3", "txt"]: From 18cc41ac59ee10853b9da5b14d40998f8d4a7cd7 Mon Sep 17 00:00:00 2001 From: Rudi Matz Date: Thu, 23 Oct 2025 10:22:02 -0400 Subject: [PATCH 61/64] feat: add type field in serializer --- ietf/group/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ietf/group/serializers.py b/ietf/group/serializers.py index b123e091e3..08e6bba81a 100644 --- a/ietf/group/serializers.py +++ b/ietf/group/serializers.py @@ -8,4 +8,4 @@ class GroupSerializer(serializers.ModelSerializer): class Meta: model = Group - fields = ["acronym", "name"] + fields = ["acronym", "name", "type"] From 23960e94473db27f085b9d88e560fa4711828f8c Mon Sep 17 00:00:00 2001 From: Rudi Matz Date: Thu, 23 Oct 2025 15:33:00 -0400 Subject: [PATCH 62/64] feat: tag subseries API endpoints for purple (#9763) * feat: tag subseries API endpoints for purple * ruff --- ietf/doc/api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ietf/doc/api.py b/ietf/doc/api.py index 90ab877bc7..a8f51bac74 100644 --- a/ietf/doc/api.py +++ b/ietf/doc/api.py @@ -10,6 +10,7 @@ from rest_framework.permissions import BasePermission from rest_framework.viewsets import GenericViewSet +from drf_spectacular.utils import extend_schema from ietf.group.models import Group from ietf.name.models import StreamName, DocTypeName from ietf.utils.timezone import RPC_TZINFO @@ -184,6 +185,7 @@ class SubseriesFilter(filters.FilterSet): ) +@extend_schema(tags=["purple", "red"]) class SubseriesViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): permission_classes: list[BasePermission] = [] lookup_field = "name" From d1aa69037eea4c94ba20d7c81c51113fd9b193b2 Mon Sep 17 00:00:00 2001 From: Rudi Matz Date: Thu, 23 Oct 2025 15:50:24 -0400 Subject: [PATCH 63/64] feat: change slugs/names (#9778) * change slugs/names * change slug names * fix: update RfcStatusSlugT --- ietf/doc/serializers.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py index e95ba19356..12761f4b2a 100644 --- a/ietf/doc/serializers.py +++ b/ietf/doc/serializers.py @@ -44,13 +44,7 @@ class DocIdentifierSerializer(serializers.Serializer): type RfcStatusSlugT = Literal[ - "standard", - "bcp", - "informational", - "experimental", - "historic", - "unknown", - "not-issued", + "std", "ps", "ds", "bcp", "inf", "exp", "hist", "unkn", "not-issued", ] @@ -62,20 +56,26 @@ class RfcStatus: # Names that aren't just the slug itself. ClassVar annotation prevents dataclass from treating this as a field. fancy_names: ClassVar[dict[RfcStatusSlugT, str]] = { - "standard": "standards track", + "std": "internet standard", + "ps": "proposed standard", + "ds": "draft standard", "bcp": "best current practice", + "inf": "informational", + "exp": "experimental", + "hist": "historic", + "unkn": "unknown", } # ClassVar annotation prevents dataclass from treating this as a field stdlevelname_slug_map: ClassVar[dict[str, RfcStatusSlugT]] = { "bcp": "bcp", - "ds": "standard", # ds is obsolete - "exp": "experimental", - "hist": "historic", - "inf": "informational", - "std": "standard", - "ps": "standard", - "unkn": "unknown", + "ds": "ds", + "exp": "exp", + "hist": "hist", + "inf": "inf", + "std": "std", + "ps": "ps", + "unkn": "unkn", } # ClassVar annotation prevents dataclass from treating this as a field From c020f0fb4472a9e54187dba2d50600e6d4793b39 Mon Sep 17 00:00:00 2001 From: Jennifer Richards Date: Thu, 23 Oct 2025 21:11:23 -0300 Subject: [PATCH 64/64] fix: remove double tag (#9787) This was meant to include the API call in both purple and red API clients. It seems this does not work, at least with some generators. Need to investigate further, but we should be able to work around it. --- ietf/doc/api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ietf/doc/api.py b/ietf/doc/api.py index a8f51bac74..2eac012c6a 100644 --- a/ietf/doc/api.py +++ b/ietf/doc/api.py @@ -185,7 +185,6 @@ class SubseriesFilter(filters.FilterSet): ) -@extend_schema(tags=["purple", "red"]) class SubseriesViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): permission_classes: list[BasePermission] = [] lookup_field = "name"