add support for global paginator (#61)

* add support for global paginator

* add warning for incomplete paginators

Author: sh-rp

dlt_openapi/detector/default/__init__.py

@@ -2,6 +2,7 @@
 Default open source detector
 """
 
+import json
 from typing import Dict, List, Optional, Tuple, Union, cast
 
 from dlt_openapi.config import Config
@@ -40,6 +41,7 @@
     BaseDetectionWarning,
     DataResponseNoBodyWarning,
     DataResponseUndetectedWarning,
+    PossiblePaginatorWarning,
     PrimaryKeyNotFoundWarning,
     UnresolvedPathParametersWarning,
     UnsupportedSecuritySchemeWarning,
@@ -64,6 +66,7 @@ def run(self, open_api: OpenapiParser) -> None:
 
         # discover stuff from responses
         self.detect_paginators_and_responses(open_api.endpoints)
+        self.detect_global_pagination(open_api)
 
         # discover parent child relationship
         self.detect_parent_child_relationships(open_api.endpoints)
@@ -86,6 +89,7 @@ def detect_security_schemes(self, open_api: OpenapiParser) -> None:
         schemes = list(open_api.security_schemes.values())
 
         # detect scheme settings
+        # TODO: make this a bit nicer
         for scheme in schemes:
 
             if scheme.type == "apiKey":
@@ -112,7 +116,7 @@ def detect_security_schemes(self, open_api: OpenapiParser) -> None:
 
         # find default scheme
         if len(schemes) and schemes[0].supported:
-            open_api.detected_default_security_scheme = schemes[0]
+            open_api.detected_global_security_scheme = schemes[0]
         elif len(schemes) and not schemes[0].supported:
             self._add_warning(UnsupportedSecuritySchemeWarning(schemes[0].name))
 
@@ -200,6 +204,40 @@ def detect_paginators_and_responses(self, endpoints: EndpointCollection) -> None
                 )
                 self.detect_primary_key(endpoint, endpoint.detected_data_response, endpoint.path)
 
+    def detect_global_pagination(self, open_api: OpenapiParser) -> None:
+        """go through all detected paginators and see which one we can set as global"""
+        paginator_by_key: Dict[str, Pagination] = {}
+        paginator_count: Dict[str, int] = {}
+
+        # count how many every paginator appears
+        for endpoint in open_api.endpoints.endpoints:
+            if not endpoint.detected_pagination:
+                continue
+            params = endpoint.detected_pagination.paginator_config
+            key = json.dumps(params, sort_keys=True)
+            paginator_by_key[key] = endpoint.detected_pagination
+            paginator_count.setdefault(key, 0)
+            paginator_count[key] += 1
+
+        # no paginators found
+        if len(paginator_by_key) == 0:
+            return
+
+        # sort dict by value descending, so most used paginator is at the top
+        sorted_paginator_count = sorted(paginator_count.items(), key=lambda item: item[1] * -1)
+
+        # we only set a global paginator, if we found one paginator, or if the top paginator has
+        # a higher count than the second most used one
+        if not (len(paginator_by_key) == 1 or sorted_paginator_count[0][1] > sorted_paginator_count[1][1]):
+            return
+
+        global_paginator = paginator_by_key[sorted_paginator_count[0][0]]
+
+        # set global paginator on base object but also set on all endpoints
+        open_api.detected_global_pagination = global_paginator
+        for e in open_api.endpoints.endpoints:
+            e.detected_global_pagination = global_paginator
+
     def detect_primary_key(self, e: Endpoint, response: Response, path: str) -> None:
         """detect the primary key from the payload"""
         if not response.detected_payload:
@@ -417,6 +455,10 @@ def detect_pagination(self, endpoint: Endpoint) -> Optional[Pagination]:
         #
         # Nothing found :(
         #
+        pagination_params = [*cursor_params, *offset_params, *limit_params, *page_params]
+        if pagination_params:
+            self._add_warning(PossiblePaginatorWarning([p.name for p in pagination_params]), endpoint)
+
         return None
 
     def detect_parent_child_relationships(self, endpoints: EndpointCollection) -> None:

dlt_openapi/detector/default/warnings.py

@@ -11,7 +11,7 @@ class UnresolvedPathParametersWarning(BaseDetectionWarning):
 
     def __init__(self, params: List[str]) -> None:
         self.params = params
-        self.msg = f"Could not resolve all path params, setting default values for: {','.join(params)}"
+        self.msg = f"Could not resolve all path params, setting default values for: {', '.join(params)}"
 
 
 class DataResponseUndetectedWarning(BaseDetectionWarning):
@@ -35,3 +35,12 @@ def __init__(self, security_scheme: str) -> None:
             f"Security Scheme {security_scheme} is not supported natively at this time. "
             + "Please provide a custom implementation."
         )
+
+
+class PossiblePaginatorWarning(BaseDetectionWarning):
+    def __init__(self, params: List[str]) -> None:
+        self.params = params
+        self.msg = (
+            "Found params that suggest this endpoint is paginated, but could not discover pagination mechnanism. "
+            + f"Params: {', '.join(params)}"
+        )

dlt_openapi/parser/endpoints.py

@@ -54,6 +54,7 @@ class Endpoint:
 
     # detected values
     detected_pagination: Optional[Pagination] = None
+    detected_global_pagination: Optional[Pagination] = None
     detected_data_response: Optional[Response] = None
     detected_resource_name: Optional[str] = None
     detected_table_name: Optional[str] = None
@@ -89,7 +90,10 @@ def list_all_parameters(self) -> List[Parameter]:
         return list(self.parameters.values())
 
     @property
-    def pagination_args(self) -> Optional[Dict[str, Union[str, int]]]:
+    def render_pagination_args(self) -> Optional[Dict[str, Union[str, int]]]:
+        # if own paginator equals global paginator, do not render anything
+        if self.detected_pagination == self.detected_global_pagination:
+            return None
         return self.detected_pagination.paginator_config if self.detected_pagination else None
 
     @property

dlt_openapi/parser/openapi_parser.py

@@ -12,6 +12,7 @@
 from dlt_openapi.parser.context import OpenapiContext
 from dlt_openapi.parser.endpoints import EndpointCollection
 from dlt_openapi.parser.info import OpenApiInfo
+from dlt_openapi.parser.pagination import Pagination
 from dlt_openapi.parser.security import SecurityScheme
 
 
@@ -21,7 +22,8 @@ class OpenapiParser:
     endpoints: EndpointCollection = None
     security_schemes: Dict[str, SecurityScheme] = {}
 
-    detected_default_security_scheme: SecurityScheme = None
+    detected_global_security_scheme: SecurityScheme = None
+    detected_global_pagination: Pagination = None
 
     def __init__(self, config: Config) -> None:
         self.config = config

dlt_openapi/parser/pagination.py

@@ -14,3 +14,8 @@ class Pagination:
     def param_names(self) -> List[str]:
         """All params used for pagination"""
         return [param.name for param in self.pagination_params]
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, Pagination):
+            return other.paginator_config == self.paginator_config
+        return False

dlt_openapi/renderer/default/__init__.py

@@ -125,7 +125,12 @@ def _render_source(self) -> str:
             source_name=self.source_name,
             endpoint_collection=self.openapi.endpoints,
             imports=[],
-            credentials=self.openapi.detected_default_security_scheme,
+            credentials=self.openapi.detected_global_security_scheme,
+            global_paginator_config=(
+                self.openapi.detected_global_pagination.paginator_config
+                if self.openapi.detected_global_pagination
+                else None
+            ),
         )
 
     def _build_pipeline(self) -> None:

dlt_openapi/renderer/default/templates/source.py.j2

@@ -24,6 +24,18 @@ def {{ source_name }}(
                 {{ credentials.detected_auth_vars }}
             },
             {% endif %}
+            {% if global_paginator_config %}
+            "paginator": {
+                {% for key, value in global_paginator_config.items() %}
+                "{{key}}":
+                    {% if value is integer %}
+                    {{value}},
+                    {% else %}
+                    "{{value}}",
+                    {% endif %}
+                {% endfor %}
+            },
+            {% endif %}
         },
         "resources": 
         [
@@ -60,9 +72,9 @@ def {{ source_name }}(
                     {% endfor %}
                 },
                 {% endif %}
-                {% if endpoint.pagination_args %}
+                {% if endpoint.render_pagination_args %}
                 "paginator": {
-                    {% for key, value in endpoint.pagination_args.items() %}
+                    {% for key, value in endpoint.render_pagination_args.items() %}
                     "{{key}}":
                         {% if value is integer %}
                         {{value}},

tests/cases/artificial_specs/global_pagination.yml

@@ -0,0 +1,92 @@
+openapi: 3.0.0
+info:
+  title: 'pagination'
+  version: 1.0.0
+  description: 'pagination example with global pagination'
+servers:
+- url: 'https://pokeapi.co/'
+
+paths:
+
+  /collection_1/:
+    get:
+      operationId: collection_1
+      parameters:
+      - in: query
+        name: cursor
+        schema:
+          description: Put cursor here
+          title: Cursor
+          type: string
+      responses:
+        '200':
+          description: "OK"
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  cursor:
+                    type: string
+                  results:
+                    type: array
+
+  /collection_2/:
+    get:
+      operationId: collection_2
+      parameters:
+      - in: query
+        name: cursor
+        schema:
+          description: Put cursor here
+          title: Cursor
+          type: string
+      responses:
+        '200':
+          description: "OK"
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  cursor:
+                    type: string
+                  other_value:
+                    type: array
+
+
+  /collection_other_paginator/:
+    get:
+      operationId: collection_other_paginator
+      parameters:
+      - in: query
+        name: page
+        schema:
+          type: string
+      responses:
+        '200':
+          description: "OK"
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  count:
+                    type: integer
+                    example: 3
+
+  /item_endpoint/{some_id}:
+    get:
+      operationId: item_endpoint
+      responses:
+        '200':
+          description: "OK"
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  id:
+                    type: string
+                  name:
+                    type: str

tests/cases/artificial_specs/pagination.yml

@@ -149,4 +149,28 @@ paths:
                 properties:
                   count:
                     type: integer
-                    example: 3
+                    example: 3
+
+
+  /cursor_pagination_incomplete/:
+    get:
+      operationId: cursor_pagination_incomplete
+      parameters:
+      - in: query
+        name: cursor
+        schema:
+          description: Put cursor here
+          title: Cursor
+          type: string
+      responses:
+        '200':
+          description: "OK"
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  other_value:
+                    type: string
+                  results:
+                    type: array

tests/integration/basics/test_paginator.py

@@ -3,7 +3,8 @@
 import pytest
 
 from dlt_openapi.config import Config
-from tests.integration.utils import get_indexed_resources
+from dlt_openapi.detector.default.warnings import PossiblePaginatorWarning
+from tests.integration.utils import get_dict_by_case, get_indexed_resources, get_project_by_case
 
 
 @pytest.fixture(scope="module")
@@ -57,3 +58,38 @@ def test_page_number_paginator_with_count(paginators: Dict[str, Any]) -> None:
         "page_param": "page",
         "total_path": "count",
     }
+
+
+def test_global_paginator() -> None:
+    api_dict = get_dict_by_case("artificial", "global_pagination.yml", config=Config(name_resources_by_operation=True))
+    # we have a global cursor paginator
+    assert api_dict["client"]["paginator"] == {
+        "type": "cursor",
+        "cursor_path": "cursor",
+        "cursor_param": "cursor",
+    }
+
+    # check endpoint pagination settings
+    resources: Any = {entry["name"]: entry for entry in api_dict["resources"]}  # type: ignore
+    assert not resources["item_endpoint"]["endpoint"].get("paginator")
+    assert not resources["collection_1"]["endpoint"].get("paginator")
+    assert not resources["collection_2"]["endpoint"].get("paginator")
+
+    # there is a different paginator on the diverging endpoint
+    assert resources["collection_other_paginator"]["endpoint"].get("paginator") == {
+        "page_param": "page",
+        "total_path": "count",
+        "type": "page_number",
+    }
+
+
+def test_incomplete_paginator_warning() -> None:
+    project = get_project_by_case("artificial", "pagination.yml", config=Config(name_resources_by_operation=True))
+
+    # check if the warnings exist that we expect
+    warnings = project.detector.get_warnings()
+
+    # warning for possible paginator
+    assert len(warnings.get("cursor_pagination_incomplete")) == 2
+    assert type(warnings.get("cursor_pagination_incomplete")[0]) == PossiblePaginatorWarning
+    assert warnings.get("cursor_pagination_incomplete")[0].params == ["cursor"]  # type: ignore