Use root ca explicitly in http-based adapters (#262)

* Use root ca explicitly in http-based adapters

* Check cert's file encoding

* Add typing

* fix imports

Author: thenno

lib/dl_api_commons/dl_api_commons/aiohttp/aiohttp_client.py

@@ -113,7 +113,7 @@ def __attrs_post_init__(self):  # type: ignore  # 2024-01-24 # TODO: Function is
         self._session = self._make_session()
 
     def _make_session(self) -> aiohttp.ClientSession:
-        ssl_context = ssl.create_default_context(cadata=self._ca_data.decode("utf-8"))
+        ssl_context = ssl.create_default_context(cadata=self._ca_data.decode("ascii"))
         return aiohttp.ClientSession(
             cookies=self.cookies,
             headers=self.headers,

lib/dl_api_lib_testing/dl_api_lib_testing/base.py

@@ -134,7 +134,7 @@ def control_api_app_settings(
             rqe_config_subprocess=rqe_config_subprocess,
         )
 
-    @pytest.fixture(scope="function")
+    @pytest.fixture(scope="session")
     def ca_data(self) -> bytes:
         return get_root_certificates()
 

lib/dl_configs/dl_configs/utils.py

@@ -1,3 +1,4 @@
+import logging
 import os
 from typing import (
     Callable,
@@ -12,6 +13,9 @@
 TEMP_ROOT_CERTIFICATES_FOLDER_PATH = "/tmp/ssl/certs/"
 
 
+LOGGER = logging.getLogger(__name__)
+
+
 _T = TypeVar("_T")
 
 
@@ -25,8 +29,21 @@ def validator(value: _T) -> _T:
 
 
 def get_root_certificates(path: str = DEFAULT_ROOT_CERTIFICATES_FILENAME) -> bytes:
+    """
+    expects a path to a file with PEM certificates
+
+    aiohttp-based clients expect certificates as an ascii string to create ssl.sslContext
+    while grpc-clients expect them as a byte representation of an ascii string to create the special grpc ssl context
+    """
     with open(path, "rb") as fobj:
-        return fobj.read()
+        ca_data = fobj.read()
+    # fail fast
+    try:
+        ca_data.decode("ascii")
+    except UnicodeDecodeError:
+        LOGGER.exception("Looks like the certificates are not in PEM format")
+        raise
+    return ca_data
 
 
 def get_root_certificates_path() -> str:

lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds/core/connection_executors.py

@@ -59,5 +59,6 @@ async def _make_target_conn_dto_pool(self) -> Sequence[BitrixGDSConnTargetDTO]:
                 connect_timeout=self._conn_options.connect_timeout,  # type: ignore  # TODO: fix
                 redis_conn_params=conn_params,
                 redis_caches_ttl=caches_ttl,
+                ca_data=self._ca_data.decode("ascii"),
             )
         ]

lib/dl_connector_bitrix_gds/dl_connector_bitrix_gds/core/target_dto.py

@@ -2,7 +2,7 @@
 
 import attr
 
-from dl_core.connection_executors.models.connection_target_dto_base import ConnTargetDTO
+from dl_core.connection_executors.models.connection_target_dto_base import BaseAiohttpConnTargetDTO
 from dl_core.utils import secrepr
 
 
@@ -15,7 +15,7 @@ def hide_pass(value: Optional[dict]) -> str:
 
 
 @attr.s(frozen=True)
-class BitrixGDSConnTargetDTO(ConnTargetDTO):
+class BitrixGDSConnTargetDTO(BaseAiohttpConnTargetDTO):
     portal: str = attr.ib(kw_only=True)
     token: str = attr.ib(kw_only=True, repr=secrepr)
 

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_base/core/connection_executors.py

@@ -34,6 +34,7 @@ async def _make_target_conn_dto_pool(self) -> list[BaseFileS3ConnTargetDTO]:  #
                     access_key_id=self._conn_dto.access_key_id,
                     secret_access_key=self._conn_dto.secret_access_key,
                     replace_secret=self._conn_dto.replace_secret,
+                    ca_data=self._ca_data.decode("ascii"),
                 )
             )
         return dto_pool

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3/chs3_base/core/target_dto.py

@@ -2,12 +2,15 @@
 
 import attr
 
-from dl_core.connection_executors.models.connection_target_dto_base import BaseSQLConnTargetDTO
+from dl_core.connection_executors.models.connection_target_dto_base import (
+    BaseAiohttpConnTargetDTO,
+    BaseSQLConnTargetDTO,
+)
 from dl_core.utils import secrepr
 
 
 @attr.s
-class BaseFileS3ConnTargetDTO(BaseSQLConnTargetDTO):
+class BaseFileS3ConnTargetDTO(BaseAiohttpConnTargetDTO, BaseSQLConnTargetDTO):
     protocol: str = attr.ib(kw_only=True)
     disable_value_processing: bool = attr.ib(kw_only=True)
 

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/base/core/base.py

@@ -107,25 +107,29 @@ def task_processor_factory(self) -> TaskProcessorFactory:
     @pytest.fixture(scope="session")
     def conn_sync_service_registry(
         self,
+        root_certificates: bytes,
         conn_bi_context: RequestContextInfo,
         task_processor_factory: TaskProcessorFactory,
     ) -> ServicesRegistry:
         return self.service_registry_factory(
             conn_exec_factory_async_env=False,
             conn_bi_context=conn_bi_context,
             task_processor_factory=task_processor_factory,
+            root_certificates_data=root_certificates,
         )
 
     @pytest.fixture(scope="session")
     def conn_async_service_registry(
         self,
+        root_certificates: bytes,
         conn_bi_context: RequestContextInfo,
         task_processor_factory: TaskProcessorFactory,
     ) -> ServicesRegistry:
         return self.service_registry_factory(
             conn_exec_factory_async_env=True,
             conn_bi_context=conn_bi_context,
             task_processor_factory=task_processor_factory,
+            root_certificates_data=root_certificates,
         )
 
     @pytest.fixture(scope="function")

lib/dl_connector_bundle_chs3/dl_connector_bundle_chs3_tests/db/conftest.py

@@ -6,6 +6,7 @@
 import pytest
 
 from dl_api_lib_testing.initialization import initialize_api_lib_test
+from dl_testing.utils import get_root_certificates
 
 from dl_connector_bundle_chs3.chs3_base.core.us_connection import BaseFileS3Connection
 from dl_connector_bundle_chs3_tests.db.config import API_TEST_CONFIG
@@ -21,3 +22,8 @@ def _patched(self: Any, s3_filename_suffix: str) -> str:
         return s3_filename_suffix
 
     monkeypatch.setattr(BaseFileS3Connection, "get_full_s3_filename", _patched)
+
+
+@pytest.fixture(scope="session")
+def root_certificates() -> bytes:
+    return get_root_certificates()

lib/dl_connector_chyt/dl_connector_chyt/core/connection_executors.py

@@ -63,6 +63,7 @@ async def _get_target_conn_dto(self) -> CHYTConnTargetDTO:
             insert_quorum=self._conn_options.insert_quorum,
             insert_quorum_timeout=self._conn_options.insert_quorum_timeout,
             disable_value_processing=self._conn_options.disable_value_processing,
+            ca_data=self._ca_data.decode("ascii"),
         )