Merge branch 'main' into http-connector

Author: edgarrmondragon

.github/workflows/constraints.txt

@@ -1,4 +1,4 @@
-pip==23.2
+pip==23.2.1
 poetry==1.5.1
 pre-commit==3.3.3
 nox==2023.4.22

.github/workflows/test.yml

@@ -37,7 +37,7 @@ env:
 
 jobs:
   tests:
-    name: Test on ${{ matrix.python-version }} (${{ matrix.session }}) / ${{ matrix.os }}
+    name: "Test on ${{ matrix.python-version }} (${{ matrix.session }}) / ${{ matrix.os }} / SQLAlchemy: ${{ matrix.sqlalchemy }}"
     runs-on: ${{ matrix.os }}
     env:
       NOXSESSION: ${{ matrix.session }}
@@ -47,9 +47,11 @@ jobs:
         session: [tests]
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
         python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        sqlalchemy: ["2.*"]
         include:
-        - { session: doctest, python-version: "3.10", os: "ubuntu-latest" }
-        - { session: mypy,    python-version: "3.8",  os: "ubuntu-latest" }
+        - { session: tests,   python-version: "3.11", os: "ubuntu-latest", sqlalchemy: "1.*" }
+        - { session: doctest, python-version: "3.10", os: "ubuntu-latest", sqlalchemy: "2.*" }
+        - { session: mypy,    python-version: "3.8",  os: "ubuntu-latest", sqlalchemy: "2.*" }
 
     steps:
     - name: Check out the repository
@@ -86,6 +88,8 @@ jobs:
         nox --version
 
     - name: Run Nox
+      env:
+        SQLALCHEMY_VERSION: ${{ matrix.sqlalchemy }}
       run: |
         nox --python=${{ matrix.python-version }}
 

.pre-commit-config.yaml

@@ -36,14 +36,14 @@ repos:
         )$
 
 - repo: https://github.com/python-jsonschema/check-jsonschema
-  rev: 0.23.2
+  rev: 0.23.3
   hooks:
   - id: check-dependabot
   - id: check-github-workflows
   - id: check-readthedocs
 
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.0.277
+  rev: v0.0.280
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix, --show-fixes]

docs/deprecation.md

@@ -11,3 +11,5 @@ incompatible way, following their deprecation, as indicated in the
   [`RESTStream.get_new_paginator`](singer_sdk.RESTStream.get_new_paginator).
 
   See the [migration guide](./guides/pagination-classes.md) for more information.
+
+- The `singer_sdk.testing.get_standard_tap_tests` and `singer_sdk.testing.get_standard_target_tests` functions will be removed. Replace them with `singer_sdk.testing.get_tap_test_class` and `singer_sdk.testing.get_target_test_class` functions respective to generate a richer test suite.

noxfile.py

@@ -86,6 +86,14 @@ def tests(session: Session) -> None:
     session.install(".[s3]")
     session.install(*test_dependencies)
 
+    sqlalchemy_version = os.environ.get("SQLALCHEMY_VERSION")
+    if sqlalchemy_version:
+        # Bypass nox-poetry use of --constraint so we can install a version of
+        # SQLAlchemy that doesn't match what's in poetry.lock.
+        session.poetry.session.install(  # type: ignore[attr-defined]
+            f"sqlalchemy=={sqlalchemy_version}",
+        )
+
     try:
         session.run(
             "coverage",
@@ -96,9 +104,6 @@ def tests(session: Session) -> None:
             "-v",
             "--durations=10",
             *session.posargs,
-            env={
-                "SQLALCHEMY_WARN_20": "1",
-            },
         )
     finally:
         if session.interactive:

poetry.lock

@@ -52,7 +52,7 @@ memoization = ">=0.3.2,<0.5.0"
 jsonpath-ng = "^1.5.3"
 joblib = "^1.0.1"
 inflection = "^0.5.1"
-sqlalchemy = "^1.4"
+sqlalchemy = ">=1.4,<3.0"
 python-dotenv = ">=0.20,<0.22"
 typing-extensions = "^4.2.0"
 simplejson = "^3.17.6"
@@ -109,7 +109,6 @@ numpy = [
     { version = ">=1.22", python = ">=3.8" },
 ]
 requests-mock = "^1.10.0"
-sqlalchemy2-stubs = {version = "^0.0.2a32", allow-prereleases = true}
 types-jsonschema = "^4.17.0.6"
 types-python-dateutil = "^2.8.19"
 types-pytz = ">=2022.7.1.2,<2024.0.0.0"
@@ -133,9 +132,6 @@ exclude = ".*simpleeval.*"
 
 [tool.pytest.ini_options]
 addopts = '-vvv --ignore=singer_sdk/helpers/_simpleeval.py -m "not external"'
-filterwarnings = [
-    "error::sqlalchemy.exc.RemovedIn20Warning",
-]
 markers = [
     "external: Tests relying on external resources",
     "windows: Tests that only run on Windows",
@@ -191,9 +187,6 @@ fail_under = 82
 [tool.mypy]
 exclude = "tests"
 files = "singer_sdk"
-plugins = [
-    "sqlalchemy.ext.mypy.plugin",
-]
 python_version = "3.8"
 warn_unused_configs = true
 warn_unused_ignores = true
@@ -216,11 +209,13 @@ requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry.scripts]
-pytest11 = { callable = "singer_sdk:testing.pytest_plugin", extras = ["testing"] }
+pytest11 = { reference = "singer_sdk:testing.pytest_plugin", extras = ["testing"], type = "console" }
 
 [tool.ruff]
 exclude = [
     "cookiecutter/*",
+    "singer_sdk/helpers/_simpleeval.py",
+    "tests/core/test_simpleeval.py",
 ]
 ignore = [
     "ANN101",  # Missing type annotation for `self` in method

pyproject.toml

@@ -197,7 +197,7 @@ def get_sqlalchemy_url(self, config: t.Mapping[str, t.Any]) -> str:
     @staticmethod
     def to_jsonschema_type(
         sql_type: (
-            str
+            str  # noqa: ANN401
             | sqlalchemy.types.TypeEngine
             | type[sqlalchemy.types.TypeEngine]
             | t.Any

singer_sdk/connectors/sql.py

@@ -12,3 +12,9 @@
     from typing import Protocol, final  # noqa: ICN003
 
 __all__ = ["metadata", "final", "Protocol"]
+if sys.version_info < (3, 9):
+    import importlib_resources as resources
+else:
+    from importlib import resources
+
+__all__ = ["metadata", "final", "resources", "Protocol"]

singer_sdk/helpers/_compat.py

@@ -252,15 +252,15 @@ def _flatten_schema(  # noqa: C901
             else:
                 items.append((new_key, v))
         elif len(v.values()) > 0:
-            if list(v.values())[0][0]["type"] == "string":
-                list(v.values())[0][0]["type"] = ["null", "string"]
-                items.append((new_key, list(v.values())[0][0]))
-            elif list(v.values())[0][0]["type"] == "array":
-                list(v.values())[0][0]["type"] = ["null", "array"]
-                items.append((new_key, list(v.values())[0][0]))
-            elif list(v.values())[0][0]["type"] == "object":
-                list(v.values())[0][0]["type"] = ["null", "object"]
-                items.append((new_key, list(v.values())[0][0]))
+            if next(iter(v.values()))[0]["type"] == "string":
+                next(iter(v.values()))[0]["type"] = ["null", "string"]
+                items.append((new_key, next(iter(v.values()))[0]))
+            elif next(iter(v.values()))[0]["type"] == "array":
+                next(iter(v.values()))[0]["type"] = ["null", "array"]
+                items.append((new_key, next(iter(v.values()))[0]))
+            elif next(iter(v.values()))[0]["type"] == "object":
+                next(iter(v.values()))[0]["type"] = ["null", "object"]
+                items.append((new_key, next(iter(v.values()))[0]))
 
     # Sort and check for duplicates
     def _key_func(item):