Skip to content
Run CI Tests

refactor: optimize query packet parameter conversion #190

Sign in to view logs

refactor: optimize query packet parameter conversion

refactor: optimize query packet parameter conversion #190

Workflow file for this run

.github/workflows/ci.yml at f803cc8
---
name: Run CI Tests
on:
push:
branches: ['1.1', '2.0', 'sqlalchemy']
pull_request:
workflow_dispatch:
inputs:
test_pypi:
description: 'Publish to Test PyPI'
required: false
default: true
type: boolean
schedule:
# Run weekly on Sundays at 2 AM UTC
- cron: '0 2 * * 0'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: false
env:
MYSQL_TEST_HOST: mariadb.example.com
MYSQL_TEST_PORT: 3306
MYSQL_TEST_USER: root
MYSQL_TEST_PASSWD: "heyPassw-!*20oRd"
MYSQL_TEST_DB: testp
jobs:
setup:
runs-on: ubuntu-latest
if: |
github.event_name != 'schedule' ||
github.ref == 'refs/heads/3.4' ||
github.ref == 'refs/heads/3.1' ||
github.ref == 'refs/heads/3.2'
outputs:
matrix: ${{ steps.set-matrix.outputs.final-matrix }}
steps:
- uses: actions/checkout@v4
- id: set-matrix
name: build matrix
uses: mariadb-corporation/connector-ci-build-matrix@main
with:
additional-matrix: '[{"name": "MariaDB 11.4", "os": "ubuntu-latest", "db-type": "community", "db-tag": "11.4", "python": "3.10"},{"name": "MariaDB 11.4", "os": "ubuntu-latest", "db-type": "community", "db-tag": "11.4", "python": "3.11"},{"name": "MariaDB 11.4", "os": "ubuntu-latest", "db-type": "community", "db-tag": "11.4", "python": "3.12"},{"name": "MariaDB 11.4", "os": "windows-latest", "db-type": "community", "db-tag": "11.4", "python": "3.12"}, {"name": "MariaDB 11.4", "os": "ubuntu-latest", "db-type": "community", "db-tag": "11.4", "python": "pypy3.11", "continue-on-error": true}]'
ci-pure-python:
name: ${{ matrix.name }} (Pure Python) ${{ matrix.python != '' && format(' - python {0}', matrix.python) || '' }}
needs: setup
timeout-minutes: 50
strategy:
matrix: ${{ fromJSON(needs.setup.outputs.matrix) }}
runs-on: ${{ matrix.os }}
continue-on-error: ${{ matrix.continue-on-error || false }}
steps:
- uses: actions/checkout@v4
- name: Setup Test Environment
id: setup-env
uses: mariadb-corporation/connector-ci-setup@master
with:
node-version: ${{ matrix.node }}
db-type: ${{ matrix.db-type }}
db-tag: ${{ matrix.db-tag }}
test-db-password: ${{ env.MYSQL_TEST_PASSWD }}
test-db-database: ${{ env.MYSQL_TEST_DB }}
test-db-port: ${{ env.MYSQL_TEST_PORT }}
additional-conf: ${{ matrix.additional-conf || '--max_allowed_packet=40M' }}
registry-user: ${{ matrix.db-type == 'enterprise' && secrets.ENTERPRISE_USER || (secrets.DOCKER_PWD != '' && 'mariadbtest' || '') }}
registry-password: ${{ matrix.db-type == 'enterprise' && secrets.ENTERPRISE_TOKEN || secrets.DOCKER_PWD }}
os: ${{ matrix.os }}
init-script-folder: ${{ format('{0}/.github/workflows/init', github.workspace) }}
- uses: actions/setup-python@v5
id: setup-python
with:
python-version: ${{ matrix.python || '3.13' }}
allow-prereleases: true
- name: Run SQLAlchemy tests
if: ${{ startsWith(matrix.python, '3.13') }}
shell: bash
run: |
python --version
python -m pip install .
cd mariadb-c && python -m pip install . && cd ..
cd mariadb-pool && python -m pip install . && cd ..
python -m pip install pytest typing_extensions
cd ${{ github.workspace }}/sqlalchemy
python -m pytest --dburi=mysql+mariadbconnector://$TEST_DB_USER:$TEST_DB_PASSWORD@$TEST_DB_HOST:$TEST_DB_PORT/$TEST_DB --backend-only
env:
TEST_DB_USER: ${{ env.MYSQL_TEST_USER }}
TEST_DB_HOST: ${{ env.MYSQL_TEST_HOST }}
TEST_DB_DATABASE: ${{ env.MYSQL_TEST_DB }}
TEST_DB_PORT: ${{ env.MYSQL_TEST_PORT }}
TEST_DB_PASSWORD: ${{ env.MYSQL_TEST_PASSWD }}
TEST_DB: ${{ env.MYSQL_TEST_DB }}
- name: Run test suite (Pure Python)
shell: bash
run: |
python --version
python -m pip install .
cd mariadb-pool && python -m pip install . && cd ..
python -m pip install pytest pytest-cov typing_extensions cryptography
python -m pytest tests -v -s --cov=mariadb --cov=mariadb_pool --cov=mariadb_shared --cov-branch --cov-report=xml
env:
TEST_DB_USER: ${{ env.MYSQL_TEST_USER }}
TEST_DB_HOST: ${{ env.MYSQL_TEST_HOST }}
TEST_DB_DATABASE: ${{ env.MYSQL_TEST_DB }}
TEST_DB_PORT: ${{ env.MYSQL_TEST_PORT }}
TEST_DB_PASSWORD: ${{ env.MYSQL_TEST_PASSWD }}
TEST_DB_SERVER_CERT: ${{ matrix.db-type != 'mysql' && steps.setup-env.outputs.database-type == 'container' && './.github/workflows/certs/ca_server.crt' || '' }}
LOCAL_DB: ${{ steps.setup-env.outputs.database-type }}
PYTHON_VERSION: ${{ steps.setup-python.outputs.python-version }}
MARIADB_PYTHON_CONNECTOR: mariadb
RUN_LONG_TEST: 1
- name: Upload coverage reports to Codecov
uses: codecov/codecov-action@v5
with:
token: ${{ secrets.CODECOV_TOKEN }}
ci-c-extension:
name: ${{ matrix.name }} (C Extension) ${{ matrix.python != '' && format(' - python {0}', matrix.python) || '' }}
needs: setup
timeout-minutes: 50
strategy:
matrix: ${{ fromJSON(needs.setup.outputs.matrix) }}
runs-on: ${{ matrix.os }}
continue-on-error: ${{ matrix.continue-on-error || false }}
steps:
- uses: actions/checkout@v4
- name: Setup Test Environment
id: setup-env
uses: mariadb-corporation/connector-ci-setup@master
with:
node-version: ${{ matrix.node }}
db-type: ${{ matrix.db-type }}
db-tag: ${{ matrix.db-tag }}
test-db-password: ${{ env.MYSQL_TEST_PASSWD }}
test-db-database: ${{ env.MYSQL_TEST_DB }}
test-db-port: ${{ env.MYSQL_TEST_PORT }}
additional-conf: ${{ matrix.additional-conf || '--max_allowed_packet=40M' }}
registry-user: ${{ matrix.db-type == 'enterprise' && secrets.ENTERPRISE_USER || (secrets.DOCKER_PWD != '' && 'mariadbtest' || '') }}
registry-password: ${{ matrix.db-type == 'enterprise' && secrets.ENTERPRISE_TOKEN || secrets.DOCKER_PWD }}
os: ${{ matrix.os }}
init-script-folder: ${{ format('{0}/.github/workflows/init', github.workspace) }}
- uses: actions/setup-python@v5
id: setup-python
with:
python-version: ${{ matrix.python || '3.13' }}
allow-prereleases: true
- name: Clone C/C
uses: GuillaumeFalourd/[email protected]
with:
branch: '3.4'
owner: 'mariadb-corporation'
repository: 'mariadb-connector-c'
- name: Clone SQLAlchemy
if: ${{ startsWith(matrix.python, '3.13') }}
uses: GuillaumeFalourd/[email protected]
with:
branch: main
owner: 'sqlalchemy'
repository: 'sqlalchemy'
- name: c/c make ubuntu
if: ${{ startsWith(matrix.os, 'ubuntu') }}
run: |
cd ${{ github.workspace }}/mariadb-connector-c
cmake . -DCMAKE_BUILD_TYPE=Release -DWITH_EXTERNAL_ZLIB=On -DCMAKE_INSTALL_PREFIX=/usr
make -j4
sudo make install
echo "MARIADB_PLUGIN_DIR=`mariadb_config --plugindir`" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib/mariadb" >> $GITHUB_ENV
- name: c/c make macos
if: ${{ startsWith(matrix.os, 'mac') }}
run: |
cd ${{ github.workspace }}/mariadb-connector-c
cmake . -DCMAKE_BUILD_TYPE=Release -DWITH_EXTERNAL_ZLIB=On
make -j4
sudo make install
ls -lrt /usr/local/lib/mariadb/plugin
echo "MARIADB_PLUGIN_DIR=`mariadb_config --plugindir`" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib/mariadb" >> $GITHUB_ENV
echo "DYLD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib/mariadb:$DYLD_LIBRARY_PATH" >> $GITHUB_ENV
echo "DYLD_FALLBACK_LIBRARY_PATH=/usr/local/lib:/usr/local/lib/mariadb" >> $GITHUB_ENV
- name: c/c make windows
if: ${{ startsWith(matrix.os, 'windows') }}
shell: powershell
run: |
cd ${{ github.workspace }}/mariadb-connector-c
$MARIADB_CC_INSTALL_DIR = "$env:USERPROFILE/conc"
cmake . -DCMAKE_BUILD_TYPE=RelWithDebInfo -DWITH_CURL=ON -DCMAKE_INSTALL_PREFIX="$MARIADB_CC_INSTALL_DIR"
cmake --build . --config RelWithDebInfo
cmake --install . --config RelWithDebInfo
echo "MARIADB_CC_INSTALL_DIR=$MARIADB_CC_INSTALL_DIR" >> $env:GITHUB_ENV
echo "MARIADB_PLUGIN_DIR=$MARIADB_CC_INSTALL_DIR/lib/mariadb/plugin" >> $env:GITHUB_ENV
- name: Run SQLAlchemy tests
if: ${{ startsWith(matrix.python, '3.13') }}
shell: bash
run: |
python --version
python -m pip install .
cd mariadb-c && python -m pip install . && cd ..
cd mariadb-pool && python -m pip install . && cd ..
python -m pip install pytest typing_extensions
cd ${{ github.workspace }}/sqlalchemy
python -m pytest --dburi=mysql+mariadbconnector://$TEST_DB_USER:$TEST_DB_PASSWORD@$TEST_DB_HOST:$TEST_DB_PORT/$TEST_DB --backend-only
env:
TEST_DB_USER: ${{ env.MYSQL_TEST_USER }}
TEST_DB_HOST: ${{ env.MYSQL_TEST_HOST }}
TEST_DB_DATABASE: ${{ env.MYSQL_TEST_DB }}
TEST_DB_PORT: ${{ env.MYSQL_TEST_PORT }}
TEST_DB_PASSWORD: ${{ env.MYSQL_TEST_PASSWD }}
TEST_DB: ${{ env.MYSQL_TEST_DB }}
- name: Enable core dumps (Linux)
if: ${{ startsWith(matrix.os, 'ubuntu') }}
shell: bash
run: |
# Enable core dumps
ulimit -c unlimited
echo "Core dump size limit: $(ulimit -c)"
# Set core dump pattern
sudo sysctl -w kernel.core_pattern=core.%e.%p
echo "Core pattern: $(cat /proc/sys/kernel/core_pattern)"
# Install debugging tools
sudo apt-get update && sudo apt-get install -y gdb python3-dbg || true
- name: Run test suite
shell: bash
run: |
python --version
python -m pip install pytest pytest-cov typing_extensions
# Install with dependencies (cryptography, packaging, etc.)
python -m pip install .[test]
cd mariadb-c && python -m pip install . && cd ..
cd mariadb-pool && python -m pip install . && cd ..
echo "=== Checking mariadb_c installation ==="
python -c "import mariadb; import mariadb_c; print(f'mariadb_c location: {mariadb_c.__file__}')"
echo "=== Checking which implementation is being used ==="
python -c "import mariadb; print(f'Implementation: {mariadb.__impl__}'); print(f'SyncConnection: {mariadb.SyncConnection}')"
echo "=== Environment variable check ==="
python -c "import os; print(f'MARIADB_PYTHON_CONNECTOR={os.environ.get(\"MARIADB_PYTHON_CONNECTOR\", \"NOT SET\")}')"
# Enable core dumps for this process
ulimit -c unlimited 2>/dev/null || true
python -m pytest tests -v -s --cov=mariadb --cov=mariadb_c --cov=mariadb_pool --cov=mariadb_shared --cov-branch --cov-report=xml --cov-report=term
echo "=== Coverage report for mariadb_c ==="
python -m coverage report --include="*/mariadb_c/*" || echo "No mariadb_c coverage found"
env:
TEST_DB_USER: ${{ env.MYSQL_TEST_USER }}
TEST_DB_HOST: ${{ env.MYSQL_TEST_HOST }}
TEST_DB_DATABASE: ${{ env.MYSQL_TEST_DB }}
TEST_DB_PORT: ${{ env.MYSQL_TEST_PORT }}
TEST_DB_PASSWORD: ${{ env.MYSQL_TEST_PASSWD }}
TEST_DB_SERVER_CERT: ${{ matrix.db-type != 'mysql' && steps.setup-env.outputs.database-type == 'container' && './.github/workflows/certs/ca_server.crt' || '' }}
LOCAL_DB: ${{ steps.setup-env.outputs.database-type }}
PYTHON_VERSION: ${{ steps.setup-python.outputs.python-version }}
MARIADB_PYTHON_CONNECTOR: mariadb_c
RUN_LONG_TEST: 1
- name: Analyze core dumps (Linux)
if: ${{ always() && startsWith(matrix.os, 'ubuntu') }}
shell: bash
run: |
echo "=== Searching for core dumps ==="
find . -name "core.*" -o -name "core" | while read core_file; do
echo "Found core dump: $core_file"
echo "=== Core dump info ==="
file "$core_file"
echo "=== Backtrace from core dump ==="
gdb -batch -ex "thread apply all bt" -ex "quit" python3 "$core_file" 2>&1 || true
echo "=== Python-specific backtrace ==="
gdb -batch -ex "py-bt" -ex "quit" python3 "$core_file" 2>&1 || true
done
# Also check system core dumps
if [ -f /var/crash/* ]; then
echo "=== System crash dumps ==="
ls -lh /var/crash/
fi
- name: Upload coverage reports to Codecov
uses: codecov/codecov-action@v5
with:
token: ${{ secrets.CODECOV_TOKEN }}
build-source-packages:
name: Build Source Packages
runs-on: ubuntu-latest
if: github.event_name == 'workflow_dispatch' || github.event_name == 'release'
needs: [ci-pure-python, ci-c-extension]
steps:
- uses: actions/checkout@v4
with:
submodules: recursive
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install build dependencies
run: |
python -m pip install --upgrade pip
pip install build setuptools wheel
- name: Build mariadb (pure Python) package
run: |
python -m build --sdist --wheel --outdir dist/
env:
MARIADB_IMPL: pure
- name: Build mariadb-c (C extension) package
run: |
cd mariadb-c
python -m build --sdist --outdir ../dist/
env:
MARIADB_IMPL: c
- name: Build mariadb-pool package
run: |
cd mariadb-pool
python -m build --sdist --wheel --outdir ../dist/
- name: List built packages
run: |
echo "Built packages:"
ls -lh dist/
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: source-packages
path: dist/*