chore: update pyspark matrix tests (#1779)

* chore: update pyspark matrix tests

* chore: update pyspark dependencies install

* chore: remove strict cast validations

* fix(linting): code formatting

---------

Co-authored-by: Azory YData Bot <[email protected]>

Author: fabclmnt

.github/workflows/tests.yml

@@ -126,22 +126,26 @@ jobs:
     continue-on-error: false
     strategy:
       matrix:
-        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
-        pyspark-version: [ "3.4" , "3.5" ]
+        include:
+          # Legacy line (Spark 3.5.x)
+          - { python-version: "3.10", pyspark-version: "3.5" }
+          - { python-version: "3.11", pyspark-version: "3.5" }
+          # Current line (Spark 4.0.x)
+          - { python-version: "3.11", pyspark-version: "4.0" }
+          - { python-version: "3.10", pyspark-version: "4.0" }
+          - { python-version: "3.12", pyspark-version: "4.0" }
 
     name: Tests Spark | Python ${{ matrix.python-version }} | PySpark ${{ matrix.pyspark-version }}
 
     steps:
       - name: Checkout Code
         uses: actions/checkout@v4
 
-      - name: Install Java (OpenJDK 11)
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y openjdk-11-jdk
-          echo "JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64" >> $GITHUB_ENV
-          echo "PATH=$JAVA_HOME/bin:$PATH" >> $GITHUB_ENV
-          java -version
+      - name: Setup Java 17 (Temurin)
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: '17'
 
       - name: Setup Python
         uses: actions/setup-python@v5
@@ -159,10 +163,15 @@ jobs:
 
       - name: Install Dependencies
         run: |
-          python -m pip install --upgrade pip setuptools wheel
-          pip install pyarrow>4.0.0 pyspark=="${{ matrix.pyspark-version }}" --no-cache-dir
-          echo "ARROW_PRE_0_15_IPC_FORMAT=1" >> $GITHUB_ENV
+          python -m pip install -U pip setuptools wheel
+          pip install "pyspark~=${{ matrix.pyspark-version }}" "pyarrow>4.0.0" --no-cache-dir
+          pip install ".[test]"
+          # Make PySpark use this Python and bind locally; give it a safe tmp dir
+          echo "PYSPARK_PYTHON=$(which python)" >> $GITHUB_ENV
+          echo "PYSPARK_DRIVER_PYTHON=$(which python)" >> $GITHUB_ENV
           echo "SPARK_LOCAL_IP=127.0.0.1" >> $GITHUB_ENV
+          echo "SPARK_LOCAL_DIRS=$RUNNER_TEMP/spark-tmp" >> $GITHUB_ENV
+          mkdir -p "$RUNNER_TEMP/spark-tmp"
 
       - name: Run Tests
         run: |

examples/features/spark_example.py

@@ -14,7 +14,10 @@
 
 if __name__ == "__main__":
     spark_session = (
-        SparkSession.builder.appName("SparkProfiling").master("local[*]").getOrCreate()
+        SparkSession.builder.appName("SparkProfiling")
+        .master("local[*]")
+        .config("spark.sql.ansi.enabled", "false")
+        .getOrCreate()
     )
 
     print(spark_session.sparkContext.uiWebUrl)  # noqa: T201

tests/conftest.py

@@ -105,7 +105,12 @@ def spark_session(spark_context):
     """
     if not has_spark:
         pytest.skip("Skipping Spark tests because PySpark is not installed.")
-    spark = SparkSession.builder.config(conf=spark_context.getConf()).getOrCreate()
+    spark = (
+        SparkSession.builder.master("local[*]")
+        .appName("pytest")
+        .config("spark.sql.ansi.enabled", "false")  # <-- restore permissive casts
+        .getOrCreate()
+    )
 
     yield spark