feat: modernize dependencies and expand Python support (#1778)

* fix: use minify-html instead of htmlmin

* feat: updating imghdr to use filetype instead

* fix: removing old imghdr tests

* feat: updating numpy to be <3.0.0

* feat: revert back to numpy <2.2

* fix: kwargs difference from minify-html

* fix: change html search param for valid output

* fix: updating html checks due to minify-html

* feat: adding python 3.13 to tests

* style: removing unused filetype import

* fix: updating required python <3.14

* style: make sure precommit fmt passes

* fix: updating meteorite url

* fix: updating pyspark version for python 3.13

* fix: loosen dataset schema HTML check for minified output

* style: correct fmt

* style: conform to repo format

* fix: updating url search for minify-html

* fix: disable ansi for spark

---------

Co-authored-by: Fabiana <[email protected]>

Author: WilliamJudge94

.github/workflows/tests.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-22.04 ]
-        python-version: ["3.9", "3.10", "3.11", "3.12" ]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13" ]
         pandas: [ "pandas>1.1" ]
         numpy: [ "numpy>=1.21" ]
     runs-on: ${{ matrix.os }}

pyproject.toml

@@ -11,7 +11,7 @@ package_name = "ydata-profiling"
 
 [project]
 name = "ydata-profiling"
-requires-python = ">=3.7,<3.13"
+requires-python = ">=3.7,<3.14"
 authors = [
     {name = "YData Labs Inc", email = "[email protected]"}
 ]
@@ -51,7 +51,8 @@ dependencies = [
     "numpy>=1.16.0,<2.2",
     # Could be optional
     # Related to HTML report
-    "htmlmin==0.1.12",
+    "minify-html>=0.15.0",
+    "filetype>=1.0.0",
     # Correlations
     "phik>=0.11.1,<0.13",
     # Examples
@@ -108,7 +109,7 @@ notebook = [
 # note that if you are using pyspark 2.3 or 2.4 and pyarrow >= 0.15, you might need to
 # set ARROW_PRE_0_15_IPC_FORMAT=1 in your conf/spark-env.sh for toPandas functions to work properly
 spark = [
-    "pyspark>=3.0",
+    "pyspark>=4.0",
     "pyarrow>=4.0.0",
     "pandas>1.1",
     "numpy>=1.16.0",

src/ydata_profiling/model/pandas/describe_image_pandas.py

@@ -1,8 +1,8 @@
-import imghdr
 from functools import partial
 from pathlib import Path
 from typing import Optional, Tuple, Union
 
+import filetype
 import imagehash
 import pandas as pd
 from PIL import ExifTags, Image
@@ -12,7 +12,6 @@
     describe_image_1d,
     named_aggregate_summary,
 )
-from ydata_profiling.utils.imghdr_patch import *  # noqa: F401,F403
 
 
 def open_image(path: Path) -> Optional[Image.Image]:
@@ -119,7 +118,8 @@ def extract_exif(image: Image) -> dict:
 
 
 def path_is_image(p: Path) -> bool:
-    return imghdr.what(p) is not None
+    guess = filetype.guess(str(p))
+    return guess is not None and guess.mime.startswith("image/")
 
 
 def count_duplicate_hashes(image_descriptions: dict) -> int:

src/ydata_profiling/model/typeset.py

@@ -1,11 +1,11 @@
 import datetime
-import imghdr
 import os
 import warnings
 from functools import partial, wraps
 from typing import Callable, Sequence, Set
 from urllib.parse import urlparse
 
+import filetype
 import pandas as pd
 import visions
 from multimethod import multimethod
@@ -295,7 +295,11 @@ def get_relations() -> Sequence[TypeRelation]:
         @multimethod
         @series_handle_nulls
         def contains_op(series: pd.Series, state: dict) -> bool:
-            return all(imghdr.what(p) for p in series)
+            return all(
+                filetype.guess(str(p))
+                and filetype.guess(str(p)).mime.startswith("image/")
+                for p in series
+            )
 
     class TimeSeries(visions.VisionsBaseType):
         @staticmethod

src/ydata_profiling/profile_report.py

@@ -427,9 +427,9 @@ def _render_html(self) -> str:
             )
 
             if self.config.html.minify_html:
-                from htmlmin.main import minify
+                import minify_html
 
-                html = minify(html, remove_all_empty_space=True, remove_comments=True)
+                html = minify_html.minify(html, keep_comments=False)
             pbar.update()
         return html
 

src/ydata_profiling/utils/common.py

@@ -8,8 +8,7 @@
 import zipfile
 from datetime import datetime, timedelta
 
-# Monkeypatch bug in imagehdr
-from imghdr import tests
+# Image type detection
 from pathlib import Path
 from typing import Mapping
 
@@ -64,35 +63,6 @@ def extract_zip(outfile, effective_path):
         raise ValueError("Bad zip file") from e
 
 
-def test_jpeg1(h, f):
-    """JPEG data in JFIF format"""
-    if b"JFIF" in h[:23]:
-        return "jpeg"
-
-
-JPEG_MARK = (
-    b"\xff\xd8\xff\xdb\x00C\x00\x08\x06\x06"
-    b"\x07\x06\x05\x08\x07\x07\x07\t\t\x08\n\x0c\x14\r\x0c\x0b\x0b\x0c\x19\x12\x13\x0f"
-)
-
-
-def test_jpeg2(h, f):
-    """JPEG with small header"""
-    if len(h) >= 32 and h[5] == 67 and h[:32] == JPEG_MARK:
-        return "jpeg"
-
-
-def test_jpeg3(h, f):
-    """JPEG data in JFIF or Exif format"""
-    if h[6:10] in (b"JFIF", b"Exif") or h[:2] == b"\xff\xd8":
-        return "jpeg"
-
-
-tests.append(test_jpeg1)
-tests.append(test_jpeg2)
-tests.append(test_jpeg3)
-
-
 def convert_timestamp_to_datetime(timestamp: int) -> datetime:
     if timestamp >= 0:
         return datetime.fromtimestamp(timestamp)

src/ydata_profiling/utils/imghdr_patch.py

@@ -83,7 +83,12 @@ def spark_context():
     if not has_spark:
         pytest.skip("Skipping Spark tests because PySpark is not installed.")
 
-    conf = SparkConf().setAppName("pytest-pyspark-tests").setMaster("local[*]")
+    conf = (
+        SparkConf()
+        .setAppName("pytest-pyspark-tests")
+        .setMaster("local[*]")
+        .set("spark.sql.ansi.enabled", "false")
+    )
 
     # Check if SparkContext exists before creating a new one
     if SparkContext._active_spark_context:

tests/conftest.py

@@ -19,4 +19,4 @@ def test_issue147(get_data_file):
     )
     html = report.to_html()
     assert type(html) == str
-    assert "Dataset statistics</p>" in html
+    assert "Dataset statistics" in html

tests/issues/test_issue147.py

@@ -32,7 +32,7 @@ def test_issue_169_column(issue_169_data):
     )
     html = report.to_html()
     assert type(html) == str
-    assert "Dataset statistics</p>" in html
+    assert "Dataset statistics" in html
 
 
 def test_issue_169_index(issue_169_data):
@@ -45,4 +45,4 @@ def test_issue_169_index(issue_169_data):
     )
     html = report.to_html()
     assert type(html) == str
-    assert "Dataset statistics</p>" in html
+    assert "Dataset statistics" in html