optimize benchmark reliability with warmup runs and reduced cursor overhead

Author: rusher

benchmarks/conftest.py

@@ -136,7 +136,11 @@ def setup_database():
         # Create test100 table (100 integer columns)
         cursor.execute("DROP TABLE IF EXISTS test100")
         cols = ",".join([f"i{i} int" for i in range(1, 101)])
-        cursor.execute(f"CREATE TABLE test100 ({cols})")
+        table_sql = f"CREATE TABLE test100 ({cols})"
+        try:
+            cursor.execute(table_sql + " ENGINE = MEMORY")
+        except:
+            cursor.execute(table_sql)
         vals = ",".join([str(i) for i in range(1, 101)])
         cursor.execute(f"INSERT INTO test100 VALUES ({vals})")
 

benchmarks/run_benchmarks.py

@@ -53,8 +53,8 @@ def run_pytest_benchmark(benchmark_file=None, driver=None, output_json=None):
     cmd = [sys.executable, '-m', 'pytest', '-v']
 
     # Add benchmark options
-    cmd.extend(['--benchmark-only', '--benchmark-warmup=on'])
-    cmd.extend(['--benchmark-min-rounds=100', '--benchmark-warmup-iterations=100'])
+    cmd.extend(['--benchmark-only', '--benchmark-warmup=on', '--benchmark-disable-gc'])
+    cmd.extend(['--benchmark-min-rounds=2000', '--benchmark-warmup-iterations=200'])
 
     if benchmark_file:
         cmd.append(benchmark_file)

benchmarks/test_bench_select_1.py

@@ -15,12 +15,17 @@
 def test_select_1(benchmark, connection, driver_name):
     """Benchmark SELECT 1 query execution."""
 
+    # Warmup: ensure connection is established and cached
+    cursor = connection.cursor()
+    for _ in range(10):
+        cursor.execute("SELECT 1")
+        cursor.fetchone()
+    
     def select_1():
-        cursor = connection.cursor()
         cursor.execute("SELECT 1")
         result = cursor.fetchone()
-        cursor.close()
         return result[0]
 
     result = benchmark(select_1)
+    cursor.close()
     print(f"\n{driver_name}: {result}")

benchmarks/test_bench_select_1000_rows.py

@@ -18,14 +18,19 @@
 def test_select_1000_rows_text(benchmark, connection, driver_name):
     """Benchmark SELECT 1000 rows using text protocol (regular execute)."""
 
+    # Warmup: ensure seq table is in cache
+    cursor = connection.cursor()
+    for _ in range(5):
+        cursor.execute(SQL)
+        cursor.fetchall()
+    
     def select_1000_rows():
-        cursor = connection.cursor()
         cursor.execute(SQL)
         rows = cursor.fetchall()
-        cursor.close()
         return len(rows)
 
     result = benchmark(select_1000_rows)
+    cursor.close()
     print(f"\n{driver_name} (text): {result}")
 
 
@@ -36,12 +41,17 @@ def test_select_1000_rows_binary(benchmark, connection, driver_name):
     if driver_name in ['pymysql', 'mysql_connector']:
         pytest.skip(f"{driver_name} doesn't support binary protocol")
 
+    # Warmup: ensure seq table is in cache
+    cursor = connection.cursor(binary=True)
+    for _ in range(5):
+        cursor.execute(SQL)
+        cursor.fetchall()
+    
     def select_1000_rows():
-        cursor = connection.cursor(binary=True)
         cursor.execute(SQL)
         rows = cursor.fetchall()
-        cursor.close()
         return len(rows)
 
     result = benchmark(select_1000_rows)
+    cursor.close()
     print(f"\n{driver_name} (binary): {result}")

benchmarks/test_bench_select_100_cols.py

@@ -16,16 +16,21 @@
 def test_select_100_cols_text(benchmark, connection, driver_name):
     """Benchmark SELECT 100 columns using text protocol."""
 
+    # Warmup: ensure table is in database cache
+    cursor = connection.cursor()
+    for _ in range(10):
+        cursor.execute("SELECT * FROM test100")
+        cursor.fetchone()
+    
     def select_100_cols():
-        cursor = connection.cursor()
         cursor.execute("SELECT * FROM test100")
         row = cursor.fetchone()
-        cursor.close()
         # Consume all values to ensure fair comparison
         total = sum(row)
         return total
 
     result = benchmark(select_100_cols)
+    cursor.close()
     print(f"\n{driver_name} (text): {result}")
 
 
@@ -37,14 +42,19 @@ def test_select_100_cols_binary(benchmark, connection, driver_name):
     if driver_name in ['pymysql', 'mysql_connector']:
         pytest.skip(f"{driver_name} doesn't support binary protocol")
 
+    # Warmup: ensure table is in database cache
+    cursor = connection.cursor(binary=True)
+    for _ in range(10):
+        cursor.execute("SELECT * FROM test100")
+        cursor.fetchone()
+    
     def select_100_cols():
-        cursor = connection.cursor(binary=True)
         cursor.execute("SELECT * FROM test100")
         row = cursor.fetchone()
-        cursor.close()
         # Consume all values to ensure fair comparison
         total = sum(row)
         return total
 
     result = benchmark(select_100_cols)
+    cursor.close()
     print(f"\n{driver_name} (binary): {result}")