Skip to content

Commit f8a9051

Browse files
committed
optimize benchmark reliability with warmup runs and reduced cursor overhead
1 parent 5cf387f commit f8a9051

File tree

5 files changed

+42
-13
lines changed

5 files changed

+42
-13
lines changed

benchmarks/conftest.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,11 @@ def setup_database():
136136
# Create test100 table (100 integer columns)
137137
cursor.execute("DROP TABLE IF EXISTS test100")
138138
cols = ",".join([f"i{i} int" for i in range(1, 101)])
139-
cursor.execute(f"CREATE TABLE test100 ({cols})")
139+
table_sql = f"CREATE TABLE test100 ({cols})"
140+
try:
141+
cursor.execute(table_sql + " ENGINE = MEMORY")
142+
except:
143+
cursor.execute(table_sql)
140144
vals = ",".join([str(i) for i in range(1, 101)])
141145
cursor.execute(f"INSERT INTO test100 VALUES ({vals})")
142146

benchmarks/run_benchmarks.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,8 @@ def run_pytest_benchmark(benchmark_file=None, driver=None, output_json=None):
5353
cmd = [sys.executable, '-m', 'pytest', '-v']
5454

5555
# Add benchmark options
56-
cmd.extend(['--benchmark-only', '--benchmark-warmup=on'])
57-
cmd.extend(['--benchmark-min-rounds=100', '--benchmark-warmup-iterations=100'])
56+
cmd.extend(['--benchmark-only', '--benchmark-warmup=on', '--benchmark-disable-gc'])
57+
cmd.extend(['--benchmark-min-rounds=2000', '--benchmark-warmup-iterations=200'])
5858

5959
if benchmark_file:
6060
cmd.append(benchmark_file)

benchmarks/test_bench_select_1.py

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,17 @@
1515
def test_select_1(benchmark, connection, driver_name):
1616
"""Benchmark SELECT 1 query execution."""
1717

18+
# Warmup: ensure connection is established and cached
19+
cursor = connection.cursor()
20+
for _ in range(10):
21+
cursor.execute("SELECT 1")
22+
cursor.fetchone()
23+
1824
def select_1():
19-
cursor = connection.cursor()
2025
cursor.execute("SELECT 1")
2126
result = cursor.fetchone()
22-
cursor.close()
2327
return result[0]
2428

2529
result = benchmark(select_1)
30+
cursor.close()
2631
print(f"\n{driver_name}: {result}")

benchmarks/test_bench_select_1000_rows.py

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,19 @@
1818
def test_select_1000_rows_text(benchmark, connection, driver_name):
1919
"""Benchmark SELECT 1000 rows using text protocol (regular execute)."""
2020

21+
# Warmup: ensure seq table is in cache
22+
cursor = connection.cursor()
23+
for _ in range(5):
24+
cursor.execute(SQL)
25+
cursor.fetchall()
26+
2127
def select_1000_rows():
22-
cursor = connection.cursor()
2328
cursor.execute(SQL)
2429
rows = cursor.fetchall()
25-
cursor.close()
2630
return len(rows)
2731

2832
result = benchmark(select_1000_rows)
33+
cursor.close()
2934
print(f"\n{driver_name} (text): {result}")
3035

3136

@@ -36,12 +41,17 @@ def test_select_1000_rows_binary(benchmark, connection, driver_name):
3641
if driver_name in ['pymysql', 'mysql_connector']:
3742
pytest.skip(f"{driver_name} doesn't support binary protocol")
3843

44+
# Warmup: ensure seq table is in cache
45+
cursor = connection.cursor(binary=True)
46+
for _ in range(5):
47+
cursor.execute(SQL)
48+
cursor.fetchall()
49+
3950
def select_1000_rows():
40-
cursor = connection.cursor(binary=True)
4151
cursor.execute(SQL)
4252
rows = cursor.fetchall()
43-
cursor.close()
4453
return len(rows)
4554

4655
result = benchmark(select_1000_rows)
56+
cursor.close()
4757
print(f"\n{driver_name} (binary): {result}")

benchmarks/test_bench_select_100_cols.py

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,16 +16,21 @@
1616
def test_select_100_cols_text(benchmark, connection, driver_name):
1717
"""Benchmark SELECT 100 columns using text protocol."""
1818

19+
# Warmup: ensure table is in database cache
20+
cursor = connection.cursor()
21+
for _ in range(10):
22+
cursor.execute("SELECT * FROM test100")
23+
cursor.fetchone()
24+
1925
def select_100_cols():
20-
cursor = connection.cursor()
2126
cursor.execute("SELECT * FROM test100")
2227
row = cursor.fetchone()
23-
cursor.close()
2428
# Consume all values to ensure fair comparison
2529
total = sum(row)
2630
return total
2731

2832
result = benchmark(select_100_cols)
33+
cursor.close()
2934
print(f"\n{driver_name} (text): {result}")
3035

3136

@@ -37,14 +42,19 @@ def test_select_100_cols_binary(benchmark, connection, driver_name):
3742
if driver_name in ['pymysql', 'mysql_connector']:
3843
pytest.skip(f"{driver_name} doesn't support binary protocol")
3944

45+
# Warmup: ensure table is in database cache
46+
cursor = connection.cursor(binary=True)
47+
for _ in range(10):
48+
cursor.execute("SELECT * FROM test100")
49+
cursor.fetchone()
50+
4051
def select_100_cols():
41-
cursor = connection.cursor(binary=True)
4252
cursor.execute("SELECT * FROM test100")
4353
row = cursor.fetchone()
44-
cursor.close()
4554
# Consume all values to ensure fair comparison
4655
total = sum(row)
4756
return total
4857

4958
result = benchmark(select_100_cols)
59+
cursor.close()
5060
print(f"\n{driver_name} (binary): {result}")

0 commit comments

Comments
 (0)