diff --git a/README.md b/README.md index 1b93676d..a18d1386 100644 --- a/README.md +++ b/README.md @@ -190,7 +190,9 @@ Define queries and perform advanced searches over your indices, including the co query = VectorQuery( vector=[0.16, -0.34, 0.98, 0.23], vector_field_name="embedding", - num_results=3 + num_results=3, + # Optional: tune search performance with runtime parameters + ef_runtime=100 # HNSW: higher for better recall ) # run the vector search query against the embedding field results = index.query(query) diff --git a/docs/api/query.rst b/docs/api/query.rst index 9d65dc9b..1410bd3a 100644 --- a/docs/api/query.rst +++ b/docs/api/query.rst @@ -20,6 +20,47 @@ VectorQuery :show-inheritance: :exclude-members: add_filter,get_args,highlight,return_field,summarize +.. note:: + **Runtime Parameters for Performance Tuning** + + VectorQuery supports runtime parameters for HNSW and SVS-VAMANA indexes that can be adjusted at query time without rebuilding the index: + + **HNSW Parameters:** + + - ``ef_runtime``: Controls search accuracy (higher = better recall, slower search) + + **SVS-VAMANA Parameters:** + + - ``search_window_size``: Size of search window for KNN searches + - ``use_search_history``: Whether to use search buffer (OFF/ON/AUTO) + - ``search_buffer_capacity``: Tuning parameter for 2-level compression + + Example with HNSW runtime parameters: + + .. code-block:: python + + from redisvl.query import VectorQuery + + query = VectorQuery( + vector=[0.1, 0.2, 0.3], + vector_field_name="embedding", + num_results=10, + ef_runtime=150 # Higher for better recall + ) + + Example with SVS-VAMANA runtime parameters: + + .. code-block:: python + + query = VectorQuery( + vector=[0.1, 0.2, 0.3], + vector_field_name="embedding", + num_results=10, + search_window_size=20, + use_search_history='ON', + search_buffer_capacity=30 + ) + VectorRangeQuery ================ @@ -34,6 +75,36 @@ VectorRangeQuery :show-inheritance: :exclude-members: add_filter,get_args,highlight,return_field,summarize +.. note:: + **Runtime Parameters for Range Queries** + + VectorRangeQuery supports runtime parameters for controlling range search behavior: + + **HNSW & SVS-VAMANA Parameters:** + + - ``epsilon``: Range search approximation factor (default: 0.01) + + **SVS-VAMANA Parameters:** + + - ``search_window_size``: Size of search window + - ``use_search_history``: Whether to use search buffer (OFF/ON/AUTO) + - ``search_buffer_capacity``: Tuning parameter for 2-level compression + + Example: + + .. code-block:: python + + from redisvl.query import VectorRangeQuery + + query = VectorRangeQuery( + vector=[0.1, 0.2, 0.3], + vector_field_name="embedding", + distance_threshold=0.3, + epsilon=0.05, # Approximation factor + search_window_size=20, # SVS-VAMANA only + use_search_history='AUTO' # SVS-VAMANA only + ) + HybridQuery ================ @@ -52,6 +123,29 @@ HybridQuery For index-level stopwords configuration (server-side), see :class:`redisvl.schema.IndexInfo.stopwords`. Using query-time stopwords with index-level ``STOPWORDS 0`` is counterproductive. +.. note:: + **Runtime Parameters for Hybrid Queries** + + **Important:** AggregateHybridQuery uses FT.AGGREGATE commands which do NOT support runtime parameters. + Runtime parameters (``ef_runtime``, ``search_window_size``, ``use_search_history``, ``search_buffer_capacity``) + are only supported with FT.SEARCH commands. + + For runtime parameter support, use :class:`VectorQuery` or :class:`VectorRangeQuery` instead of AggregateHybridQuery. + + Example with VectorQuery (supports runtime parameters): + + .. code-block:: python + + from redisvl.query import VectorQuery + + query = VectorQuery( + vector=[0.1, 0.2, 0.3], + vector_field_name="embedding", + return_fields=["description"], + num_results=10, + ef_runtime=150 # Runtime parameters work with VectorQuery + ) + TextQuery ================ diff --git a/docs/api/schema.rst b/docs/api/schema.rst index c5b8ab68..fefcfe6b 100644 --- a/docs/api/schema.rst +++ b/docs/api/schema.rst @@ -208,11 +208,16 @@ HNSW (Hierarchical Navigable Small World) - Graph-based approximate search with **Performance characteristics:** - - **Search speed**: Very fast approximate search with tunable accuracy + - **Search speed**: Very fast approximate search with tunable accuracy (via ``ef_runtime`` at query time) - **Memory usage**: Higher than compressed SVS-VAMANA but reasonable for most applications - - **Recall quality**: Excellent recall rates (95-99%), often better than other approximate methods + - **Recall quality**: Excellent recall rates (95-99%), tunable via ``ef_runtime`` parameter - **Build time**: Moderate construction time, faster than SVS-VAMANA for smaller datasets + **Runtime parameters** (adjustable at query time without rebuilding index): + + - ``ef_runtime``: Controls search accuracy (higher = better recall, slower search). Default: 10 + - ``epsilon``: Range search approximation factor for VectorRangeQuery. Default: 0.01 + .. autoclass:: HNSWVectorField :members: :show-inheritance: @@ -234,10 +239,10 @@ HNSW (Hierarchical Navigable Small World) - Graph-based approximate search with dims: 768 distance_metric: cosine datatype: float32 - # Balanced settings for good recall and performance - m: 16 - ef_construction: 200 - ef_runtime: 10 + # Index-time parameters (set during index creation) + m: 16 # Graph connectivity + ef_construction: 200 # Build-time accuracy + # Note: ef_runtime can be set at query time via VectorQuery **High-recall configuration:** @@ -250,10 +255,10 @@ HNSW (Hierarchical Navigable Small World) - Graph-based approximate search with dims: 768 distance_metric: cosine datatype: float32 - # Tuned for maximum accuracy + # Index-time parameters tuned for maximum accuracy m: 32 ef_construction: 400 - ef_runtime: 50 + # Note: ef_runtime=50 can be set at query time for higher recall SVS-VAMANA Vector Fields ------------------------ @@ -278,6 +283,13 @@ SVS-VAMANA (Scalable Vector Search with VAMANA graph algorithm) provides fast ap - **vs HNSW**: Better memory efficiency with compression, similar or better recall, Intel-optimized + **Runtime parameters** (adjustable at query time without rebuilding index): + + - ``epsilon``: Range search approximation factor. Default: 0.01 + - ``search_window_size``: Size of search window for KNN searches. Higher = better recall, slower search + - ``use_search_history``: Whether to use search buffer (OFF/ON/AUTO). Default: AUTO + - ``search_buffer_capacity``: Tuning parameter for 2-level compression. Default: search_window_size + **Compression selection guide:** - **No compression**: Best performance, standard memory usage @@ -314,10 +326,10 @@ SVS-VAMANA (Scalable Vector Search with VAMANA graph algorithm) provides fast ap dims: 768 distance_metric: cosine datatype: float32 - # Standard settings for balanced performance + # Index-time parameters (set during index creation) graph_max_degree: 40 construction_window_size: 250 - search_window_size: 20 + # Note: search_window_size and other runtime params can be set at query time **High-performance configuration with compression:** @@ -330,14 +342,14 @@ SVS-VAMANA (Scalable Vector Search with VAMANA graph algorithm) provides fast ap dims: 768 distance_metric: cosine datatype: float32 - # Tuned for better recall + # Index-time parameters tuned for better recall graph_max_degree: 64 construction_window_size: 500 - search_window_size: 40 # Maximum compression with dimensionality reduction compression: LeanVec4x8 reduce: 384 # 50% dimensionality reduction training_threshold: 1000 + # Note: search_window_size=40 can be set at query time for higher recall **Important Notes:** @@ -345,7 +357,7 @@ SVS-VAMANA (Scalable Vector Search with VAMANA graph algorithm) provides fast ap - **Datatype limitations**: SVS-VAMANA only supports `float16` and `float32` datatypes (not `bfloat16` or `float64`). - **Compression compatibility**: The `reduce` parameter is only valid with LeanVec compression types (`LeanVec4x8` or `LeanVec8x8`). - **Platform considerations**: Intel's proprietary LVQ and LeanVec optimizations are not available in Redis Open Source. On non-Intel platforms and Redis Open Source, SVS-VAMANA with compression falls back to basic 8-bit scalar quantization. -- **Performance tip**: Start with default parameters and tune `search_window_size` first for your speed vs accuracy requirements. +- **Performance tip**: Runtime parameters like ``search_window_size``, ``epsilon``, and ``use_search_history`` can be adjusted at query time without rebuilding the index. Start with defaults and tune ``search_window_size`` first for your speed vs accuracy requirements. FLAT Vector Fields ------------------ @@ -487,8 +499,8 @@ Performance Characteristics **Recall Quality:** - FLAT: 100% (exact search) - - HNSW: 95-99% (tunable via ef_runtime) - - SVS-VAMANA: 90-95% (depends on compression) + - HNSW: 95-99% (tunable via ``ef_runtime`` at query time) + - SVS-VAMANA: 90-95% (tunable via ``search_window_size`` at query time, also depends on compression) Migration Considerations ------------------------ @@ -496,7 +508,7 @@ Migration Considerations **From FLAT to HNSW:** - Straightforward migration - Expect slight recall reduction but major speed improvement - - Tune ef_runtime to balance speed vs accuracy + - Tune ``ef_runtime`` at query time to balance speed vs accuracy (no index rebuild needed) **From HNSW to SVS-VAMANA:** - Requires Redis >= 8.2 with RediSearch >= 2.8.10 diff --git a/docs/user_guide/01_getting_started.ipynb b/docs/user_guide/01_getting_started.ipynb index 9f5034c9..13bb7062 100644 --- a/docs/user_guide/01_getting_started.ipynb +++ b/docs/user_guide/01_getting_started.ipynb @@ -290,21 +290,21 @@ "\n", "\n", "Index Information:\n", - "╭──────────────────────┬──────────────────────┬──────────────────────┬──────────────────────┬──────────────────────╮\n", - "│ Index Name │ Storage Type │ Prefixes │ Index Options │ Indexing │\n", - "├──────────────────────┼──────────────────────┼──────────────────────┼──────────────────────┼──────────────────────┤\n", + "\u256d\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u256e\n", + "\u2502 Index Name \u2502 Storage Type \u2502 Prefixes \u2502 Index Options \u2502 Indexing \u2502\n", + "\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n", "| user_simple | HASH | ['user_simple_docs'] | [] | 0 |\n", - "╰──────────────────────┴──────────────────────┴──────────────────────┴──────────────────────┴──────────────────────╯\n", + "\u2570\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u256f\n", "Index Fields:\n", - "╭─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────┬─────────────────╮\n", - "│ Name │ Attribute │ Type │ Field Option │ Option Value │ Field Option │ Option Value │ Field Option │ Option Value │ Field Option │ Option Value │\n", - "├─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┼─────────────────┤\n", - "│ user │ user │ TAG │ SEPARATOR │ , │ │ │ │ │ │ │\n", - "│ credit_score │ credit_score │ TAG │ SEPARATOR │ , │ │ │ │ │ │ │\n", - "│ job │ job │ TEXT │ WEIGHT │ 1 │ │ │ │ │ │ │\n", - "│ age │ age │ NUMERIC │ │ │ │ │ │ │ │ │\n", - "│ user_embedding │ user_embedding │ VECTOR │ algorithm │ FLAT │ data_type │ FLOAT32 │ dim │ 3 │ distance_metric │ COSINE │\n", - "╰─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────┴─────────────────╯\n" + "\u256d\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u256e\n", + "\u2502 Name \u2502 Attribute \u2502 Type \u2502 Field Option \u2502 Option Value \u2502 Field Option \u2502 Option Value \u2502 Field Option \u2502 Option Value \u2502 Field Option \u2502 Option Value \u2502\n", + "\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n", + "\u2502 user \u2502 user \u2502 TAG \u2502 SEPARATOR \u2502 , \u2502 \u2502 \u2502 \u2502 \u2502 \u2502 \u2502\n", + "\u2502 credit_score \u2502 credit_score \u2502 TAG \u2502 SEPARATOR \u2502 , \u2502 \u2502 \u2502 \u2502 \u2502 \u2502 \u2502\n", + "\u2502 job \u2502 job \u2502 TEXT \u2502 WEIGHT \u2502 1 \u2502 \u2502 \u2502 \u2502 \u2502 \u2502 \u2502\n", + "\u2502 age \u2502 age \u2502 NUMERIC \u2502 \u2502 \u2502 \u2502 \u2502 \u2502 \u2502 \u2502 \u2502\n", + "\u2502 user_embedding \u2502 user_embedding \u2502 VECTOR \u2502 algorithm \u2502 FLAT \u2502 data_type \u2502 FLOAT32 \u2502 dim \u2502 3 \u2502 distance_metric \u2502 COSINE \u2502\n", + "\u2570\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u256f\n" ] } ], @@ -447,6 +447,26 @@ ")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Note:** For HNSW and SVS-VAMANA indexes, you can tune search performance using runtime parameters:\n", + "\n", + "```python\n", + "# Example with HNSW runtime parameters\n", + "query = VectorQuery(\n", + " vector=[0.1, 0.1, 0.5],\n", + " vector_field_name=\"user_embedding\",\n", + " return_fields=[\"user\", \"age\", \"job\"],\n", + " num_results=3,\n", + " ef_runtime=50 # Higher for better recall (HNSW only)\n", + ")\n", + "```\n", + "\n", + "See the [SVS-VAMANA guide](09_svs_vamana.ipynb) and [Advanced Queries guide](11_advanced_queries.ipynb) for more details on runtime parameters." + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -658,29 +678,29 @@ "text": [ "\n", "Statistics:\n", - "╭─────────────────────────────┬────────────╮\n", - "│ Stat Key │ Value │\n", - "├─────────────────────────────┼────────────┤\n", - "│ num_docs │ 10 │\n", - "│ num_terms │ 0 │\n", - "│ max_doc_id │ 10 │\n", - "│ num_records │ 50 │\n", - "│ percent_indexed │ 1 │\n", - "│ hash_indexing_failures │ 0 │\n", - "│ number_of_uses │ 2 │\n", - "│ bytes_per_record_avg │ 19.5200004 │\n", - "│ doc_table_size_mb │ 0.00105857 │\n", - "│ inverted_sz_mb │ 9.30786132 │\n", - "│ key_table_size_mb │ 4.70161437 │\n", - "│ offset_bits_per_record_avg │ nan │\n", - "│ offset_vectors_sz_mb │ 0 │\n", - "│ offsets_per_term_avg │ 0 │\n", - "│ records_per_doc_avg │ 5 │\n", - "│ sortable_values_size_mb │ 0 │\n", - "│ total_indexing_time │ 0.16899999 │\n", - "│ total_inverted_index_blocks │ 11 │\n", - "│ vector_index_sz_mb │ 0.23619842 │\n", - "╰─────────────────────────────┴────────────╯\n" + "\u256d\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u256e\n", + "\u2502 Stat Key \u2502 Value \u2502\n", + "\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\n", + "\u2502 num_docs \u2502 10 \u2502\n", + "\u2502 num_terms \u2502 0 \u2502\n", + "\u2502 max_doc_id \u2502 10 \u2502\n", + "\u2502 num_records \u2502 50 \u2502\n", + "\u2502 percent_indexed \u2502 1 \u2502\n", + "\u2502 hash_indexing_failures \u2502 0 \u2502\n", + "\u2502 number_of_uses \u2502 2 \u2502\n", + "\u2502 bytes_per_record_avg \u2502 19.5200004 \u2502\n", + "\u2502 doc_table_size_mb \u2502 0.00105857 \u2502\n", + "\u2502 inverted_sz_mb \u2502 9.30786132 \u2502\n", + "\u2502 key_table_size_mb \u2502 4.70161437 \u2502\n", + "\u2502 offset_bits_per_record_avg \u2502 nan \u2502\n", + "\u2502 offset_vectors_sz_mb \u2502 0 \u2502\n", + "\u2502 offsets_per_term_avg \u2502 0 \u2502\n", + "\u2502 records_per_doc_avg \u2502 5 \u2502\n", + "\u2502 sortable_values_size_mb \u2502 0 \u2502\n", + "\u2502 total_indexing_time \u2502 0.16899999 \u2502\n", + "\u2502 total_inverted_index_blocks \u2502 11 \u2502\n", + "\u2502 vector_index_sz_mb \u2502 0.23619842 \u2502\n", + "\u2570\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u256f\n" ] } ], @@ -780,4 +800,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} +} \ No newline at end of file diff --git a/docs/user_guide/02_hybrid_queries.ipynb b/docs/user_guide/02_hybrid_queries.ipynb index 9414c07d..e7f8d225 100644 --- a/docs/user_guide/02_hybrid_queries.ipynb +++ b/docs/user_guide/02_hybrid_queries.ipynb @@ -219,6 +219,27 @@ "result_print(index.query(v))" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Performance Tip:** For HNSW and SVS-VAMANA indexes, you can add runtime parameters to tune search performance:\n", + "\n", + "```python\n", + "# Example with runtime parameters for better recall\n", + "v = VectorQuery(\n", + " vector=[0.1, 0.1, 0.5],\n", + " vector_field_name=\"user_embedding\",\n", + " return_fields=[\"user\", \"credit_score\", \"age\"],\n", + " filter_expression=t,\n", + " ef_runtime=100, # HNSW: higher for better recall\n", + " search_window_size=40 # SVS-VAMANA: larger window for better recall\n", + ")\n", + "```\n", + "\n", + "These parameters can be adjusted at query time without rebuilding the index. See the [Advanced Queries guide](11_advanced_queries.ipynb) for more details." + ] + }, { "cell_type": "code", "execution_count": 9, @@ -1454,4 +1475,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/docs/user_guide/09_svs_vamana.ipynb b/docs/user_guide/09_svs_vamana.ipynb index 56ec4d88..1fb4c482 100644 --- a/docs/user_guide/09_svs_vamana.ipynb +++ b/docs/user_guide/09_svs_vamana.ipynb @@ -159,7 +159,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "✅ Created SVS-VAMANA index: svs_demo\n", + "\u2705 Created SVS-VAMANA index: svs_demo\n", " Algorithm: svs-vamana\n", " Compression: LeanVec4x8\n", " Dimensions: 1024\n", @@ -193,7 +193,7 @@ "index = SearchIndex.from_dict(schema, redis_url=REDIS_URL)\n", "index.create(overwrite=True)\n", "\n", - "print(f\"✅ Created SVS-VAMANA index: {index.name}\")\n", + "print(f\"\u2705 Created SVS-VAMANA index: {index.name}\")\n", "print(f\" Algorithm: {config['algorithm']}\")\n", "print(f\" Compression: {config['compression']}\")\n", "print(f\" Dimensions: {dims}\")\n", @@ -220,7 +220,7 @@ "output_type": "stream", "text": [ "Creating vectors with 512 dimensions (reduced from 1024 if applicable)\n", - "✅ Loaded 10 documents into the index\n", + "\u2705 Loaded 10 documents into the index\n", " Index now contains 0 documents\n" ] } @@ -268,7 +268,7 @@ "\n", "# Load data into the index\n", "index.load(data_to_load)\n", - "print(f\"✅ Loaded {len(data_to_load)} documents into the index\")\n", + "print(f\"\u2705 Loaded {len(data_to_load)} documents into the index\")\n", "\n", "# Wait a moment for indexing to complete\n", "import time\n", @@ -297,7 +297,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "🔍 Vector Search Results:\n", + "\ud83d\udd0d Vector Search Results:\n", "==================================================\n" ] } @@ -321,7 +321,7 @@ "\n", "results = index.query(query)\n", "\n", - "print(\"🔍 Vector Search Results:\")\n", + "print(\"\ud83d\udd0d Vector Search Results:\")\n", "print(\"=\" * 50)\n", "for i, result in enumerate(results, 1):\n", " distance = result.get('vector_distance', 'N/A')\n", @@ -330,6 +330,94 @@ " print()" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Runtime Parameters for Performance Tuning\n", + "\n", + "SVS-VAMANA supports runtime parameters that can be adjusted at query time without rebuilding the index. These parameters allow you to fine-tune the trade-off between search speed and accuracy.\n", + "\n", + "**Available Runtime Parameters:**\n", + "\n", + "- **`search_window_size`**: Controls the size of the search window during KNN search (higher = better recall, slower search)\n", + "- **`epsilon`**: Approximation factor for range queries (default: 0.01)\n", + "- **`use_search_history`**: Whether to use search buffer (OFF/ON/AUTO, default: AUTO)\n", + "- **`search_buffer_capacity`**: Tuning parameter for 2-level compression (default: search_window_size)\n", + "\n", + "Let's see how these parameters affect search performance:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Example 1: Basic query with default parameters\n", + "basic_query = VectorQuery(\n", + " vector=query_vector.tolist(),\n", + " vector_field_name=\"embedding\",\n", + " return_fields=[\"content\", \"category\"],\n", + " num_results=5\n", + ")\n", + "\n", + "print(\"\ud83d\udd0d Basic Query (default parameters):\")\n", + "results = index.query(basic_query)\n", + "print(f\"Found {len(results)} results\\n\")\n", + "\n", + "# Example 2: Query with tuned runtime parameters for higher recall\n", + "tuned_query = VectorQuery(\n", + " vector=query_vector.tolist(),\n", + " vector_field_name=\"embedding\",\n", + " return_fields=[\"content\", \"category\"],\n", + " num_results=5,\n", + " search_window_size=40, # Larger window for better recall\n", + " use_search_history='ON', # Use search history\n", + " search_buffer_capacity=50 # Larger buffer capacity\n", + ")\n", + "\n", + "print(\"\ud83c\udfaf Tuned Query (higher recall parameters):\")\n", + "results = index.query(tuned_query)\n", + "print(f\"Found {len(results)} results\")\n", + "print(\"\\nNote: Higher search_window_size improves recall but may increase latency\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Runtime Parameters with Range Queries\n", + "\n", + "Runtime parameters are also useful for range queries, where you want to find all vectors within a certain distance threshold:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from redisvl.query import VectorRangeQuery\n", + "\n", + "# Range query with runtime parameters\n", + "range_query = VectorRangeQuery(\n", + " vector=query_vector.tolist(),\n", + " vector_field_name=\"embedding\",\n", + " return_fields=[\"content\", \"category\"],\n", + " distance_threshold=0.3,\n", + " epsilon=0.05, # Approximation factor\n", + " search_window_size=30, # Search window size\n", + " use_search_history='AUTO' # Automatic history management\n", + ")\n", + "\n", + "results = index.query(range_query)\n", + "print(f\"\ud83c\udfaf Range Query Results: Found {len(results)} vectors within distance threshold 0.3\")\n", + "for i, result in enumerate(results[:3], 1):\n", + " distance = result.get('vector_distance', 'N/A')\n", + " print(f\"{i}. {result['content'][:50]}... (distance: {distance})\")" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -354,21 +442,21 @@ "MEMORY Priority:\n", " Compression: LeanVec4x8\n", " Datatype: float16\n", - " Dimensionality reduction: 1024 → 512\n", + " Dimensionality reduction: 1024 \u2192 512\n", " Search window size: 20\n", " Memory savings: 81.2%\n", "\n", "SPEED Priority:\n", " Compression: LeanVec4x8\n", " Datatype: float16\n", - " Dimensionality reduction: 1024 → 256\n", + " Dimensionality reduction: 1024 \u2192 256\n", " Search window size: 40\n", " Memory savings: 90.6%\n", "\n", "BALANCED Priority:\n", " Compression: LeanVec4x8\n", " Datatype: float16\n", - " Dimensionality reduction: 1024 → 512\n", + " Dimensionality reduction: 1024 \u2192 512\n", " Search window size: 30\n", " Memory savings: 81.2%\n" ] @@ -392,7 +480,7 @@ " print(f\" Compression: {config['compression']}\")\n", " print(f\" Datatype: {config['datatype']}\")\n", " if \"reduce\" in config:\n", - " print(f\" Dimensionality reduction: {dims} → {config['reduce']}\")\n", + " print(f\" Dimensionality reduction: {dims} \u2192 {config['reduce']}\")\n", " print(f\" Search window size: {config['search_window_size']}\")\n", " print(f\" Memory savings: {savings}%\")" ] @@ -478,7 +566,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "🔍 Hybrid Search Results (Technology category only):\n", + "\ud83d\udd0d Hybrid Search Results (Technology category only):\n", "=======================================================\n" ] } @@ -497,7 +585,7 @@ "\n", "filtered_results = index.query(hybrid_query)\n", "\n", - "print(\"🔍 Hybrid Search Results (Technology category only):\")\n", + "print(\"\ud83d\udd0d Hybrid Search Results (Technology category only):\")\n", "print(\"=\" * 55)\n", "for i, result in enumerate(filtered_results, 1):\n", " distance = result.get('vector_distance', 'N/A')\n", @@ -524,7 +612,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "📊 Index Statistics:\n", + "\ud83d\udcca Index Statistics:\n", "==============================\n", "Documents: 0\n", "Vector index size: 0.00 MB\n", @@ -537,7 +625,7 @@ "# Get detailed index information\n", "info = index.info()\n", "\n", - "print(\"📊 Index Statistics:\")\n", + "print(\"\ud83d\udcca Index Statistics:\")\n", "print(\"=\" * 30)\n", "print(f\"Documents: {info.get('num_docs', 0)}\")\n", "\n", @@ -664,16 +752,25 @@ "- **Applications** that can tolerate slight recall trade-offs for speed and memory savings\n", "\n", "### Parameter Tuning Guidelines\n", - "- **Start with CompressionAdvisor** recommendations\n", - "- **Increase search_window_size** if you need higher recall\n", - "- **Use LeanVec** for high-dimensional vectors (≥1024 dims)\n", + "\n", + "**Index-time parameters** (set during index creation):\n", + "- **Start with CompressionAdvisor** recommendations for compression and datatype\n", + "- **Use LeanVec** for high-dimensional vectors (\u22651024 dims)\n", "- **Use LVQ** for lower-dimensional vectors (<1024 dims)\n", + "- **graph_max_degree**: Higher values improve recall but increase memory usage\n", + "- **construction_window_size**: Higher values improve index quality but slow down build time\n", + "\n", + "**Runtime parameters** (adjustable at query time without rebuilding index):\n", + "- **search_window_size**: Start with 20, increase to 40-100 for higher recall\n", + "- **epsilon**: Use 0.01-0.05 for range queries (higher = faster but less accurate)\n", + "- **use_search_history**: Use 'AUTO' (default) or 'ON' for better recall\n", + "- **search_buffer_capacity**: Usually set equal to search_window_size\n", "\n", "### Performance Considerations\n", "- **Index build time** increases with higher construction_window_size\n", - "- **Search latency** increases with higher search_window_size\n", + "- **Search latency** increases with higher search_window_size (tunable at query time!)\n", "- **Memory usage** decreases with more aggressive compression\n", - "- **Recall quality** may decrease with more aggressive compression" + "- **Recall quality** may decrease with more aggressive compression or lower search_window_size" ] }, { @@ -729,4 +826,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} +} \ No newline at end of file diff --git a/docs/user_guide/11_advanced_queries.ipynb b/docs/user_guide/11_advanced_queries.ipynb index 831857d7..3125d848 100644 --- a/docs/user_guide/11_advanced_queries.ipynb +++ b/docs/user_guide/11_advanced_queries.ipynb @@ -30,16 +30,18 @@ }, { "cell_type": "code", - "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T00:42:12.222169Z", "iopub.status.busy": "2025-11-21T00:42:12.222058Z", "iopub.status.idle": "2025-11-21T00:42:12.301776Z", "shell.execute_reply": "2025-11-21T00:42:12.301163Z" + }, + "ExecuteTime": { + "end_time": "2025-11-21T21:27:49.998123Z", + "start_time": "2025-11-21T21:27:49.993513Z" } }, - "outputs": [], "source": [ "import numpy as np\n", "from jupyterutils import result_print\n", @@ -107,7 +109,9 @@ " 'image_embedding': np.array([0.2, 0.8], dtype=np.float32).tobytes(),\n", " },\n", "]" - ] + ], + "outputs": [], + "execution_count": 6 }, { "cell_type": "markdown", @@ -124,16 +128,18 @@ }, { "cell_type": "code", - "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T00:42:12.303593Z", "iopub.status.busy": "2025-11-21T00:42:12.303450Z", "iopub.status.idle": "2025-11-21T00:42:12.305709Z", "shell.execute_reply": "2025-11-21T00:42:12.305407Z" + }, + "ExecuteTime": { + "end_time": "2025-11-21T21:27:50.362957Z", + "start_time": "2025-11-21T21:27:50.360561Z" } }, - "outputs": [], "source": [ "schema = {\n", " \"index\": {\n", @@ -170,7 +176,9 @@ " }\n", " ],\n", "}" - ] + ], + "outputs": [], + "execution_count": 7 }, { "cell_type": "markdown", @@ -181,16 +189,18 @@ }, { "cell_type": "code", - "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T00:42:12.306952Z", "iopub.status.busy": "2025-11-21T00:42:12.306869Z", "iopub.status.idle": "2025-11-21T00:42:12.416481Z", "shell.execute_reply": "2025-11-21T00:42:12.415926Z" + }, + "ExecuteTime": { + "end_time": "2025-11-21T21:27:50.727271Z", + "start_time": "2025-11-21T21:27:50.715789Z" } }, - "outputs": [], "source": [ "from redisvl.index import SearchIndex\n", "\n", @@ -202,7 +212,18 @@ "keys = index.load(data)\n", "\n", "print(f\"Loaded {len(keys)} products into the index\")" - ] + ], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "16:27:50 redisvl.index.index INFO Index already exists, overwriting.\n", + "Loaded 6 products into the index\n" + ] + } + ], + "execution_count": 8 }, { "cell_type": "markdown", @@ -219,16 +240,18 @@ }, { "cell_type": "code", - "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T00:42:12.433591Z", "iopub.status.busy": "2025-11-21T00:42:12.433464Z", "iopub.status.idle": "2025-11-21T00:42:13.709475Z", "shell.execute_reply": "2025-11-21T00:42:13.708647Z" + }, + "ExecuteTime": { + "end_time": "2025-11-21T21:27:51.127508Z", + "start_time": "2025-11-21T21:27:51.123980Z" } }, - "outputs": [], "source": [ "from redisvl.query import TextQuery\n", "\n", @@ -242,7 +265,25 @@ "\n", "results = index.query(text_query)\n", "result_print(results)" - ] + ], + "outputs": [ + { + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "
scoreproduct_idbrief_descriptioncategoryprice
4.080705480646511prod_1comfortable running shoes for athletesfootwear89.99
1.4504838715161907prod_5basketball shoes with excellent ankle supportfootwear139.99
1.431980178975859prod_2lightweight running jacket with water resistanceouterwear129.99
" + ] + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + } + ], + "execution_count": 9 }, { "cell_type": "markdown", @@ -255,16 +296,18 @@ }, { "cell_type": "code", - "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T00:42:13.711396Z", "iopub.status.busy": "2025-11-21T00:42:13.711221Z", "iopub.status.idle": "2025-11-21T00:42:13.749216Z", "shell.execute_reply": "2025-11-21T00:42:13.748398Z" + }, + "ExecuteTime": { + "end_time": "2025-11-21T21:27:51.537001Z", + "start_time": "2025-11-21T21:27:51.532996Z" } }, - "outputs": [], "source": [ "# BM25 standard scoring (default)\n", "bm25_query = TextQuery(\n", @@ -278,20 +321,47 @@ "print(\"Results with BM25 scoring:\")\n", "results = index.query(bm25_query)\n", "result_print(results)" - ] + ], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Results with BM25 scoring:\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "
scoreproduct_idbrief_descriptionprice
4.165936382048982prod_1comfortable running shoes for athletes89.99
1.769051138581863prod_4yoga mat with extra cushioning for comfort39.99
1.2306902673750557prod_5basketball shoes with excellent ankle support139.99
" + ] + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + } + ], + "execution_count": 10 }, { "cell_type": "code", - "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T00:42:13.750799Z", "iopub.status.busy": "2025-11-21T00:42:13.750686Z", "iopub.status.idle": "2025-11-21T00:42:13.754896Z", "shell.execute_reply": "2025-11-21T00:42:13.754345Z" + }, + "ExecuteTime": { + "end_time": "2025-11-21T21:27:51.747761Z", + "start_time": "2025-11-21T21:27:51.742796Z" } }, - "outputs": [], "source": [ "# TFIDF scoring\n", "tfidf_query = TextQuery(\n", @@ -305,7 +375,32 @@ "print(\"Results with TFIDF scoring:\")\n", "results = index.query(tfidf_query)\n", "result_print(results)" - ] + ], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Results with TFIDF scoring:\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "
scoreproduct_idbrief_descriptionprice
1.3333333333333333prod_1comfortable running shoes for athletes89.99
1.3333333333333333prod_1comfortable running shoes for athletes89.99
1.0prod_5basketball shoes with excellent ankle support139.99
" + ] + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + } + ], + "execution_count": 11 }, { "cell_type": "markdown", @@ -318,16 +413,18 @@ }, { "cell_type": "code", - "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T00:42:13.756368Z", "iopub.status.busy": "2025-11-21T00:42:13.756224Z", "iopub.status.idle": "2025-11-21T00:42:13.760388Z", "shell.execute_reply": "2025-11-21T00:42:13.759844Z" + }, + "ExecuteTime": { + "end_time": "2025-11-21T21:27:52.153660Z", + "start_time": "2025-11-21T21:27:52.150061Z" } }, - "outputs": [], "source": [ "from redisvl.query.filter import Tag, Num\n", "\n", @@ -342,20 +439,40 @@ "\n", "results = index.query(filtered_text_query)\n", "result_print(results)" - ] + ], + "outputs": [ + { + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "
scoreproduct_idbrief_descriptioncategoryprice
2.385806908729779prod_1comfortable running shoes for athletesfootwear89.99
2.385806908729779prod_1comfortable running shoes for athletesfootwear89.99
1.9340948871093797prod_5basketball shoes with excellent ankle supportfootwear139.99
" + ] + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + } + ], + "execution_count": 12 }, { "cell_type": "code", - "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T00:42:13.761654Z", "iopub.status.busy": "2025-11-21T00:42:13.761566Z", "iopub.status.idle": "2025-11-21T00:42:13.765694Z", "shell.execute_reply": "2025-11-21T00:42:13.765316Z" + }, + "ExecuteTime": { + "end_time": "2025-11-21T21:27:52.357623Z", + "start_time": "2025-11-21T21:27:52.351735Z" } }, - "outputs": [], "source": [ "# Search for products under $100\n", "price_filtered_query = TextQuery(\n", @@ -368,7 +485,25 @@ "\n", "results = index.query(price_filtered_query)\n", "result_print(results)" - ] + ], + "outputs": [ + { + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "
scoreproduct_idbrief_descriptionprice
2.2775029612659465prod_1comfortable running shoes for athletes89.99
1.1387514806329733prod_1comfortable running shoes for athletes89.99
1.1190633543347508prod_4yoga mat with extra cushioning for comfort39.99
1.1190633543347508prod_4yoga mat with extra cushioning for comfort39.99
" + ] + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + } + ], + "execution_count": 13 }, { "cell_type": "markdown", @@ -382,16 +517,18 @@ }, { "cell_type": "code", - "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T00:42:13.767228Z", "iopub.status.busy": "2025-11-21T00:42:13.767102Z", "iopub.status.idle": "2025-11-21T00:42:13.771059Z", "shell.execute_reply": "2025-11-21T00:42:13.770555Z" + }, + "ExecuteTime": { + "end_time": "2025-11-21T21:27:52.754720Z", + "start_time": "2025-11-21T21:27:52.751189Z" } }, - "outputs": [], "source": [ "weighted_query = TextQuery(\n", " text=\"shoes\",\n", @@ -402,7 +539,25 @@ "\n", "results = index.query(weighted_query)\n", "result_print(results)" - ] + ], + "outputs": [ + { + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "
scoreproduct_idbrief_description
3.040323653363804prod_1comfortable running shoes for athletes
3.040323653363804prod_1comfortable running shoes for athletes
1.289396591406253prod_5basketball shoes with excellent ankle support
" + ] + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + } + ], + "execution_count": 14 }, { "cell_type": "markdown", @@ -415,16 +570,18 @@ }, { "cell_type": "code", - "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T00:42:13.772513Z", "iopub.status.busy": "2025-11-21T00:42:13.772419Z", "iopub.status.idle": "2025-11-21T00:42:13.776286Z", "shell.execute_reply": "2025-11-21T00:42:13.775861Z" + }, + "ExecuteTime": { + "end_time": "2025-11-21T21:27:53.171295Z", + "start_time": "2025-11-21T21:27:53.167127Z" } }, - "outputs": [], "source": [ "# Use English stopwords (default)\n", "query_with_stopwords = TextQuery(\n", @@ -437,20 +594,40 @@ "\n", "results = index.query(query_with_stopwords)\n", "result_print(results)" - ] + ], + "outputs": [ + { + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "
scoreproduct_idbrief_description
4.1444591833267275prod_1comfortable running shoes for athletes
4.1444591833267275prod_1comfortable running shoes for athletes
1.4875097606385526prod_5basketball shoes with excellent ankle support
" + ] + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + } + ], + "execution_count": 15 }, { "cell_type": "code", - "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T00:42:13.777294Z", "iopub.status.busy": "2025-11-21T00:42:13.777220Z", "iopub.status.idle": "2025-11-21T00:42:13.781329Z", "shell.execute_reply": "2025-11-21T00:42:13.780713Z" + }, + "ExecuteTime": { + "end_time": "2025-11-21T21:27:53.528245Z", + "start_time": "2025-11-21T21:27:53.525116Z" } }, - "outputs": [], "source": [ "# Use custom stopwords\n", "custom_stopwords_query = TextQuery(\n", @@ -463,20 +640,40 @@ "\n", "results = index.query(custom_stopwords_query)\n", "result_print(results)" - ] + ], + "outputs": [ + { + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "
scoreproduct_idbrief_description
2.5107799078325prod_1comfortable running shoes for athletes
2.5107799078325prod_1comfortable running shoes for athletes
2.482820220115406prod_3professional tennis racket for competitive players
" + ] + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + } + ], + "execution_count": 16 }, { "cell_type": "code", - "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T00:42:13.782401Z", "iopub.status.busy": "2025-11-21T00:42:13.782323Z", "iopub.status.idle": "2025-11-21T00:42:13.787197Z", "shell.execute_reply": "2025-11-21T00:42:13.786617Z" + }, + "ExecuteTime": { + "end_time": "2025-11-21T21:27:53.892142Z", + "start_time": "2025-11-21T21:27:53.888038Z" } }, - "outputs": [], "source": [ "# No stopwords\n", "no_stopwords_query = TextQuery(\n", @@ -489,7 +686,25 @@ "\n", "results = index.query(no_stopwords_query)\n", "result_print(results)" - ] + ], + "outputs": [ + { + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "
scoreproduct_idbrief_description
3.69730364515632prod_1comfortable running shoes for athletes
3.69730364515632prod_1comfortable running shoes for athletes
1.5329921800414583prod_5basketball shoes with excellent ankle support
" + ] + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + } + ], + "execution_count": 17 }, { "cell_type": "markdown", @@ -536,16 +751,18 @@ }, { "cell_type": "code", - "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T00:42:13.788835Z", "iopub.status.busy": "2025-11-21T00:42:13.788717Z", "iopub.status.idle": "2025-11-21T00:42:13.795247Z", "shell.execute_reply": "2025-11-21T00:42:13.794662Z" + }, + "ExecuteTime": { + "end_time": "2025-11-21T21:27:55.430188Z", + "start_time": "2025-11-21T21:27:55.420369Z" } }, - "outputs": [], "source": [ "# Create a schema with index-level stopwords disabled\n", "from redisvl.index import SearchIndex\n", @@ -568,20 +785,32 @@ "company_index.create(overwrite=True, drop=True)\n", "\n", "print(f\"Index created with STOPWORDS 0: {company_index}\")" - ] + ], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Index created with STOPWORDS 0: \n" + ] + } + ], + "execution_count": 18 }, { "cell_type": "code", - "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T00:42:13.796880Z", "iopub.status.busy": "2025-11-21T00:42:13.796745Z", "iopub.status.idle": "2025-11-21T00:42:13.802750Z", "shell.execute_reply": "2025-11-21T00:42:13.802098Z" + }, + "ExecuteTime": { + "end_time": "2025-11-21T21:27:55.640718Z", + "start_time": "2025-11-21T21:27:55.635077Z" } }, - "outputs": [], "source": [ "# Load sample data with company names containing common stopwords\n", "companies = [\n", @@ -595,21 +824,33 @@ "for i, company in enumerate(companies):\n", " company_index.load([company], keys=[f\"company:{i}\"])\n", "\n", - "print(f\"✓ Loaded {len(companies)} companies\")" - ] + "print(f\"\u2713 Loaded {len(companies)} companies\")" + ], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u2713 Loaded 5 companies\n" + ] + } + ], + "execution_count": 19 }, { "cell_type": "code", - "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T00:42:13.804059Z", "iopub.status.busy": "2025-11-21T00:42:13.803942Z", "iopub.status.idle": "2025-11-21T00:42:13.807026Z", "shell.execute_reply": "2025-11-21T00:42:13.806491Z" + }, + "ExecuteTime": { + "end_time": "2025-11-21T21:27:55.833033Z", + "start_time": "2025-11-21T21:27:55.829220Z" } }, - "outputs": [], "source": [ "# Search for \"Bank of Glasberliner\" - with STOPWORDS 0, \"of\" is indexed and searchable\n", "from redisvl.query import FilterQuery\n", @@ -624,7 +865,18 @@ "print(f\"Found {len(results.docs)} results for 'Bank of Glasberliner':\")\n", "for doc in results.docs:\n", " print(f\" - {doc.company_name}: {doc.description}\")" - ] + ], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found 1 results for 'Bank of Glasberliner':\n", + " - Bank of Glasberliner: Major financial institution\n" + ] + } + ], + "execution_count": 20 }, { "cell_type": "markdown", @@ -634,9 +886,9 @@ "\n", "If we had used the default stopwords (not specifying `stopwords` in the schema), the word \"of\" would be filtered out during indexing. This means:\n", "\n", - "- ❌ Searching for `\"Bank of Glasberliner\"` might not find exact matches\n", - "- ❌ The phrase would be indexed as `\"Bank Berlin\"` (without \"of\")\n", - "- ✅ With `STOPWORDS 0`, all words including \"of\" are indexed\n", + "- \u274c Searching for `\"Bank of Glasberliner\"` might not find exact matches\n", + "- \u274c The phrase would be indexed as `\"Bank Berlin\"` (without \"of\")\n", + "- \u2705 With `STOPWORDS 0`, all words including \"of\" are indexed\n", "\n", "**Custom Stopwords Example:**\n", "\n", @@ -645,16 +897,18 @@ }, { "cell_type": "code", - "execution_count": null, "metadata": { "execution": { "iopub.execute_input": "2025-11-21T00:42:13.808543Z", "iopub.status.busy": "2025-11-21T00:42:13.808418Z", "iopub.status.idle": "2025-11-21T00:42:13.810612Z", "shell.execute_reply": "2025-11-21T00:42:13.810083Z" + }, + "ExecuteTime": { + "end_time": "2025-11-21T21:27:56.463470Z", + "start_time": "2025-11-21T21:27:56.461409Z" } }, - "outputs": [], "source": [ "# Example: Create index with custom stopwords\n", "custom_stopwords_schema = {\n", @@ -670,7 +924,17 @@ "\n", "# This would create an index where \"inc\", \"llc\", \"corp\" are not indexed\n", "print(\"Custom stopwords:\", custom_stopwords_schema[\"index\"][\"stopwords\"])" - ] + ], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Custom stopwords: ['inc', 'llc', 'corp']\n" + ] + } + ], + "execution_count": 21 }, { "cell_type": "markdown", @@ -709,14 +973,27 @@ }, { "cell_type": "code", - "metadata": {}, + "metadata": { + "ExecuteTime": { + "end_time": "2025-11-21T21:27:57.036397Z", + "start_time": "2025-11-21T21:27:57.030555Z" + } + }, "source": [ "# Cleanup\n", "company_index.delete(drop=True)\n", - "print(\"✓ Cleaned up company_index\")" + "print(\"\u2713 Cleaned up company_index\")" ], - "outputs": [], - "execution_count": null + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u2713 Cleaned up company_index\n" + ] + } + ], + "execution_count": 22 }, { "cell_type": "markdown", @@ -729,7 +1006,12 @@ }, { "cell_type": "code", - "metadata": {}, + "metadata": { + "ExecuteTime": { + "end_time": "2025-11-21T21:27:57.725041Z", + "start_time": "2025-11-21T21:27:57.719775Z" + } + }, "source": [ "from redisvl.query import AggregateHybridQuery\n", "\n", @@ -746,8 +1028,24 @@ "results = index.query(hybrid_query)\n", "result_print(results)" ], - "outputs": [], - "execution_count": null + "outputs": [ + { + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "
vector_distanceproduct_idbrief_descriptioncategorypricevector_similaritytext_scorehybrid_score
5.96046447754e-08prod_1comfortable running shoes for athletesfootwear89.990.9999999701984.829774426092.14893230697
5.96046447754e-08prod_1comfortable running shoes for athletesfootwear89.990.9999999701984.829774426092.14893230697
5.96046447754e-08prod_1comfortable running shoes for athletesfootwear89.990.9999999701984.829774426092.14893230697
0.0038834810257prod_4yoga mat with extra cushioning for comfortaccessories39.990.99805825948700.698640781641
0.0038834810257prod_4yoga mat with extra cushioning for comfortaccessories39.990.99805825948700.698640781641
" + ] + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + } + ], + "execution_count": 23 }, { "cell_type": "markdown", @@ -763,7 +1061,12 @@ }, { "cell_type": "code", - "metadata": {}, + "metadata": { + "ExecuteTime": { + "end_time": "2025-11-21T21:28:02.908824Z", + "start_time": "2025-11-21T21:28:02.902585Z" + } + }, "source": [ "# More emphasis on vector search (alpha=0.9)\n", "vector_heavy_query = AggregateHybridQuery(\n", @@ -780,8 +1083,31 @@ "results = index.query(vector_heavy_query)\n", "result_print(results)" ], - "outputs": [], - "execution_count": null + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Results with alpha=0.9 (vector-heavy):\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "
vector_distanceproduct_idbrief_descriptionvector_similaritytext_scorehybrid_score
-1.19209289551e-07prod_4yoga mat with extra cushioning for comfort1.00000005961.538380705411.05383812419
-1.19209289551e-07prod_4yoga mat with extra cushioning for comfort1.00000005961.538380705411.05383812419
-1.19209289551e-07prod_4yoga mat with extra cushioning for comfort1.00000005961.538380705411.05383812419
" + ] + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + } + ], + "execution_count": 24 }, { "cell_type": "markdown", @@ -794,7 +1120,12 @@ }, { "cell_type": "code", - "metadata": {}, + "metadata": { + "ExecuteTime": { + "end_time": "2025-11-21T21:28:04.309151Z", + "start_time": "2025-11-21T21:28:04.302860Z" + } + }, "source": [ "# Hybrid search with a price filter\n", "filtered_hybrid_query = AggregateHybridQuery(\n", @@ -810,8 +1141,24 @@ "results = index.query(filtered_hybrid_query)\n", "result_print(results)" ], - "outputs": [], - "execution_count": null + "outputs": [ + { + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "
vector_distanceproduct_idbrief_descriptioncategorypricevector_similaritytext_scorehybrid_score
-1.19209289551e-07prod_3professional tennis racket for competitive playersequipment199.991.00000005961.547237055061.16417115824
-1.19209289551e-07prod_3professional tennis racket for competitive playersequipment199.991.00000005961.547237055061.16417115824
-1.19209289551e-07prod_3professional tennis racket for competitive playersequipment199.991.00000005961.547237055061.16417115824
0.411657452583prod_2lightweight running jacket with water resistanceouterwear129.990.79417127370800.555919891596
0.411657452583prod_2lightweight running jacket with water resistanceouterwear129.990.79417127370800.555919891596
" + ] + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + } + ], + "execution_count": 25 }, { "cell_type": "markdown", @@ -824,7 +1171,12 @@ }, { "cell_type": "code", - "metadata": {}, + "metadata": { + "ExecuteTime": { + "end_time": "2025-11-21T21:28:05.455328Z", + "start_time": "2025-11-21T21:28:05.450590Z" + } + }, "source": [ "# Aggregate Hybrid query with TFIDF scorer\n", "hybrid_tfidf = AggregateHybridQuery(\n", @@ -840,8 +1192,43 @@ "results = index.query(hybrid_tfidf)\n", "result_print(results)" ], - "outputs": [], - "execution_count": null + "outputs": [ + { + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "
vector_distanceproduct_idbrief_descriptionvector_similaritytext_scorehybrid_score
0prod_5basketball shoes with excellent ankle support131.6
0prod_2lightweight running jacket with water resistance100.7
0prod_2lightweight running jacket with water resistance100.7
" + ] + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + } + ], + "execution_count": 26 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": [ + "### Runtime Parameters for Vector Search Tuning\n", + "\n", + "**Important:** `AggregateHybridQuery` uses FT.AGGREGATE commands which do NOT support runtime parameters.\n", + "\n", + "Runtime parameters (such as `ef_runtime` for HNSW indexes or `search_window_size` for SVS-VAMANA indexes) are only supported with FT.SEARCH commands.\n", + "\n", + "**For runtime parameter support, use `VectorQuery` or `VectorRangeQuery` instead:**\n", + "\n", + "- `VectorQuery`: Supports all runtime parameters (HNSW and SVS-VAMANA)\n", + "- `VectorRangeQuery`: Supports all runtime parameters (HNSW and SVS-VAMANA)\n", + "- `AggregateHybridQuery`: Does NOT support runtime parameters (uses FT.AGGREGATE)\n", + "\n", + "See the **Runtime Parameters** section earlier in this notebook for examples of using runtime parameters with `VectorQuery`." + ] }, { "cell_type": "markdown", @@ -1135,4 +1522,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} +} \ No newline at end of file diff --git a/redisvl/query/aggregate.py b/redisvl/query/aggregate.py index 299de0ce..09981ca1 100644 --- a/redisvl/query/aggregate.py +++ b/redisvl/query/aggregate.py @@ -137,6 +137,11 @@ def __init__( within the query text. Defaults to None, as no modifications will be made to the text_scorer score. + Note: + AggregateHybridQuery uses FT.AGGREGATE commands which do NOT support runtime + parameters. For runtime parameter support (ef_runtime, search_window_size, etc.), + use VectorQuery or VectorRangeQuery which use FT.SEARCH commands. + Raises: ValueError: If the text string is empty, or if the text string becomes empty after stopwords are removed. @@ -183,7 +188,7 @@ def params(self) -> Dict[str, Any]: else: vector = self._vector - params = {self.VECTOR_PARAM: vector} + params: Dict[str, Any] = {self.VECTOR_PARAM: vector} return params @@ -303,8 +308,13 @@ def _build_query_string(self) -> str: if isinstance(self._filter_expression, FilterExpression): filter_expression = str(self._filter_expression) - # base KNN query - knn_query = f"KNN {self._num_results} @{self._vector_field} ${self.VECTOR_PARAM} AS {self.DISTANCE_ID}" + # Build KNN query + knn_query = ( + f"KNN {self._num_results} @{self._vector_field} ${self.VECTOR_PARAM}" + ) + + # Add distance field alias + knn_query += f" AS {self.DISTANCE_ID}" text = f"(~@{self._text_field}:({self._tokenize_and_escape_query(self._text)})" diff --git a/redisvl/query/query.py b/redisvl/query/query.py index 1237c07f..7d4cbe88 100644 --- a/redisvl/query/query.py +++ b/redisvl/query/query.py @@ -421,8 +421,19 @@ def _build_query_string(self) -> str: class BaseVectorQuery: DISTANCE_ID: str = "vector_distance" VECTOR_PARAM: str = "vector" + + # HNSW runtime parameters EF_RUNTIME: str = "EF_RUNTIME" EF_RUNTIME_PARAM: str = "EF" + EPSILON_PARAM: str = "EPSILON" + + # SVS-VAMANA runtime parameters + SEARCH_WINDOW_SIZE: str = "SEARCH_WINDOW_SIZE" + SEARCH_WINDOW_SIZE_PARAM: str = "SEARCH_WINDOW_SIZE" + USE_SEARCH_HISTORY: str = "USE_SEARCH_HISTORY" + USE_SEARCH_HISTORY_PARAM: str = "USE_SEARCH_HISTORY" + SEARCH_BUFFER_CAPACITY: str = "SEARCH_BUFFER_CAPACITY" + SEARCH_BUFFER_CAPACITY_PARAM: str = "SEARCH_BUFFER_CAPACITY" _normalize_vector_distance: bool = False @@ -450,6 +461,10 @@ def __init__( hybrid_policy: Optional[str] = None, batch_size: Optional[int] = None, ef_runtime: Optional[int] = None, + epsilon: Optional[float] = None, + search_window_size: Optional[int] = None, + use_search_history: Optional[str] = None, + search_buffer_capacity: Optional[int] = None, normalize_vector_distance: bool = False, ): """A query for running a vector search along with an optional filter @@ -494,6 +509,22 @@ def __init__( ef_runtime (Optional[int]): Controls the size of the dynamic candidate list for HNSW algorithm at query time. Higher values improve recall at the expense of slower search performance. Defaults to None, which uses the index-defined value. + epsilon (Optional[float]): The range search approximation factor for HNSW and SVS-VAMANA + indexes. Sets boundaries for candidates within radius * (1 + epsilon). Higher values + allow more extensive search and more accurate results at the expense of run time. + Defaults to None, which uses the index-defined value (typically 0.01). + search_window_size (Optional[int]): The size of the search window for SVS-VAMANA KNN searches. + Increasing this value generally yields more accurate but slower search results. + Defaults to None, which uses the index-defined value (typically 10). + use_search_history (Optional[str]): For SVS-VAMANA indexes, controls whether to use the + search buffer or entire search history. Options are "OFF", "ON", or "AUTO". + "AUTO" is always evaluated internally as "ON". Using the entire history may yield + a slightly better graph at the cost of more search time. + Defaults to None, which uses the index-defined value (typically "AUTO"). + search_buffer_capacity (Optional[int]): Tuning parameter for SVS-VAMANA indexes using + two-level compression (LVQx or LeanVec types). Determines the number of vector + candidates to collect in the first level of search before the re-ranking level. + Defaults to None, which uses the index-defined value (typically SEARCH_WINDOW_SIZE). normalize_vector_distance (bool): Redis supports 3 distance metrics: L2 (euclidean), IP (inner product), and COSINE. By default, L2 distance returns an unbounded value. COSINE distance returns a value between 0 and 2. IP returns a value determined by @@ -514,6 +545,10 @@ def __init__( self._hybrid_policy: Optional[HybridPolicy] = None self._batch_size: Optional[int] = None self._ef_runtime: Optional[int] = None + self._epsilon: Optional[float] = None + self._search_window_size: Optional[int] = None + self._use_search_history: Optional[str] = None + self._search_buffer_capacity: Optional[int] = None self._normalize_vector_distance = normalize_vector_distance self.set_filter(filter_expression) @@ -547,6 +582,18 @@ def __init__( if ef_runtime is not None: self.set_ef_runtime(ef_runtime) + if epsilon is not None: + self.set_epsilon(epsilon) + + if search_window_size is not None: + self.set_search_window_size(search_window_size) + + if use_search_history is not None: + self.set_use_search_history(use_search_history) + + if search_buffer_capacity is not None: + self.set_search_buffer_capacity(search_buffer_capacity) + def _build_query_string(self) -> str: """Build the full query string for vector search with optional filtering.""" filter_expression = self._filter_expression @@ -566,10 +613,28 @@ def _build_query_string(self) -> str: if self._hybrid_policy == HybridPolicy.BATCHES and self._batch_size: knn_query += f" BATCH_SIZE {self._batch_size}" - # Add EF_RUNTIME parameter if specified + # Add EF_RUNTIME parameter if specified (HNSW) if self._ef_runtime: knn_query += f" {self.EF_RUNTIME} ${self.EF_RUNTIME_PARAM}" + # Add EPSILON parameter if specified (HNSW and SVS-VAMANA) + if self._epsilon is not None: + knn_query += f" EPSILON ${self.EPSILON_PARAM}" + + # Add SEARCH_WINDOW_SIZE parameter if specified (SVS-VAMANA) + if self._search_window_size is not None: + knn_query += f" {self.SEARCH_WINDOW_SIZE} ${self.SEARCH_WINDOW_SIZE_PARAM}" + + # Add USE_SEARCH_HISTORY parameter if specified (SVS-VAMANA) + if self._use_search_history is not None: + knn_query += f" {self.USE_SEARCH_HISTORY} ${self.USE_SEARCH_HISTORY_PARAM}" + + # Add SEARCH_BUFFER_CAPACITY parameter if specified (SVS-VAMANA) + if self._search_buffer_capacity is not None: + knn_query += ( + f" {self.SEARCH_BUFFER_CAPACITY} ${self.SEARCH_BUFFER_CAPACITY_PARAM}" + ) + # Add distance field alias knn_query += f" AS {self.DISTANCE_ID}" @@ -634,6 +699,92 @@ def set_ef_runtime(self, ef_runtime: int): # Invalidate the query string self._built_query_string = None + def set_epsilon(self, epsilon: float): + """Set the epsilon parameter for the query. + + Args: + epsilon (float): The range search approximation factor for HNSW and SVS-VAMANA + indexes. Sets boundaries for candidates within radius * (1 + epsilon). + Higher values allow more extensive search and more accurate results at the + expense of run time. + + Raises: + TypeError: If epsilon is not a float or int + ValueError: If epsilon is negative + """ + if not isinstance(epsilon, (float, int)): + raise TypeError("epsilon must be of type float or int") + if epsilon < 0: + raise ValueError("epsilon must be non-negative") + self._epsilon = epsilon + + # Invalidate the query string + self._built_query_string = None + + def set_search_window_size(self, search_window_size: int): + """Set the SEARCH_WINDOW_SIZE parameter for the query. + + Args: + search_window_size (int): The size of the search window for SVS-VAMANA KNN searches. + Increasing this value generally yields more accurate but slower search results. + + Raises: + TypeError: If search_window_size is not an integer + ValueError: If search_window_size is not positive + """ + if not isinstance(search_window_size, int): + raise TypeError("search_window_size must be an integer") + if search_window_size <= 0: + raise ValueError("search_window_size must be positive") + self._search_window_size = search_window_size + + # Invalidate the query string + self._built_query_string = None + + def set_use_search_history(self, use_search_history: str): + """Set the USE_SEARCH_HISTORY parameter for the query. + + Args: + use_search_history (str): For SVS-VAMANA indexes, controls whether to use the + search buffer or entire search history. Options are "OFF", "ON", or "AUTO". + + Raises: + TypeError: If use_search_history is not a string + ValueError: If use_search_history is not one of "OFF", "ON", or "AUTO" + """ + if not isinstance(use_search_history, str): + raise TypeError("use_search_history must be a string") + valid_options = ["OFF", "ON", "AUTO"] + if use_search_history not in valid_options: + raise ValueError( + f"use_search_history must be one of {', '.join(valid_options)}" + ) + self._use_search_history = use_search_history + + # Invalidate the query string + self._built_query_string = None + + def set_search_buffer_capacity(self, search_buffer_capacity: int): + """Set the SEARCH_BUFFER_CAPACITY parameter for the query. + + Args: + search_buffer_capacity (int): Tuning parameter for SVS-VAMANA indexes using + two-level compression. Determines the number of vector candidates to collect + in the first level of search before the re-ranking level. + + Raises: + TypeError: If search_buffer_capacity is not an integer + ValueError: If search_buffer_capacity is not positive + """ + if not isinstance(search_buffer_capacity, int): + raise TypeError("search_buffer_capacity must be an integer") + if search_buffer_capacity <= 0: + raise ValueError("search_buffer_capacity must be positive") + self._search_buffer_capacity = search_buffer_capacity + + # Invalidate the query string + self._built_query_string = None + @property def hybrid_policy(self) -> Optional[str]: """Return the hybrid policy for the query. @@ -661,6 +812,42 @@ def ef_runtime(self) -> Optional[int]: """ return self._ef_runtime + @property + def epsilon(self) -> Optional[float]: + """Return the epsilon parameter for the query. + + Returns: + Optional[float]: The epsilon value for the query. + """ + return self._epsilon + + @property + def search_window_size(self) -> Optional[int]: + """Return the SEARCH_WINDOW_SIZE parameter for the query. + + Returns: + Optional[int]: The SEARCH_WINDOW_SIZE value for the query. + """ + return self._search_window_size + + @property + def use_search_history(self) -> Optional[str]: + """Return the USE_SEARCH_HISTORY parameter for the query. + + Returns: + Optional[str]: The USE_SEARCH_HISTORY value for the query. + """ + return self._use_search_history + + @property + def search_buffer_capacity(self) -> Optional[int]: + """Return the SEARCH_BUFFER_CAPACITY parameter for the query. + + Returns: + Optional[int]: The SEARCH_BUFFER_CAPACITY value for the query. + """ + return self._search_buffer_capacity + @property def params(self) -> Dict[str, Any]: """Return the parameters for the query. @@ -675,10 +862,26 @@ def params(self) -> Dict[str, Any]: params: Dict[str, Any] = {self.VECTOR_PARAM: vector} - # Add EF_RUNTIME parameter if specified + # Add EF_RUNTIME parameter if specified (HNSW) if self._ef_runtime is not None: params[self.EF_RUNTIME_PARAM] = self._ef_runtime + # Add EPSILON parameter if specified (HNSW and SVS-VAMANA) + if self._epsilon is not None: + params[self.EPSILON_PARAM] = self._epsilon + + # Add SEARCH_WINDOW_SIZE parameter if specified (SVS-VAMANA) + if self._search_window_size is not None: + params[self.SEARCH_WINDOW_SIZE_PARAM] = self._search_window_size + + # Add USE_SEARCH_HISTORY parameter if specified (SVS-VAMANA) + if self._use_search_history is not None: + params[self.USE_SEARCH_HISTORY_PARAM] = self._use_search_history + + # Add SEARCH_BUFFER_CAPACITY parameter if specified (SVS-VAMANA) + if self._search_buffer_capacity is not None: + params[self.SEARCH_BUFFER_CAPACITY_PARAM] = self._search_buffer_capacity + return params @@ -697,6 +900,9 @@ def __init__( dtype: str = "float32", distance_threshold: float = 0.2, epsilon: Optional[float] = None, + search_window_size: Optional[int] = None, + use_search_history: Optional[str] = None, + search_buffer_capacity: Optional[int] = None, num_results: int = 10, return_score: bool = True, dialect: int = 2, @@ -727,6 +933,18 @@ def __init__( This controls how extensive the search is beyond the specified radius. Higher values increase recall at the expense of performance. Defaults to None, which uses the index-defined epsilon (typically 0.01). + search_window_size (Optional[int]): The size of the search window for SVS-VAMANA range searches. + Increasing this value generally yields more accurate but slower search results. + Defaults to None, which uses the index-defined value (typically 10). + use_search_history (Optional[str]): For SVS-VAMANA indexes, controls whether to use the + search buffer or entire search history. Options are "OFF", "ON", or "AUTO". + "AUTO" is always evaluated internally as "ON". Using the entire history may yield + a slightly better graph at the cost of more search time. + Defaults to None, which uses the index-defined value (typically "AUTO"). + search_buffer_capacity (Optional[int]): Tuning parameter for SVS-VAMANA indexes using + two-level compression (LVQx or LeanVec types). Determines the number of vector + candidates to collect in the first level of search before the re-ranking level. + Defaults to None, which uses the index-defined value (typically SEARCH_WINDOW_SIZE). num_results (int): The MAX number of results to return. Defaults to 10. return_score (bool, optional): Whether to return the vector @@ -772,6 +990,9 @@ def __init__( self._num_results = num_results self._distance_threshold: float = 0.2 # Initialize with default self._epsilon: Optional[float] = None + self._search_window_size: Optional[int] = None + self._use_search_history: Optional[str] = None + self._search_buffer_capacity: Optional[int] = None self._hybrid_policy: Optional[HybridPolicy] = None self._batch_size: Optional[int] = None self._normalize_vector_distance = normalize_vector_distance @@ -783,6 +1004,15 @@ def __init__( if epsilon is not None: self.set_epsilon(epsilon) + if search_window_size is not None: + self.set_search_window_size(search_window_size) + + if use_search_history is not None: + self.set_use_search_history(use_search_history) + + if search_buffer_capacity is not None: + self.set_search_buffer_capacity(search_buffer_capacity) + if hybrid_policy is not None: self.set_hybrid_policy(hybrid_policy) @@ -856,6 +1086,68 @@ def set_epsilon(self, epsilon: float): # Invalidate the query string self._built_query_string = None + def set_search_window_size(self, search_window_size: int): + """Set the SEARCH_WINDOW_SIZE parameter for the range query. + + Args: + search_window_size (int): The size of the search window for SVS-VAMANA range searches. + + Raises: + TypeError: If search_window_size is not an integer + ValueError: If search_window_size is not positive + """ + if not isinstance(search_window_size, int): + raise TypeError("search_window_size must be an integer") + if search_window_size <= 0: + raise ValueError("search_window_size must be positive") + self._search_window_size = search_window_size + + # Invalidate the query string + self._built_query_string = None + + def set_use_search_history(self, use_search_history: str): + """Set the USE_SEARCH_HISTORY parameter for the range query. + + Args: + use_search_history (str): Controls whether to use the search buffer or entire history. + Must be one of "OFF", "ON", or "AUTO". + + Raises: + TypeError: If use_search_history is not a string + ValueError: If use_search_history is not one of the valid options + """ + if not isinstance(use_search_history, str): + raise TypeError("use_search_history must be a string") + valid_options = ["OFF", "ON", "AUTO"] + if use_search_history not in valid_options: + raise ValueError( + f"use_search_history must be one of {', '.join(valid_options)}" + ) + self._use_search_history = use_search_history + + # Invalidate the query string + self._built_query_string = None + + def set_search_buffer_capacity(self, search_buffer_capacity: int): + """Set the SEARCH_BUFFER_CAPACITY parameter for the range query. + + Args: + search_buffer_capacity (int): Tuning parameter for SVS-VAMANA indexes using + two-level compression. + + Raises: + TypeError: If search_buffer_capacity is not an integer + ValueError: If search_buffer_capacity is not positive + """ + if not isinstance(search_buffer_capacity, int): + raise TypeError("search_buffer_capacity must be an integer") + if search_buffer_capacity <= 0: + raise ValueError("search_buffer_capacity must be positive") + self._search_buffer_capacity = search_buffer_capacity + + # Invalidate the query string + self._built_query_string = None + def set_hybrid_policy(self, hybrid_policy: str): """Set the hybrid policy for the query. @@ -904,9 +1196,24 @@ def _build_query_string(self) -> str: attr_parts = [] attr_parts.append(f"$YIELD_DISTANCE_AS: {self.DISTANCE_ID}") + # Add EPSILON parameter if specified (HNSW and SVS-VAMANA) if self._epsilon is not None: attr_parts.append(f"$EPSILON: {self._epsilon}") + # Add SEARCH_WINDOW_SIZE parameter if specified (SVS-VAMANA) + if self._search_window_size is not None: + attr_parts.append(f"$SEARCH_WINDOW_SIZE: {self._search_window_size}") + + # Add USE_SEARCH_HISTORY parameter if specified (SVS-VAMANA) + if self._use_search_history is not None: + attr_parts.append(f"$USE_SEARCH_HISTORY: {self._use_search_history}") + + # Add SEARCH_BUFFER_CAPACITY parameter if specified (SVS-VAMANA) + if self._search_buffer_capacity is not None: + attr_parts.append( + f"$SEARCH_BUFFER_CAPACITY: {self._search_buffer_capacity}" + ) + # Add query attributes section attr_section = f"=>{{{'; '.join(attr_parts)}}}" @@ -937,6 +1244,33 @@ def epsilon(self) -> Optional[float]: """ return self._epsilon + @property + def search_window_size(self) -> Optional[int]: + """Return the SEARCH_WINDOW_SIZE parameter for the query. + + Returns: + Optional[int]: The SEARCH_WINDOW_SIZE value for the query. + """ + return self._search_window_size + + @property + def use_search_history(self) -> Optional[str]: + """Return the USE_SEARCH_HISTORY parameter for the query. + + Returns: + Optional[str]: The USE_SEARCH_HISTORY value for the query. + """ + return self._use_search_history + + @property + def search_buffer_capacity(self) -> Optional[int]: + """Return the SEARCH_BUFFER_CAPACITY parameter for the query. + + Returns: + Optional[int]: The SEARCH_BUFFER_CAPACITY value for the query. + """ + return self._search_buffer_capacity + @property def hybrid_policy(self) -> Optional[str]: """Return the hybrid policy for the query. diff --git a/tests/unit/test_aggregation_types.py b/tests/unit/test_aggregation_types.py index 49eb1529..22e89975 100644 --- a/tests/unit/test_aggregation_types.py +++ b/tests/unit/test_aggregation_types.py @@ -245,6 +245,12 @@ def test_hybrid_query_text_weights(): text_weights={"first": 0.2, "second": -0.1}, ) + +def test_aggregate_hybrid_query_text_weights_validation(): + """Test that AggregateHybridQuery validates text_weights properly.""" + vector = [0.1, 0.2, 0.3, 0.4] + vector_field = "embedding" + with pytest.raises(ValueError): _ = AggregateHybridQuery( text="sample text query", diff --git a/tests/unit/test_query_types.py b/tests/unit/test_query_types.py index 7cadaff4..5f37f96c 100644 --- a/tests/unit/test_query_types.py +++ b/tests/unit/test_query_types.py @@ -754,6 +754,122 @@ def test_vector_range_query_error_handling(): # Removed: Test invalid ef_runtime since VectorRangeQuery doesn't use EF_RUNTIME +def test_vector_range_query_search_window_size(): + """Test that VectorRangeQuery correctly handles search_window_size parameter (SVS-VAMANA).""" + # Create a range query with search_window_size + range_query = VectorRangeQuery( + [0.1, 0.2, 0.3, 0.4], + "vector_field", + distance_threshold=0.3, + search_window_size=40, + ) + + # Check properties + assert range_query.search_window_size == 40 + + # Check query string + query_string = str(range_query) + assert "$SEARCH_WINDOW_SIZE: 40" in query_string + + +def test_vector_range_query_invalid_search_window_size(): + """Test error handling for invalid search_window_size values in VectorRangeQuery.""" + # Test with invalid type + with pytest.raises(TypeError, match="search_window_size must be an integer"): + VectorRangeQuery( + [0.1, 0.2, 0.3, 0.4], + "vector_field", + distance_threshold=0.3, + search_window_size="40", + ) + + # Test with invalid value + with pytest.raises(ValueError, match="search_window_size must be positive"): + VectorRangeQuery( + [0.1, 0.2, 0.3, 0.4], + "vector_field", + distance_threshold=0.3, + search_window_size=0, + ) + + +def test_vector_range_query_use_search_history(): + """Test that VectorRangeQuery correctly handles use_search_history parameter (SVS-VAMANA).""" + # Test with valid values + for value in ["OFF", "ON", "AUTO"]: + range_query = VectorRangeQuery( + [0.1, 0.2, 0.3, 0.4], + "vector_field", + distance_threshold=0.3, + use_search_history=value, + ) + + # Check properties + assert range_query.use_search_history == value + + # Check query string + query_string = str(range_query) + assert f"$USE_SEARCH_HISTORY: {value}" in query_string + + +def test_vector_range_query_invalid_use_search_history(): + """Test error handling for invalid use_search_history values in VectorRangeQuery.""" + # Test with invalid value + with pytest.raises( + ValueError, match="use_search_history must be one of OFF, ON, AUTO" + ): + VectorRangeQuery( + [0.1, 0.2, 0.3, 0.4], + "vector_field", + distance_threshold=0.3, + use_search_history="INVALID", + ) + + +def test_vector_range_query_search_buffer_capacity(): + """Test that VectorRangeQuery correctly handles search_buffer_capacity parameter (SVS-VAMANA).""" + # Create a range query with search_buffer_capacity + range_query = VectorRangeQuery( + [0.1, 0.2, 0.3, 0.4], + "vector_field", + distance_threshold=0.3, + search_buffer_capacity=50, + ) + + # Check properties + assert range_query.search_buffer_capacity == 50 + + # Check query string + query_string = str(range_query) + assert "$SEARCH_BUFFER_CAPACITY: 50" in query_string + + +def test_vector_range_query_all_svs_params(): + """Test VectorRangeQuery with all SVS-VAMANA runtime parameters.""" + range_query = VectorRangeQuery( + [0.1, 0.2, 0.3, 0.4], + "vector_field", + distance_threshold=0.3, + epsilon=0.05, + search_window_size=40, + use_search_history="ON", + search_buffer_capacity=50, + ) + + # Check all properties + assert range_query.epsilon == 0.05 + assert range_query.search_window_size == 40 + assert range_query.use_search_history == "ON" + assert range_query.search_buffer_capacity == 50 + + # Check query string contains all parameters + query_string = str(range_query) + assert "$EPSILON: 0.05" in query_string + assert "$SEARCH_WINDOW_SIZE: 40" in query_string + assert "$USE_SEARCH_HISTORY: ON" in query_string + assert "$SEARCH_BUFFER_CAPACITY: 50" in query_string + + def test_vector_query_ef_runtime(): """Test that VectorQuery correctly handles EF_RUNTIME parameter.""" # Create a vector query with ef_runtime @@ -848,3 +964,213 @@ def test_vector_query_update_ef_runtime(): assert f"{VectorQuery.EF_RUNTIME} ${VectorQuery.EF_RUNTIME_PARAM}" in qs2 params2 = vq.params assert params2.get(VectorQuery.EF_RUNTIME_PARAM) == 200 + + +def test_vector_query_epsilon(): + """Test that VectorQuery correctly handles epsilon parameter.""" + # Create a vector query with epsilon + vector_query = VectorQuery([0.1, 0.2, 0.3, 0.4], "vector_field", epsilon=0.05) + + # Check properties + assert vector_query.epsilon == 0.05 + + # Check query string + query_string = str(vector_query) + assert f"EPSILON ${VectorQuery.EPSILON_PARAM}" in query_string + + # Check params dictionary + assert vector_query.params.get(VectorQuery.EPSILON_PARAM) == 0.05 + + +def test_vector_query_invalid_epsilon(): + """Test error handling for invalid epsilon values in VectorQuery.""" + # Test with invalid epsilon type + with pytest.raises(TypeError, match="epsilon must be of type float or int"): + VectorQuery([0.1, 0.2, 0.3, 0.4], "vector_field", epsilon="0.05") + + # Test with negative epsilon + with pytest.raises(ValueError, match="epsilon must be non-negative"): + VectorQuery([0.1, 0.2, 0.3, 0.4], "vector_field", epsilon=-0.05) + + # Create a valid vector query + vector_query = VectorQuery([0.1, 0.2, 0.3, 0.4], "vector_field") + + # Test with invalid epsilon via setter + with pytest.raises(TypeError, match="epsilon must be of type float or int"): + vector_query.set_epsilon("0.05") + + with pytest.raises(ValueError, match="epsilon must be non-negative"): + vector_query.set_epsilon(-0.05) + + +def test_vector_query_search_window_size(): + """Test that VectorQuery correctly handles search_window_size parameter (SVS-VAMANA).""" + # Create a vector query with search_window_size + vector_query = VectorQuery( + [0.1, 0.2, 0.3, 0.4], "vector_field", search_window_size=40 + ) + + # Check properties + assert vector_query.search_window_size == 40 + + # Check query string + query_string = str(vector_query) + assert ( + f"{VectorQuery.SEARCH_WINDOW_SIZE} ${VectorQuery.SEARCH_WINDOW_SIZE_PARAM}" + in query_string + ) + + # Check params dictionary + assert vector_query.params.get(VectorQuery.SEARCH_WINDOW_SIZE_PARAM) == 40 + + +def test_vector_query_invalid_search_window_size(): + """Test error handling for invalid search_window_size values.""" + # Test with invalid type + with pytest.raises(TypeError, match="search_window_size must be an integer"): + VectorQuery([0.1, 0.2, 0.3, 0.4], "vector_field", search_window_size="40") + + # Test with invalid value + with pytest.raises(ValueError, match="search_window_size must be positive"): + VectorQuery([0.1, 0.2, 0.3, 0.4], "vector_field", search_window_size=0) + + with pytest.raises(ValueError, match="search_window_size must be positive"): + VectorQuery([0.1, 0.2, 0.3, 0.4], "vector_field", search_window_size=-10) + + # Create a valid vector query + vector_query = VectorQuery([0.1, 0.2, 0.3, 0.4], "vector_field") + + # Test with invalid search_window_size via setter + with pytest.raises(TypeError, match="search_window_size must be an integer"): + vector_query.set_search_window_size("40") + + with pytest.raises(ValueError, match="search_window_size must be positive"): + vector_query.set_search_window_size(0) + + +def test_vector_query_use_search_history(): + """Test that VectorQuery correctly handles use_search_history parameter (SVS-VAMANA).""" + # Test with valid values + for value in ["OFF", "ON", "AUTO"]: + vector_query = VectorQuery( + [0.1, 0.2, 0.3, 0.4], "vector_field", use_search_history=value + ) + + # Check properties + assert vector_query.use_search_history == value + + # Check query string + query_string = str(vector_query) + assert ( + f"{VectorQuery.USE_SEARCH_HISTORY} ${VectorQuery.USE_SEARCH_HISTORY_PARAM}" + in query_string + ) + + # Check params dictionary + assert vector_query.params.get(VectorQuery.USE_SEARCH_HISTORY_PARAM) == value + + +def test_vector_query_invalid_use_search_history(): + """Test error handling for invalid use_search_history values.""" + # Test with invalid type + with pytest.raises(TypeError, match="use_search_history must be a string"): + VectorQuery([0.1, 0.2, 0.3, 0.4], "vector_field", use_search_history=123) + + # Test with invalid value + with pytest.raises( + ValueError, match="use_search_history must be one of OFF, ON, AUTO" + ): + VectorQuery([0.1, 0.2, 0.3, 0.4], "vector_field", use_search_history="INVALID") + + # Create a valid vector query + vector_query = VectorQuery([0.1, 0.2, 0.3, 0.4], "vector_field") + + # Test with invalid use_search_history via setter + with pytest.raises(TypeError, match="use_search_history must be a string"): + vector_query.set_use_search_history(123) + + with pytest.raises( + ValueError, match="use_search_history must be one of OFF, ON, AUTO" + ): + vector_query.set_use_search_history("MAYBE") + + +def test_vector_query_search_buffer_capacity(): + """Test that VectorQuery correctly handles search_buffer_capacity parameter (SVS-VAMANA).""" + # Create a vector query with search_buffer_capacity + vector_query = VectorQuery( + [0.1, 0.2, 0.3, 0.4], "vector_field", search_buffer_capacity=50 + ) + + # Check properties + assert vector_query.search_buffer_capacity == 50 + + # Check query string + query_string = str(vector_query) + assert ( + f"{VectorQuery.SEARCH_BUFFER_CAPACITY} ${VectorQuery.SEARCH_BUFFER_CAPACITY_PARAM}" + in query_string + ) + + # Check params dictionary + assert vector_query.params.get(VectorQuery.SEARCH_BUFFER_CAPACITY_PARAM) == 50 + + +def test_vector_query_invalid_search_buffer_capacity(): + """Test error handling for invalid search_buffer_capacity values.""" + # Test with invalid type + with pytest.raises(TypeError, match="search_buffer_capacity must be an integer"): + VectorQuery([0.1, 0.2, 0.3, 0.4], "vector_field", search_buffer_capacity="50") + + # Test with invalid value + with pytest.raises(ValueError, match="search_buffer_capacity must be positive"): + VectorQuery([0.1, 0.2, 0.3, 0.4], "vector_field", search_buffer_capacity=0) + + with pytest.raises(ValueError, match="search_buffer_capacity must be positive"): + VectorQuery([0.1, 0.2, 0.3, 0.4], "vector_field", search_buffer_capacity=-10) + + # Create a valid vector query + vector_query = VectorQuery([0.1, 0.2, 0.3, 0.4], "vector_field") + + # Test with invalid search_buffer_capacity via setter + with pytest.raises(TypeError, match="search_buffer_capacity must be an integer"): + vector_query.set_search_buffer_capacity("50") + + with pytest.raises(ValueError, match="search_buffer_capacity must be positive"): + vector_query.set_search_buffer_capacity(0) + + +def test_vector_query_all_runtime_params(): + """Test VectorQuery with all runtime parameters combined (HNSW + SVS-VAMANA).""" + vector_query = VectorQuery( + [0.1, 0.2, 0.3, 0.4], + "vector_field", + ef_runtime=100, + epsilon=0.05, + search_window_size=40, + use_search_history="ON", + search_buffer_capacity=50, + ) + + # Check all properties + assert vector_query.ef_runtime == 100 + assert vector_query.epsilon == 0.05 + assert vector_query.search_window_size == 40 + assert vector_query.use_search_history == "ON" + assert vector_query.search_buffer_capacity == 50 + + # Check query string contains all parameters + query_string = str(vector_query) + assert "EF_RUNTIME $EF" in query_string + assert "EPSILON $EPSILON" in query_string + assert "SEARCH_WINDOW_SIZE $SEARCH_WINDOW_SIZE" in query_string + assert "USE_SEARCH_HISTORY $USE_SEARCH_HISTORY" in query_string + assert "SEARCH_BUFFER_CAPACITY $SEARCH_BUFFER_CAPACITY" in query_string + + # Check params dictionary contains all parameters + params = vector_query.params + assert params["EF"] == 100 + assert params["EPSILON"] == 0.05 + assert params["SEARCH_WINDOW_SIZE"] == 40 + assert params["USE_SEARCH_HISTORY"] == "ON" + assert params["SEARCH_BUFFER_CAPACITY"] == 50