refactor: unify rag storage with instance-specific client support (#3455)

- ignore line length errors globally
- migrate knowledge/memory and crew query_knowledge to `SearchResult`
- remove legacy chromadb utils; fix empty metadata handling
- restore openai as default embedding provider; support instance-specific clients
- update and fix tests for `SearchResult` migration and rag changes

Author: greysonlalonde

pyproject.toml

@@ -131,13 +131,14 @@ select = [
     "I001",   # sort imports
     "I002",   # remove unused imports
 ]
-ignore = ["E501"]  # ignore line too long
+ignore = ["E501"]  # ignore line too long globally
 
 [tool.ruff.lint.per-file-ignores]
-"tests/**/*.py" = ["S101"]  # Allow assert statements in tests
+"tests/**/*.py" = ["S101", "RET504"]  # Allow assert statements and unnecessary assignments before return in tests
 
 [tool.mypy]
-exclude = ["src/crewai/cli/templates", "tests"]
+exclude = ["src/crewai/cli/templates", "tests/"]
+
 
 [tool.bandit]
 exclude_dirs = ["src/crewai/cli/templates"]

src/crewai/crew.py

@@ -1,10 +1,11 @@
 import os
-from typing import Any, Dict, List, Optional
+from typing import Any
 
 from pydantic import BaseModel, ConfigDict, Field
 
 from crewai.knowledge.source.base_knowledge_source import BaseKnowledgeSource
 from crewai.knowledge.storage.knowledge_storage import KnowledgeStorage
+from crewai.rag.types import SearchResult
 
 os.environ["TOKENIZERS_PARALLELISM"] = "false"  # removes logging from fastembed
 
@@ -13,23 +14,23 @@ class Knowledge(BaseModel):
     """
     Knowledge is a collection of sources and setup for the vector store to save and query relevant context.
     Args:
-        sources: List[BaseKnowledgeSource] = Field(default_factory=list)
-        storage: Optional[KnowledgeStorage] = Field(default=None)
-        embedder: Optional[Dict[str, Any]] = None
+        sources: list[BaseKnowledgeSource] = Field(default_factory=list)
+        storage: KnowledgeStorage | None = Field(default=None)
+        embedder: dict[str, Any] | None = None
     """
 
-    sources: List[BaseKnowledgeSource] = Field(default_factory=list)
+    sources: list[BaseKnowledgeSource] = Field(default_factory=list)
     model_config = ConfigDict(arbitrary_types_allowed=True)
-    storage: Optional[KnowledgeStorage] = Field(default=None)
-    embedder: Optional[Dict[str, Any]] = None
-    collection_name: Optional[str] = None
+    storage: KnowledgeStorage | None = Field(default=None)
+    embedder: dict[str, Any] | None = None
+    collection_name: str | None = None
 
     def __init__(
         self,
         collection_name: str,
-        sources: List[BaseKnowledgeSource],
-        embedder: Optional[Dict[str, Any]] = None,
-        storage: Optional[KnowledgeStorage] = None,
+        sources: list[BaseKnowledgeSource],
+        embedder: dict[str, Any] | None = None,
+        storage: KnowledgeStorage | None = None,
         **data,
     ):
         super().__init__(**data)
@@ -40,11 +41,10 @@ def __init__(
                 embedder=embedder, collection_name=collection_name
             )
         self.sources = sources
-        self.storage.initialize_knowledge_storage()
 
     def query(
-        self, query: List[str], results_limit: int = 3, score_threshold: float = 0.35
-    ) -> List[Dict[str, Any]]:
+        self, query: list[str], results_limit: int = 3, score_threshold: float = 0.35
+    ) -> list[SearchResult]:
         """
         Query across all knowledge sources to find the most relevant information.
         Returns the top_k most relevant chunks.
@@ -55,12 +55,11 @@ def query(
         if self.storage is None:
             raise ValueError("Storage is not initialized.")
 
-        results = self.storage.search(
+        return self.storage.search(
             query,
             limit=results_limit,
             score_threshold=score_threshold,
         )
-        return results
 
     def add_sources(self):
         try:

src/crewai/knowledge/knowledge.py

@@ -1,5 +1,7 @@
 from abc import ABC, abstractmethod
-from typing import Any, Dict, List, Optional
+from typing import Any
+
+from crewai.rag.types import SearchResult
 
 
 class BaseKnowledgeStorage(ABC):
@@ -8,22 +10,17 @@ class BaseKnowledgeStorage(ABC):
     @abstractmethod
     def search(
         self,
-        query: List[str],
+        query: list[str],
         limit: int = 3,
-        filter: Optional[dict] = None,
+        metadata_filter: dict[str, Any] | None = None,
         score_threshold: float = 0.35,
-    ) -> List[Dict[str, Any]]:
+    ) -> list[SearchResult]:
         """Search for documents in the knowledge base."""
-        pass
 
     @abstractmethod
-    def save(
-        self, documents: List[str], metadata: Dict[str, Any] | List[Dict[str, Any]]
-    ) -> None:
+    def save(self, documents: list[str]) -> None:
         """Save documents to the knowledge base."""
-        pass
 
     @abstractmethod
     def reset(self) -> None:
         """Reset the knowledge base."""
-        pass