chore: abstract models and daos into superset-core (#35259)

Author: villebro

.gitignore

@@ -33,6 +33,7 @@ cover
 .env
 .envrc
 .idea
+.roo
 .mypy_cache
 .python-version
 .tox

docs/developer_portal/extensions/interacting-with-host.md

@@ -26,6 +26,8 @@ under the License.
 
 Extensions interact with Superset through well-defined, versioned APIs provided by the `@apache-superset/core` (frontend) and `apache-superset-core` (backend) packages. These APIs are designed to be stable, discoverable, and consistent for both built-in and external extensions.
 
+**Note**: The `superset_core.api` module provides abstract classes that are replaced with concrete implementations via dependency injection when Superset initializes. This allows extensions to use the same interfaces as the host application.
+
 **Frontend APIs** (via `@apache-superset/core)`:
 
 The frontend extension APIs in Superset are organized into logical namespaces such as `authentication`, `commands`, `extensions`, `sqlLab`, and others. Each namespace groups related functionality, making it easy for extension authors to discover and use the APIs relevant to their needs. For example, the `sqlLab` namespace provides events and methods specific to SQL Lab, allowing extensions to react to user actions and interact with the SQL Lab environment:
@@ -90,31 +92,38 @@ Backend APIs follow a similar pattern, providing access to Superset's models, se
 Extension endpoints are registered under a dedicated `/extensions` namespace to avoid conflicting with built-in endpoints and also because they don't share the same version constraints. By grouping all extension endpoints under `/extensions`, Superset establishes a clear boundary between core and extension functionality, making it easier to manage, document, and secure both types of APIs.
 
 ``` python
-from superset_core.api import rest_api, models, query
+from superset_core.api.models import Database, get_session
+from superset_core.api.daos import DatabaseDAO
+from superset_core.api.rest_api import add_extension_api
 from .api import DatasetReferencesAPI
 
 # Register a new extension REST API
-rest_api.add_extension_api(DatasetReferencesAPI)
+add_extension_api(DatasetReferencesAPI)
+
+# Fetch Superset entities via the DAO to apply base filters that filter out entities
+# that the user doesn't have access to
+databases = DatabaseDAO.find_all()
 
-# Access Superset models with simple queries that filter out entities that
-# the user doesn't have access to
-databases = models.get_databases(id=database_id)
+# ..or apply simple filters on top of base filters
+databases = DatabaseDAO.filter_by(uuid=database.uuid)
 if not databases:
-    return self.response_404()
+    raise Exception("Database not found")
 
-database = databases[0]
+return databases[0]
 
-# Perform complex queries using SQLAlchemy BaseQuery, also filtering
-# out inaccessible entities
-session = models.get_session()
-db_model = models.get_database_model())
-database_query = session.query(db_model.database_name.ilike("%abc%")
-databases_containing_abc = models.get_databases(query)
+# Perform complex queries using SQLAlchemy Query, also filtering out
+# inaccessible entities
+session = get_session()
+databases_query = session.query(Database).filter(
+    Database.database_name.ilike("%abc%")
+)
+return DatabaseDAO.query(databases_query)
 
 # Bypass security model for highly custom use cases
-session = models.get_session()
-db_model = models.get_database_model())
-all_databases_containg_abc = session.query(db_model.database_name.ilike("%abc%").all()
+session = get_session()
+all_databases_containing_abc = session.query(Database).filter(
+    Database.database_name.ilike("%abc%")
+).all()
 ```
 
 In the future, we plan to expand the backend APIs to support configuring security models, database engines, SQL Alchemy dialects, etc.

docs/developer_portal/extensions/quick-start.md

@@ -128,7 +128,7 @@ The CLI generated a basic `backend/src/hello_world/entrypoint.py`. We'll create
 ```python
 from flask import Response
 from flask_appbuilder.api import expose, protect, safe
-from superset_core.api.types.rest_api import RestApi
+from superset_core.api.rest_api import RestApi
 
 
 class HelloWorldAPI(RestApi):

superset-core/README.md

@@ -49,7 +49,7 @@ The package is organized into logical modules, each providing specific functiona
 from flask import request, Response
 from flask_appbuilder.api import expose, permission_name, protect, safe
 from superset_core.api import models, query, rest_api
-from superset_core.api.types.rest_api import RestApi
+from superset_core.api.rest_api import RestApi
 
 class DatasetReferencesAPI(RestApi):
     """Example extension API demonstrating core functionality."""

superset-core/pyproject.toml

@@ -43,6 +43,10 @@ classifiers = [
 ]
 dependencies = [
     "flask-appbuilder>=5.0.2,<6",
+    "sqlalchemy>=1.4.0,<2.0",
+    "sqlalchemy-utils>=0.38.0",
+    "sqlglot>=27.15.2, <28",
+    "typing-extensions>=4.0.0",
 ]
 
 [project.urls]

superset-core/src/superset_core/__init__.py

@@ -14,3 +14,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+
+"""
+Apache Superset Core - Public API with core functions of Superset
+"""

superset-core/src/superset_core/api/__init__.py

@@ -14,11 +14,3 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
-from .types.models import CoreModelsApi
-from .types.query import CoreQueryApi
-from .types.rest_api import CoreRestApi
-
-models: CoreModelsApi
-rest_api: CoreRestApi
-query: CoreQueryApi

superset-core/src/superset_core/api/daos.py

@@ -0,0 +1,262 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Data Access Object API for superset-core.
+
+Provides dependency-injected DAO classes that will be replaced by
+host implementations during initialization.
+
+Usage:
+    from superset_core.api.daos import DatasetDAO, DatabaseDAO
+
+    # Use standard BaseDAO methods
+    datasets = DatasetDAO.find_all()
+    dataset = DatasetDAO.find_one_or_none(id=123)
+    DatasetDAO.create(attributes={"name": "New Dataset"})
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any, ClassVar, Generic, TypeVar
+
+from flask_appbuilder.models.filters import BaseFilter
+from sqlalchemy.orm import Query as SQLAQuery
+
+from superset_core.api.models import (
+    Chart,
+    CoreModel,
+    Dashboard,
+    Database,
+    Dataset,
+    KeyValue,
+    Query,
+    SavedQuery,
+    Tag,
+    User,
+)
+
+# Type variable bound to our CoreModel
+T = TypeVar("T", bound=CoreModel)
+
+
+class BaseDAO(Generic[T], ABC):
+    """
+    Abstract base class for DAOs.
+
+    This ABC defines the base that all DAOs should implement,
+    providing consistent CRUD operations across Superset and extensions.
+    """
+
+    # Due to mypy limitations, we can't have `type[T]` here
+    model_cls: ClassVar[type[Any] | None]
+    base_filter: ClassVar[BaseFilter | None]
+    id_column_name: ClassVar[str]
+    uuid_column_name: ClassVar[str]
+
+    @classmethod
+    @abstractmethod
+    def find_all(cls) -> list[T]:
+        """Get all entities that fit the base_filter."""
+        ...
+
+    @classmethod
+    @abstractmethod
+    def find_one_or_none(cls, **filter_by: Any) -> T | None:
+        """Get the first entity that fits the base_filter."""
+        ...
+
+    @classmethod
+    @abstractmethod
+    def create(
+        cls,
+        item: T | None = None,
+        attributes: dict[str, Any] | None = None,
+    ) -> T:
+        """Create an object from the specified item and/or attributes."""
+        ...
+
+    @classmethod
+    @abstractmethod
+    def update(
+        cls,
+        item: T | None = None,
+        attributes: dict[str, Any] | None = None,
+    ) -> T:
+        """Update an object from the specified item and/or attributes."""
+        ...
+
+    @classmethod
+    @abstractmethod
+    def delete(cls, items: list[T]) -> None:
+        """Delete the specified items."""
+        ...
+
+    @classmethod
+    @abstractmethod
+    def query(cls, query: SQLAQuery) -> list[T]:
+        """Execute query with base_filter applied."""
+        ...
+
+    @classmethod
+    @abstractmethod
+    def filter_by(cls, **filter_by: Any) -> list[T]:
+        """Get all entries that fit the base_filter."""
+        ...
+
+
+class DatasetDAO(BaseDAO[Dataset]):
+    """
+    Abstract Dataset DAO interface.
+
+    Host implementations will replace this class during initialization
+    with a concrete implementation providing actual functionality.
+    """
+
+    # Class variables that will be set by host implementation
+    model_cls = None
+    base_filter = None
+    id_column_name = "id"
+    uuid_column_name = "uuid"
+
+
+class DatabaseDAO(BaseDAO[Database]):
+    """
+    Abstract Database DAO interface.
+
+    Host implementations will replace this class during initialization
+    with a concrete implementation providing actual functionality.
+    """
+
+    # Class variables that will be set by host implementation
+    model_cls = None
+    base_filter = None
+    id_column_name = "id"
+    uuid_column_name = "uuid"
+
+
+class ChartDAO(BaseDAO[Chart]):
+    """
+    Abstract Chart DAO interface.
+
+    Host implementations will replace this class during initialization
+    with a concrete implementation providing actual functionality.
+    """
+
+    # Class variables that will be set by host implementation
+    model_cls = None
+    base_filter = None
+    id_column_name = "id"
+    uuid_column_name = "uuid"
+
+
+class DashboardDAO(BaseDAO[Dashboard]):
+    """
+    Abstract Dashboard DAO interface.
+
+    Host implementations will replace this class during initialization
+    with a concrete implementation providing actual functionality.
+    """
+
+    # Class variables that will be set by host implementation
+    model_cls = None
+    base_filter = None
+    id_column_name = "id"
+    uuid_column_name = "uuid"
+
+
+class UserDAO(BaseDAO[User]):
+    """
+    Abstract User DAO interface.
+
+    Host implementations will replace this class during initialization
+    with a concrete implementation providing actual functionality.
+    """
+
+    # Class variables that will be set by host implementation
+    model_cls = None
+    base_filter = None
+    id_column_name = "id"
+
+
+class QueryDAO(BaseDAO[Query]):
+    """
+    Abstract Query DAO interface.
+
+    Host implementations will replace this class during initialization
+    with a concrete implementation providing actual functionality.
+    """
+
+    # Class variables that will be set by host implementation
+    model_cls = None
+    base_filter = None
+    id_column_name = "id"
+
+
+class SavedQueryDAO(BaseDAO[SavedQuery]):
+    """
+    Abstract SavedQuery DAO interface.
+
+    Host implementations will replace this class during initialization
+    with a concrete implementation providing actual functionality.
+    """
+
+    # Class variables that will be set by host implementation
+    model_cls = None
+    base_filter = None
+    id_column_name = "id"
+
+
+class TagDAO(BaseDAO[Tag]):
+    """
+    Abstract Tag DAO interface.
+
+    Host implementations will replace this class during initialization
+    with a concrete implementation providing actual functionality.
+    """
+
+    # Class variables that will be set by host implementation
+    model_cls = None
+    base_filter = None
+    id_column_name = "id"
+
+
+class KeyValueDAO(BaseDAO[KeyValue]):
+    """
+    Abstract KeyValue DAO interface.
+
+    Host implementations will replace this class during initialization
+    with a concrete implementation providing actual functionality.
+    """
+
+    # Class variables that will be set by host implementation
+    model_cls = None
+    base_filter = None
+    id_column_name = "id"
+
+
+__all__ = [
+    "BaseDAO",
+    "DatasetDAO",
+    "DatabaseDAO",
+    "ChartDAO",
+    "DashboardDAO",
+    "UserDAO",
+    "QueryDAO",
+    "SavedQueryDAO",
+    "TagDAO",
+    "KeyValueDAO",
+]