Search: allow deleting search index for a project (#12584)

This allows us to mark a project so it's never indexed for search.
Currently, we will mark these projects manually and create the
notification so users are aware of this change and can re-enable search
indexing in case we made a mistake without contacting support.

ref
readthedocs/meta#197 (comment)

---------

Co-authored-by: Copilot <[email protected]>

Author: stsewd

readthedocs/builds/querysets.py

@@ -131,13 +131,15 @@ def for_reindex(self):
         - It's active and has been built at least once successfully.
           Since that means that it has files to be indexed.
         - Its project is not delisted or marked as spam.
+        - Its project has search indexing enabled.
         """
         return (
             self.filter(
                 active=True,
                 built=True,
                 builds__state=BUILD_STATE_FINISHED,
                 builds__success=True,
+                project__search_indexing_enabled=True,
             )
             .exclude(project__delisted=True)
             .exclude(project__is_spam=True)

readthedocs/projects/forms.py

@@ -25,6 +25,7 @@
 from readthedocs.core.utils.extend import SettingsOverrideObject
 from readthedocs.integrations.models import Integration
 from readthedocs.invitations.models import Invitation
+from readthedocs.notifications.models import Notification
 from readthedocs.oauth.models import RemoteRepository
 from readthedocs.organizations.models import Team
 from readthedocs.projects.constants import ADDONS_FLYOUT_SORTING_CUSTOM_PATTERN
@@ -36,6 +37,8 @@
 from readthedocs.projects.models import Project
 from readthedocs.projects.models import ProjectRelationship
 from readthedocs.projects.models import WebHook
+from readthedocs.projects.notifications import MESSAGE_PROJECT_SEARCH_INDEXING_DISABLED
+from readthedocs.projects.tasks.search import index_project
 from readthedocs.projects.templatetags.projects_tags import sort_version_aware
 from readthedocs.redirects.models import Redirect
 
@@ -431,6 +434,7 @@ class Meta:
             "versioning_scheme",
             "default_branch",
             "readthedocs_yaml_path",
+            "search_indexing_enabled",
             # Meta data
             "programming_language",
             "project_url",
@@ -452,6 +456,8 @@ class Meta:
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
+        self.had_search_disabled = not self.instance.search_indexing_enabled
+
         # Remove empty choice from options.
         self.fields["versioning_scheme"].choices = [
             (key, value) for key, value in self.fields["versioning_scheme"].choices if key
@@ -467,6 +473,11 @@ def __init__(self, *args, **kwargs):
             )
             self.fields["versioning_scheme"].disabled = True
 
+        # Only show this field if search is disabled for the project.
+        # We allow enabling it from the form, but not disabling it.
+        if self.instance.search_indexing_enabled:
+            self.fields.pop("search_indexing_enabled")
+
         # NOTE: we are deprecating this feature.
         # However, we will keep it available for projects that already using it.
         # Old projects not using it already or new projects won't be able to enable.
@@ -565,6 +576,17 @@ def clean_tags(self):
                 )
         return tags
 
+    def save(self, commit=True):
+        instance = super().save(commit)
+        # Trigger a reindex when enabling search from the form.
+        if self.had_search_disabled and instance.search_indexing_enabled:
+            index_project.delay(project_slug=instance.slug)
+            Notification.objects.cancel(
+                message_id=MESSAGE_PROJECT_SEARCH_INDEXING_DISABLED,
+                attached_to=instance,
+            )
+        return instance
+
 
 class ProjectRelationshipForm(forms.ModelForm):
     """Form to add/update project relationships."""

readthedocs/projects/migrations/0156_project_search_indexing_enabled.py

@@ -0,0 +1,36 @@
+# Generated by Django 5.2.7 on 2025-11-19 19:47
+
+from django.db import migrations
+from django.db import models
+from django_safemigrate import Safe
+
+
+class Migration(migrations.Migration):
+    safe = Safe.before_deploy()
+
+    dependencies = [
+        ("projects", "0155_custom_git_checkout_step"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="historicalproject",
+            name="search_indexing_enabled",
+            field=models.BooleanField(
+                db_default=True,
+                default=True,
+                help_text="Enable/disable search indexing for this project",
+                verbose_name="Enable search indexing",
+            ),
+        ),
+        migrations.AddField(
+            model_name="project",
+            name="search_indexing_enabled",
+            field=models.BooleanField(
+                db_default=True,
+                default=True,
+                help_text="Enable/disable search indexing for this project",
+                verbose_name="Enable search indexing",
+            ),
+        ),
+    ]

readthedocs/projects/models.py

@@ -538,6 +538,13 @@ class Project(models.Model):
         ),
     )
 
+    search_indexing_enabled = models.BooleanField(
+        _("Enable search indexing"),
+        default=True,
+        db_default=True,
+        help_text=_("Enable/disable search indexing for this project"),
+    )
+
     privacy_level = models.CharField(
         _("Privacy Level"),
         max_length=20,

readthedocs/projects/notifications.py

@@ -19,6 +19,7 @@
 MESSAGE_PROJECT_ADDONS_BY_DEFAULT = "project:addons:by-default"
 MESSAGE_PROJECT_SSH_KEY_WITH_WRITE_ACCESS = "project:ssh-key-with-write-access"
 MESSAGE_PROJECT_DEPRECATED_WEBHOOK = "project:webhooks:deprecated"
+MESSAGE_PROJECT_SEARCH_INDEXING_DISABLED = "project:search:indexing-disabled"
 
 messages = [
     Message(
@@ -208,5 +209,19 @@
         ),
         type=INFO,
     ),
+    # NOTE: Disabling search for projects and notifying users is done manually for now.
+    Message(
+        id=MESSAGE_PROJECT_SEARCH_INDEXING_DISABLED,
+        header=_("Search indexing has been disabled for this project"),
+        body=_(
+            textwrap.dedent(
+                """
+                Your project hasn't received any searches recently, to optimize resources, we've disabled search indexing for this project.
+                If you want to re-enable search indexing, check the "Enable search indexing" option from <a href="{% url 'projects_edit' instance.slug %}">the project settings</a>.
+                """
+            ).strip(),
+        ),
+        type=INFO,
+    ),
 ]
 registry.add(messages)

readthedocs/projects/tasks/search.py

@@ -161,8 +161,16 @@ def _get_indexers(*, version: Version, build: Build, search_index_name=None):
     # This is because saving the objects in the DB will give them an id,
     # and we neeed this id to be `None` when indexing the objects in ES.
     # ES will generate a unique id for each document.
-    # NOTE: If the version is external, we don't create a search index for it.
-    if not version.is_external:
+    # NOTE: We don't create a search indexer for:
+    # - External versions
+    # - Versions from projects with search indexing disabled
+    # - Versions from delisted projects
+    skip_search_indexing = (
+        not version.project.search_indexing_enabled
+        or version.is_external
+        or version.project.delisted
+    )
+    if not skip_search_indexing:
         search_indexer = SearchIndexer(
             project=version.project,
             version=version,

readthedocs/projects/tests/test_indexers.py

@@ -0,0 +1,72 @@
+from django.test import TestCase
+from django_dynamic_fixture import get
+
+from readthedocs.builds.constants import BUILD_STATE_FINISHED, EXTERNAL
+from readthedocs.builds.models import Build, Version
+from readthedocs.projects.models import Project
+from readthedocs.projects.tasks.search import SearchIndexer, _get_indexers
+
+
+class TestSearchIndexing(TestCase):
+    """Tests for search_indexing_enabled field behavior."""
+
+    def test_search_indexer_not_created_when_disabled(self):
+        project = get(Project, search_indexing_enabled=False)
+        version = project.versions.first()
+        build = get(Build, version=version, state=BUILD_STATE_FINISHED, success=True)
+
+        indexers = _get_indexers(version=version, build=build)
+
+        # Check that no SearchIndexer is in the list
+        search_indexers = [
+            indexer for indexer in indexers if isinstance(indexer, SearchIndexer)
+        ]
+        assert len(search_indexers) == 0
+
+    def test_search_indexer_created_when_enabled(self):
+        """Test that SearchIndexer is created when search_indexing_enabled is True."""
+        project = get(
+            Project,
+            search_indexing_enabled=True,
+        )
+        version = project.versions.first()
+        build = get(Build, version=version, state=BUILD_STATE_FINISHED, success=True)
+
+        indexers = _get_indexers(version=version, build=build)
+
+        # Check that SearchIndexer is in the list
+        search_indexers = [
+            indexer for indexer in indexers if isinstance(indexer, SearchIndexer)
+        ]
+        assert len(search_indexers) == 1
+
+    def test_search_indexer_not_created_for_delisted_project(self):
+        """Test that SearchIndexer is not created for delisted projects."""
+        project = get(
+            Project,
+            delisted=True,
+        )
+        version = project.versions.first()
+        build = get(Build, version=version, state=BUILD_STATE_FINISHED, success=True)
+
+        indexers = _get_indexers(version=version, build=build)
+
+        # Check that no SearchIndexer is in the list
+        search_indexers = [
+            indexer for indexer in indexers if isinstance(indexer, SearchIndexer)
+        ]
+        assert len(search_indexers) == 0
+
+    def test_search_indexer_not_created_for_external_version(self):
+        """Test that SearchIndexer is not created for external versions."""
+        project = get(Project)
+        version = get(Version, project=project, slug="123", built=True, type=EXTERNAL)
+        build = get(Build, version=version, state=BUILD_STATE_FINISHED, success=True)
+
+        indexers = _get_indexers(version=version, build=build)
+
+        # Check that no SearchIndexer is in the list
+        search_indexers = [
+            indexer for indexer in indexers if isinstance(indexer, SearchIndexer)
+        ]
+        assert len(search_indexers) == 0

readthedocs/projects/tests/test_views.py

@@ -1,5 +1,6 @@
 from allauth.account.models import EmailAddress
 from django.contrib.auth.models import User
+from unittest import mock
 from django.contrib.messages import get_messages
 from django.test import TestCase, override_settings
 from django.urls import reverse
@@ -317,3 +318,55 @@ def test_download_files(self):
                 resp["X-Accel-Redirect"],
                 f"/proxito/media/{type_}/project/latest/project.{extension}",
             )
+
+
+class TestProjectEditView(TestCase):
+    def setUp(self):
+        self.user = get(User)
+        self.project = get(Project, slug="project", users=[self.user], repo="https://github.com/user/repo")
+        self.url = reverse("projects_edit", args=[self.project.slug])
+        self.client.force_login(self.user)
+
+    @mock.patch("readthedocs.projects.forms.trigger_build")
+    @mock.patch("readthedocs.projects.forms.index_project")
+    def test_search_indexing_enabled(self, index_project, trigger_build):
+        resp = self.client.get(self.url)
+        assert resp.status_code == 200
+        form = resp.context["form"]
+        assert "search_indexing_enabled" not in form.fields
+
+        self.project.search_indexing_enabled = False
+        self.project.save()
+
+        resp = self.client.get(self.url)
+        assert resp.status_code == 200
+        form = resp.context["form"]
+        assert "search_indexing_enabled" in form.fields
+
+        data = {
+            "name": self.project.name,
+            "repo": self.project.repo,
+            "language": self.project.language,
+            "default_version": self.project.default_version,
+            "versioning_scheme": self.project.versioning_scheme,
+        }
+
+        data["search_indexing_enabled"] = False
+        resp = self.client.post(
+            self.url,
+            data=data,
+        )
+        assert resp.status_code == 302
+        self.project.refresh_from_db()
+        assert not self.project.search_indexing_enabled
+        index_project.delay.assert_not_called()
+
+        data["search_indexing_enabled"] = True
+        resp = self.client.post(
+            self.url,
+            data=data,
+        )
+        assert resp.status_code == 302
+        self.project.refresh_from_db()
+        assert self.project.search_indexing_enabled
+        index_project.delay.assert_called_once_with(project_slug=self.project.slug)

readthedocs/search/documents.py

@@ -59,7 +59,13 @@ def get_queryset(self):
         but it's not a priority to find a solution for this as long as "delisted" projects are
         understood to be projects with a negative reason for being delisted.
         """
-        return super().get_queryset().exclude(delisted=True).exclude(is_spam=True)
+        return (
+            super()
+            .get_queryset()
+            .filter(search_indexing_enabled=True)
+            .exclude(delisted=True)
+            .exclude(is_spam=True)
+        )
 
     class Django:
         model = Project
@@ -122,7 +128,8 @@ def get_queryset(self):
         """Don't include ignored files and delisted projects."""
         queryset = super().get_queryset()
         queryset = (
-            queryset.exclude(ignore=True)
+            queryset.filter(project__search_indexing_enabled=True)
+            .exclude(ignore=True)
             .exclude(project__delisted=True)
             .exclude(project__is_spam=True)
             .select_related("version", "project")