Storages: Deprecated libclara for normal build (#10325)

CMakeLists.txt

@@ -44,6 +44,8 @@ include (cmake/utils.cmake)
 
 option(ENABLE_PCH "Enable `Precompiled header`" OFF)
 
+option(ENABLE_CLARA "Enable `Clara` library" OFF)
+
 include (cmake/find_ccache.cmake)
 
 # Write compile_commands.json
@@ -312,7 +314,9 @@ include (cmake/find_tipb.cmake)
 include (cmake/find_curl.cmake)
 include (cmake/find_prometheus.cmake)
 include (cmake/find_tiflash_proxy.cmake)
-include (cmake/find_libclara.cmake)
+if (ENABLE_CLARA)
+    include (cmake/find_libclara.cmake)
+endif ()
 include (cmake/find_xxhash.cmake)
 
 if (OS_LINUX AND ARCH_AMD64 AND TIFLASH_ENABLE_AVX512_SUPPORT)

dbms/CMakeLists.txt

@@ -224,7 +224,6 @@ target_link_libraries (dbms
     absl::synchronization
     tiflash_contrib::aws_s3
     tiflash_vector_search
-    clara_shared
 
     etcdpb
     tiflash_parsers
@@ -240,6 +239,10 @@ target_link_libraries (dbms
     kvstore
 )
 
+if (ENABLE_CLARA)
+    target_link_libraries(dbms clara_shared)
+endif ()
+
 # always add GmSSL include dir to the include path for static analysis
 target_include_directories(dbms PRIVATE ${TiFlash_SOURCE_DIR}/contrib/GmSSL/include)
 if (USE_GM_SSL)
@@ -374,7 +377,7 @@ if (ENABLE_TESTS)
                 DESTINATION ".")
     endif ()
 
-    if (USE_INTERNAL_LIBCLARA)
+    if (ENABLE_CLARA AND USE_INTERNAL_LIBCLARA)
         install (TARGETS clara_shared
                 COMPONENT tiflash-gtest
                 DESTINATION ".")
@@ -383,7 +386,9 @@ if (ENABLE_TESTS)
     set_target_properties(gtests_dbms PROPERTIES BUILD_RPATH "$ORIGIN/")
     set_target_properties(gtests_dbms PROPERTIES INSTALL_RPATH "$ORIGIN/")
 
-    install (SCRIPT ${TiFlash_SOURCE_DIR}/libs/libclara-cmake/linux_post_install.cmake COMPONENT tiflash-gtest)
+    if (ENABLE_CLARA)
+        install (SCRIPT ${TiFlash_SOURCE_DIR}/libs/libclara-cmake/linux_post_install.cmake COMPONENT tiflash-gtest)
+    endif ()
 
     target_compile_options(gtests_dbms PRIVATE -Wno-unknown-pragmas -Wno-deprecated-copy)
     add_check(gtests_dbms)

dbms/src/Common/TiFlashBuildInfo.cpp

@@ -146,6 +146,11 @@ String getEnabledFeatures()
 #if ENABLE_NEXT_GEN
             "next-gen", 
 #endif
+
+// Clara
+#if ENABLE_CLARA
+            "clara",
+#endif
     };
     {
         auto f = DB::DM::VectorIndexHNSWSIMDFeatures::get();

dbms/src/Common/config.h.in

@@ -7,3 +7,4 @@
 #cmakedefine01 USE_GM_SSL
 #cmakedefine01 USE_QPL
 #cmakedefine01 ENABLE_NEXT_GEN
+#cmakedefine01 ENABLE_CLARA

dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp

@@ -18,6 +18,7 @@
 #include <Common/Stopwatch.h>
 #include <Common/TiFlashException.h>
 #include <Common/TiFlashMetrics.h>
+#include <Common/config.h> // For ENABLE_CLARA
 #include <DataStreams/ExpressionBlockInputStream.h>
 #include <DataStreams/FilterBlockInputStream.h>
 #include <DataStreams/GeneratedColumnPlaceholderBlockInputStream.h>
@@ -41,7 +42,6 @@
 #include <Operators/NullSourceOp.h>
 #include <Operators/UnorderedSourceOp.h>
 #include <Parsers/makeDummyQuery.h>
-#include <Storages/DeltaMerge/Index/FullTextIndex/Stream/Ctx.h>
 #include <Storages/DeltaMerge/Index/VectorIndex/Stream/Ctx.h>
 #include <Storages/DeltaMerge/Remote/DisaggSnapshot.h>
 #include <Storages/DeltaMerge/Remote/WNDisaggSnapshotManager.h>
@@ -62,6 +62,10 @@
 #include <kvproto/coprocessor.pb.h>
 #include <tipb/select.pb.h>
 
+#if ENABLE_CLARA
+#include <Storages/DeltaMerge/Index/FullTextIndex/Stream/Ctx.h>
+#endif
+
 namespace DB
 {
 namespace FailPoints
@@ -1571,8 +1575,10 @@ std::pair<Names, std::vector<UInt8>> DAGStorageInterpreter::getColumnsForTableSc
             name = MutSup::extra_table_id_column_name;
         else if (cid == DM::VectorIndexStreamCtx::VIRTUAL_DISTANCE_CD.id)
             name = DM::VectorIndexStreamCtx::VIRTUAL_DISTANCE_CD.name;
+#if ENABLE_CLARA
         else if (cid == DM::FullTextIndexStreamCtx::VIRTUAL_SCORE_CD.id)
             name = DM::FullTextIndexStreamCtx::VIRTUAL_SCORE_CD.name;
+#endif
         else
             name = storage_for_logical_table->getTableInfo().getColumnName(cid);
         required_columns_tmp.emplace_back(std::move(name));

dbms/src/Server/CMakeLists.txt

@@ -124,7 +124,7 @@ if (OS_LINUX)
                 DESTINATION ".")
     endif()
 
-    if (USE_INTERNAL_LIBCLARA)
+    if (ENABLE_CLARA AND USE_INTERNAL_LIBCLARA)
         install (TARGETS clara_shared
                 COMPONENT tiflash-release
                 DESTINATION ".")
@@ -134,12 +134,14 @@ if (OS_LINUX)
         install (SCRIPT ${TiFlash_SOURCE_DIR}/cmake/tiflash_linux_post_install.cmake COMPONENT tiflash-release)
     endif()
     install (SCRIPT ${TiFlash_SOURCE_DIR}/contrib/tiflash-proxy-cmake/tiflash_proxy_linux_post_install.cmake COMPONENT tiflash-release)
-    install (SCRIPT ${TiFlash_SOURCE_DIR}/libs/libclara-cmake/linux_post_install.cmake COMPONENT tiflash-release)
+    if (ENABLE_CLARA)
+        install (SCRIPT ${TiFlash_SOURCE_DIR}/libs/libclara-cmake/linux_post_install.cmake COMPONENT tiflash-release)
+    endif()
 elseif(APPLE)
     # set build rpaths, so that executables can be directly called in build tree (easy to debug)
     set_target_properties(tiflash PROPERTIES BUILD_RPATH "@executable_path/;@executable_path/${TIFLASH_PROXY_LIB_RPATH}/")
     set_target_properties(tiflash PROPERTIES INSTALL_RPATH "@executable_path/")
-    if (USE_INTERNAL_LIBCLARA)
+    if (ENABLE_CLARA AND USE_INTERNAL_LIBCLARA)
         install (TARGETS clara_shared
                 COMPONENT tiflash-release
                 DESTINATION ".")

dbms/src/Storages/DeltaMerge/CMakeLists.txt

@@ -32,8 +32,10 @@ add_headers_and_sources(delta_merge ./Index/InvertedIndex)
 add_headers_and_sources(delta_merge ./Index/InvertedIndex/Reader)
 add_headers_and_sources(delta_merge ./Index/VectorIndex)
 add_headers_and_sources(delta_merge ./Index/VectorIndex/Stream)
-add_headers_and_sources(delta_merge ./Index/FullTextIndex)
-add_headers_and_sources(delta_merge ./Index/FullTextIndex/Stream)
+if (ENABLE_CLARA)
+    add_headers_and_sources(delta_merge ./Index/FullTextIndex)
+    add_headers_and_sources(delta_merge ./Index/FullTextIndex/Stream)
+endif ()
 add_headers_and_sources(delta_merge ./ReadThread)
 add_headers_and_sources(delta_merge ./Remote)
 add_headers_and_sources(delta_merge ./Remote/DataStore)

dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.cpp

@@ -12,13 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <Common/config.h> // For ENABLE_CLARA
 #include <Interpreters/Context.h>
 #include <Storages/DeltaMerge/File/DMFileBlockInputStream.h>
-#include <Storages/DeltaMerge/Index/FullTextIndex/Perf.h>
-#include <Storages/DeltaMerge/Index/FullTextIndex/Reader.h>
-#include <Storages/DeltaMerge/Index/FullTextIndex/Stream/BruteScoreInputStream.h>
-#include <Storages/DeltaMerge/Index/FullTextIndex/Stream/Ctx.h>
-#include <Storages/DeltaMerge/Index/FullTextIndex/Stream/DMFileInputStream.h>
 #include <Storages/DeltaMerge/Index/VectorIndex/Perf.h>
 #include <Storages/DeltaMerge/Index/VectorIndex/Reader.h>
 #include <Storages/DeltaMerge/Index/VectorIndex/Stream/Ctx.h>
@@ -27,6 +23,13 @@
 #include <Storages/DeltaMerge/ReadThread/SegmentReader.h>
 #include <Storages/DeltaMerge/ScanContext.h>
 
+#if ENABLE_CLARA
+#include <Storages/DeltaMerge/Index/FullTextIndex/Perf.h>
+#include <Storages/DeltaMerge/Index/FullTextIndex/Reader.h>
+#include <Storages/DeltaMerge/Index/FullTextIndex/Stream/BruteScoreInputStream.h>
+#include <Storages/DeltaMerge/Index/FullTextIndex/Stream/Ctx.h>
+#include <Storages/DeltaMerge/Index/FullTextIndex/Stream/DMFileInputStream.h>
+#endif
 namespace DB::DM
 {
 
@@ -145,10 +148,12 @@ SkippableBlockInputStreamPtr DMFileBlockInputStreamBuilder::build(
     const RowKeyRanges & rowkey_ranges,
     const ScanContextPtr & scan_context)
 {
+    // Note: this file may not have index built
     {
-        // Note: this file may not have index built
+#if ENABLE_CLARA
         if (fts_index_ctx)
             return buildForFullTextIndex(dmfile, read_columns, rowkey_ranges, scan_context);
+#endif
         if (vec_index_ctx)
             return buildForVectorIndex(dmfile, read_columns, rowkey_ranges, scan_context);
     }
@@ -241,6 +246,7 @@ SkippableBlockInputStreamPtr DMFileBlockInputStreamBuilder::buildForVectorIndex(
         std::move(rest_columns_reader));
 }
 
+#if ENABLE_CLARA
 SkippableBlockInputStreamPtr DMFileBlockInputStreamBuilder::buildForFullTextIndex(
     const DMFilePtr & dmfile,
     const ColumnDefines & read_columns,
@@ -327,5 +333,6 @@ SkippableBlockInputStreamPtr DMFileBlockInputStreamBuilder::buildForFullTextInde
         dmfile,
         std::move(rest_columns_reader));
 }
+#endif
 
 } // namespace DB::DM

dbms/src/Storages/DeltaMerge/File/DMFileBlockInputStream.h

@@ -15,18 +15,21 @@
 #pragma once
 
 #include <Common/Exception.h>
+#include <Common/config.h> // For ENABLE_CLARA
 #include <Interpreters/Context_fwd.h>
 #include <Interpreters/Settings.h>
 #include <Storages/DeltaMerge/DeltaMergeDefines.h>
 #include <Storages/DeltaMerge/File/ColumnCache.h>
 #include <Storages/DeltaMerge/File/ColumnCacheLongTerm_fwd.h>
 #include <Storages/DeltaMerge/File/DMFileReader.h>
-#include <Storages/DeltaMerge/Index/FullTextIndex/Stream/Ctx_fwd.h>
 #include <Storages/DeltaMerge/Index/VectorIndex/Stream/Ctx_fwd.h>
 #include <Storages/DeltaMerge/RowKeyRange.h>
 #include <Storages/DeltaMerge/ScanContext_fwd.h>
 #include <Storages/DeltaMerge/SkippableBlockInputStream.h>
 
+#if ENABLE_CLARA
+#include <Storages/DeltaMerge/Index/FullTextIndex/Stream/Ctx_fwd.h>
+#endif
 
 namespace DB::DM
 {
@@ -122,11 +125,13 @@ class DMFileBlockInputStreamBuilder
         return *this;
     }
 
+#if ENABLE_CLARA
     DMFileBlockInputStreamBuilder & setFtsIndexQuery(const FullTextIndexStreamCtxPtr & ctx)
     {
         fts_index_ctx = ctx;
         return *this;
     }
+#endif
 
     DMFileBlockInputStreamBuilder & setReadPacks(const IdSetPtr & read_packs_)
     {
@@ -201,12 +206,14 @@ class DMFileBlockInputStreamBuilder
         const RowKeyRanges & rowkey_ranges,
         const ScanContextPtr & scan_context);
 
+#if ENABLE_CLARA
     /// The returned stream should be plugged into a FullTextIndexInputStream. Plug to somewhere else will not work.
     SkippableBlockInputStreamPtr buildForFullTextIndex(
         const DMFilePtr & dmfile,
         const ColumnDefines & read_columns,
         const RowKeyRanges & rowkey_ranges,
         const ScanContextPtr & scan_context);
+#endif
 
 private:
     // These methods are called by the ctor
@@ -253,7 +260,9 @@ class DMFileBlockInputStreamBuilder
     /// If set, will *try* to build a VectorIndexDMFileInputStream
     /// instead of a normal DMFileBlockInputStream.
     VectorIndexStreamCtxPtr vec_index_ctx = nullptr;
+#if ENABLE_CLARA
     FullTextIndexStreamCtxPtr fts_index_ctx = nullptr;
+#endif
 
     // Note: column_cache_long_term is currently only filled when performing Vector Search.
     ColumnCacheLongTermPtr column_cache_long_term = nullptr;

dbms/src/Storages/DeltaMerge/File/ReadBlockInfo.cpp

@@ -12,12 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <Common/config.h> // For ENABLE_CLARA
 #include <Storages/DeltaMerge/File/ReadBlockInfo.h>
-#include <Storages/DeltaMerge/Index/FullTextIndex/Stream/IProvideFullTextIndex.h>
 #include <Storages/DeltaMerge/Index/VectorIndex/Stream/IProvideVectorIndex.h>
 
 #include <span>
 
+#if ENABLE_CLARA
+#include <Storages/DeltaMerge/Index/FullTextIndex/Stream/IProvideFullTextIndex.h>
+#endif
+
 namespace DB::DM
 {
 
@@ -140,11 +144,13 @@ template ReadBlockInfos ReadBlockInfo::createWithRowIDs<IProvideVectorIndex::Sea
     const DMFileMeta::PackStats & pack_stats,
     size_t rows_threshold_per_read);
 
+#if ENABLE_CLARA
 template ReadBlockInfos ReadBlockInfo::createWithRowIDs<IProvideFullTextIndex::SearchResult>(
     std::span<IProvideFullTextIndex::SearchResult> row_ids,
     const std::vector<size_t> & pack_offset,
     const RSResults & pack_res,
     const DMFileMeta::PackStats & pack_stats,
     size_t rows_threshold_per_read);
+#endif
 
 } // namespace DB::DM

dbms/src/Storages/DeltaMerge/Filter/PushDownExecutor.cpp

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <Common/Exception.h>
+#include <Common/config.h> // For ENABLE_CLARA
 #include <DataStreams/GeneratedColumnPlaceholderBlockInputStream.h>
 #include <Flash/Coprocessor/DAGExpressionAnalyzer.h>
 #include <Flash/Coprocessor/DAGQueryInfo.h>
@@ -21,13 +23,16 @@
 #include <Storages/DeltaMerge/Filter/PushDownExecutor.h>
 #include <Storages/SelectQueryInfo.h>
 #include <TiDB/Decode/TypeMapping.h>
+#include <tipb/executor.pb.h>
 
 namespace DB::DM
 {
 PushDownExecutorPtr PushDownExecutor::build(
     const RSOperatorPtr & rs_operator,
     const ANNQueryInfoPtr & ann_query_info,
+#if ENABLE_CLARA
     const FTSQueryInfoPtr & fts_query_info,
+#endif
     const TiDB::ColumnInfos & table_scan_column_info,
     const google::protobuf::RepeatedPtrField<tipb::Expr> & pushed_down_filters,
     const ColumnDefines & columns_to_read,
@@ -44,7 +49,13 @@ PushDownExecutorPtr PushDownExecutor::build(
     if (pushed_down_filters.empty())
     {
         LOG_DEBUG(tracing_logger, "Push down filter is empty");
-        return std::make_shared<PushDownExecutor>(rs_operator, ann_query_info, fts_query_info, column_range);
+        return std::make_shared<PushDownExecutor>(
+            rs_operator,
+            ann_query_info,
+#if ENABLE_CLARA
+            fts_query_info,
+#endif
+            column_range);
     }
     std::unordered_map<ColumnID, ColumnDefine> columns_to_read_map;
     for (const auto & column : columns_to_read)
@@ -160,7 +171,9 @@ PushDownExecutorPtr PushDownExecutor::build(
     return std::make_shared<PushDownExecutor>(
         rs_operator,
         ann_query_info,
+#if ENABLE_CLARA
         fts_query_info,
+#endif
         before_where,
         project_after_where,
         filter_columns,
@@ -196,9 +209,14 @@ PushDownExecutorPtr PushDownExecutor::build(
     ANNQueryInfoPtr ann_query_info = nullptr;
     if (dag_query->ann_query_info.query_type() != tipb::ANNQueryType::InvalidQueryType)
         ann_query_info = std::make_shared<tipb::ANNQueryInfo>(dag_query->ann_query_info);
+#if ENABLE_CLARA
     FTSQueryInfoPtr fts_query_info = nullptr;
     if (dag_query->fts_query_info.query_type() != tipb::FTSQueryType::FTSQueryTypeInvalid)
         fts_query_info = std::make_shared<tipb::FTSQueryInfo>(dag_query->fts_query_info);
+#else
+    if (dag_query->fts_query_info.query_type() != tipb::FTSQueryType::FTSQueryTypeInvalid)
+        throw Exception("FTS query is not supported", ErrorCodes::NOT_IMPLEMENTED);
+#endif
     // build push down filter
     const auto & pushed_down_filters = dag_query->pushed_down_filters;
     if (unlikely(context.getSettingsRef().force_push_down_all_filters_to_scan) && !dag_query->filters.empty())
@@ -210,7 +228,9 @@ PushDownExecutorPtr PushDownExecutor::build(
         return PushDownExecutor::build(
             rs_operator,
             ann_query_info,
+#if ENABLE_CLARA
             fts_query_info,
+#endif
             columns_to_read_info,
             merged_filters,
             columns_to_read,
@@ -221,7 +241,9 @@ PushDownExecutorPtr PushDownExecutor::build(
     return PushDownExecutor::build(
         rs_operator,
         ann_query_info,
+#if ENABLE_CLARA
         fts_query_info,
+#endif
         columns_to_read_info,
         pushed_down_filters,
         columns_to_read,