Multi Field support -use single tantivy call for multiple fields

Author: Amit Kumar

internal/core/src/common/QueryInfo.h

@@ -46,6 +46,30 @@ struct SearchInfo {
     std::optional<std::string> json_path_;
     std::optional<milvus::DataType> json_type_;
     bool strict_cast_{false};
+
+    // Multi-field TEXT_BM25 search support
+    // Additional field IDs for multi-field search (primary field is in field_id_)
+    std::vector<FieldId> additional_field_ids_;
+    // Weights for each field (first weight is for field_id_, rest for additional_field_ids_)
+    std::vector<float> bm25_weights_;
+    // If true, use max aggregation; otherwise use weighted_sum (default)
+    bool bm25_use_max_aggregation_{false};
+
+    // Helper to check if this is a multi-field search
+    bool
+    IsMultiFieldTextSearch() const {
+        return !additional_field_ids_.empty();
+    }
+
+    // Get all field IDs (primary + additional)
+    std::vector<FieldId>
+    GetAllTextFieldIds() const {
+        std::vector<FieldId> all_ids;
+        all_ids.push_back(field_id_);
+        all_ids.insert(
+            all_ids.end(), additional_field_ids_.begin(), additional_field_ids_.end());
+        return all_ids;
+    }
 };
 
 using SearchInfoPtr = std::shared_ptr<SearchInfo>;

internal/core/src/index/TextMatchIndex.cpp

@@ -428,4 +428,92 @@ TextMatchIndex::BM25SearchQueryWithFilter(const std::string& query,
     return {std::move(seg_offsets), std::move(scores)};
 }
 
+std::pair<std::vector<int64_t>, std::vector<float>>
+TextMatchIndex::BM25MultiFieldSearch(const std::vector<TextMatchIndex*>& indexes,
+                                     const std::string& query,
+                                     int64_t topk,
+                                     const std::vector<float>& weights,
+                                     bool use_max_aggregation) {
+    tracer::AutoSpan span(
+        "TextMatchIndex::BM25MultiFieldSearch", tracer::GetRootSpan());
+
+    if (indexes.empty() || weights.empty() || indexes.size() != weights.size()) {
+        return {{}, {}};
+    }
+
+    // Collect wrapper pointers - indexes are expected to be ready for reading
+    std::vector<TantivyIndexWrapper*> wrappers;
+    wrappers.reserve(indexes.size());
+    for (auto* idx : indexes) {
+        if (idx != nullptr && idx->wrapper_ != nullptr) {
+            wrappers.push_back(idx->wrapper_.get());
+        }
+    }
+
+    if (wrappers.empty()) {
+        return {{}, {}};
+    }
+
+    auto aggregation = use_max_aggregation ? BM25AggregationType::Max
+                                           : BM25AggregationType::WeightedSum;
+
+    auto [doc_ids, scores] = TantivyIndexWrapper::bm25_multi_field_search(
+        wrappers, query, static_cast<uintptr_t>(topk), weights, aggregation);
+
+    // Convert uint32_t doc_ids to int64_t seg_offsets
+    std::vector<int64_t> seg_offsets;
+    seg_offsets.reserve(doc_ids.size());
+    for (auto doc_id : doc_ids) {
+        seg_offsets.push_back(static_cast<int64_t>(doc_id));
+    }
+
+    return {std::move(seg_offsets), std::move(scores)};
+}
+
+std::pair<std::vector<int64_t>, std::vector<float>>
+TextMatchIndex::BM25MultiFieldSearchWithFilter(
+    const std::vector<TextMatchIndex*>& indexes,
+    const std::string& query,
+    int64_t topk,
+    const std::vector<float>& weights,
+    bool use_max_aggregation,
+    const uint8_t* filter_bitset,
+    size_t filter_bitset_len) {
+    tracer::AutoSpan span(
+        "TextMatchIndex::BM25MultiFieldSearchWithFilter", tracer::GetRootSpan());
+
+    if (indexes.empty() || weights.empty() || indexes.size() != weights.size()) {
+        return {{}, {}};
+    }
+
+    // Collect wrapper pointers - indexes are expected to be ready for reading
+    std::vector<TantivyIndexWrapper*> wrappers;
+    wrappers.reserve(indexes.size());
+    for (auto* idx : indexes) {
+        if (idx != nullptr && idx->wrapper_ != nullptr) {
+            wrappers.push_back(idx->wrapper_.get());
+        }
+    }
+
+    if (wrappers.empty()) {
+        return {{}, {}};
+    }
+
+    auto aggregation = use_max_aggregation ? BM25AggregationType::Max
+                                           : BM25AggregationType::WeightedSum;
+
+    auto [doc_ids, scores] = TantivyIndexWrapper::bm25_multi_field_search_with_filter(
+        wrappers, query, static_cast<uintptr_t>(topk), weights, aggregation,
+        filter_bitset, filter_bitset_len);
+
+    // Convert uint32_t doc_ids to int64_t seg_offsets
+    std::vector<int64_t> seg_offsets;
+    seg_offsets.reserve(doc_ids.size());
+    for (auto doc_id : doc_ids) {
+        seg_offsets.push_back(static_cast<int64_t>(doc_id));
+    }
+
+    return {std::move(seg_offsets), std::move(scores)};
+}
+
 }  // namespace milvus::index

internal/core/src/index/TextMatchIndex.h

@@ -113,6 +113,32 @@ class TextMatchIndex : public InvertedIndexTantivy<std::string> {
                                const uint8_t* filter_bitset,
                                size_t filter_bitset_len);
 
+    /// Multi-field BM25 search - queries multiple text indexes and aggregates scores.
+    /// This enables native multi-field text search without requiring HybridSearch + Reranker.
+    ///
+    /// @param indexes - Vector of TextMatchIndex pointers for each field
+    /// @param query - Search query text
+    /// @param topk - Number of top results per field (before aggregation)
+    /// @param weights - Weight for each field (must match indexes length)
+    /// @param use_max_aggregation - If true, use max; otherwise use weighted_sum
+    /// @return (seg_offsets, scores) sorted by aggregated score
+    static std::pair<std::vector<int64_t>, std::vector<float>>
+    BM25MultiFieldSearch(const std::vector<TextMatchIndex*>& indexes,
+                         const std::string& query,
+                         int64_t topk,
+                         const std::vector<float>& weights,
+                         bool use_max_aggregation = false);
+
+    /// Multi-field BM25 search with filter bitset.
+    static std::pair<std::vector<int64_t>, std::vector<float>>
+    BM25MultiFieldSearchWithFilter(const std::vector<TextMatchIndex*>& indexes,
+                                   const std::string& query,
+                                   int64_t topk,
+                                   const std::vector<float>& weights,
+                                   bool use_max_aggregation,
+                                   const uint8_t* filter_bitset,
+                                   size_t filter_bitset_len);
+
  private:
     bool
     shouldTriggerCommit();

internal/core/src/query/PlanProto.cpp

@@ -15,6 +15,7 @@
 
 #include <cstdint>
 #include <memory>
+#include <sstream>
 #include <string>
 #include <vector>
 
@@ -130,6 +131,41 @@ ProtoParser::PlanNodeFromProto(const planpb::PlanNode& plan_node_proto) {
             }
         }
 
+        // Parse multi-field TEXT_BM25 parameters from search_params
+        // bm25_field_ids: comma-separated additional field IDs (e.g., "101,102,103")
+        // bm25_weights: comma-separated weights for all fields (e.g., "1.0,0.5,0.5")
+        // bm25_agg: aggregation type ("weighted_sum" or "max")
+        if (search_info.search_params_.contains("bm25_field_ids")) {
+            std::string field_ids_str =
+                search_info.search_params_["bm25_field_ids"].get<std::string>();
+            std::stringstream ss(field_ids_str);
+            std::string token;
+            while (std::getline(ss, token, ',')) {
+                if (!token.empty()) {
+                    search_info.additional_field_ids_.push_back(
+                        FieldId(std::stoll(token)));
+                }
+            }
+        }
+
+        if (search_info.search_params_.contains("bm25_weights")) {
+            std::string weights_str =
+                search_info.search_params_["bm25_weights"].get<std::string>();
+            std::stringstream ss(weights_str);
+            std::string token;
+            while (std::getline(ss, token, ',')) {
+                if (!token.empty()) {
+                    search_info.bm25_weights_.push_back(std::stof(token));
+                }
+            }
+        }
+
+        if (search_info.search_params_.contains("bm25_agg")) {
+            std::string agg_type =
+                search_info.search_params_["bm25_agg"].get<std::string>();
+            search_info.bm25_use_max_aggregation_ = (agg_type == "max");
+        }
+
         return search_info;
     };
 

internal/core/src/segcore/SegmentInterface.cpp

@@ -503,35 +503,93 @@ SegmentInternalInterface::text_search(SearchInfo& search_info,
                                       SearchResult& output) const {
     tracer::AutoSpan span("text_search", tracer::GetRootSpan());
 
-    auto field_id = search_info.field_id_;
     auto topk = search_info.topk_;
-
-    // Get the text index for this field
-    auto pinned_index = GetTextIndex(op_context, field_id);
-    auto text_index = pinned_index.get();
-
     std::vector<int64_t> seg_offsets;
     std::vector<float> scores;
 
-    // Use filtered BM25 search if there's a filter, otherwise use unfiltered
-    // This is more efficient than searching and then filtering post-hoc
-    // because Tantivy can skip filtered documents during search.
-    if (!bitset.empty()) {
-        // Pass the bitset to Tantivy for efficient filtered search
-        auto [filtered_offsets, filtered_scores] =
-            text_index->BM25SearchQueryWithFilter(
-                query_text,
-                topk,
-                bitset.data(),
-                bitset.size());
-        seg_offsets = std::move(filtered_offsets);
-        scores = std::move(filtered_scores);
+    // Check if this is a multi-field TEXT_BM25 search
+    if (search_info.IsMultiFieldTextSearch()) {
+        // Multi-field search: gather all text indexes
+        auto all_field_ids = search_info.GetAllTextFieldIds();
+
+        // Collect pinned indexes (to keep them alive during search)
+        std::vector<PinWrapper<index::TextMatchIndex*>> pinned_indexes;
+        std::vector<index::TextMatchIndex*> text_indexes;
+        pinned_indexes.reserve(all_field_ids.size());
+        text_indexes.reserve(all_field_ids.size());
+
+        for (const auto& fid : all_field_ids) {
+            auto pinned = GetTextIndex(op_context, fid);
+            text_indexes.push_back(pinned.get());
+            pinned_indexes.push_back(std::move(pinned));
+        }
+
+        // Get weights (use equal weights if not specified)
+        std::vector<float> weights = search_info.bm25_weights_;
+        if (weights.empty()) {
+            weights.assign(all_field_ids.size(), 1.0f);
+        }
+
+        // Ensure weights vector matches field count
+        if (weights.size() != all_field_ids.size()) {
+            throw SegcoreError(
+                milvus::ErrorCode::FieldNotLoaded,
+                fmt::format("BM25 weights count ({}) doesn't match field count ({})",
+                           weights.size(), all_field_ids.size()));
+        }
+
+        // Use filtered or unfiltered multi-field search
+        if (!bitset.empty()) {
+            auto [filtered_offsets, filtered_scores] =
+                index::TextMatchIndex::BM25MultiFieldSearchWithFilter(
+                    text_indexes,
+                    query_text,
+                    topk,
+                    weights,
+                    search_info.bm25_use_max_aggregation_,
+                    bitset.data(),
+                    bitset.size());
+            seg_offsets = std::move(filtered_offsets);
+            scores = std::move(filtered_scores);
+        } else {
+            auto [unfiltered_offsets, unfiltered_scores] =
+                index::TextMatchIndex::BM25MultiFieldSearch(
+                    text_indexes,
+                    query_text,
+                    topk,
+                    weights,
+                    search_info.bm25_use_max_aggregation_);
+            seg_offsets = std::move(unfiltered_offsets);
+            scores = std::move(unfiltered_scores);
+        }
     } else {
-        // No filter, use regular BM25 search
-        auto [unfiltered_offsets, unfiltered_scores] =
-            text_index->BM25SearchQuery(query_text, topk);
-        seg_offsets = std::move(unfiltered_offsets);
-        scores = std::move(unfiltered_scores);
+        // Single-field search (original path)
+        auto field_id = search_info.field_id_;
+
+        // Get the text index for this field
+        auto pinned_index = GetTextIndex(op_context, field_id);
+        auto text_index = pinned_index.get();
+
+        // Use filtered BM25 search if there's a filter, otherwise use unfiltered
+        // This is more efficient than searching and then filtering post-hoc
+        // because Tantivy can skip filtered documents during search.
+        if (!bitset.empty()) {
+            // Pass the bitset to Tantivy for efficient filtered search
+            auto [filtered_offsets, filtered_scores] =
+                text_index->BM25SearchQueryWithFilter(
+                    query_text,
+                    topk,
+                    bitset.data(),
+                    bitset.size());
+            seg_offsets = std::move(filtered_offsets);
+            scores = std::move(filtered_scores);
+        } else {
+            // No filter, use regular BM25 search
+            auto [unfiltered_offsets, unfiltered_scores] =
+                text_index->BM25SearchQuery(query_text, topk);
+            seg_offsets = std::move(unfiltered_offsets);
+            scores = std::move(unfiltered_scores);
+        }
     }
 
     // Populate the search result

internal/core/thirdparty/tantivy/tantivy-binding/include/tantivy-binding.h

@@ -6,6 +6,13 @@
 #include <ostream>
 #include <new>
 
+/// BM25 aggregation type for multi-field search.
+/// 0 = WeightedSum (default), 1 = Max
+enum class BM25AggregationType {
+  WeightedSum = 0,
+  Max = 1,
+};
+
 enum class TantivyDataType : uint8_t {
   Text,
   Keyword,
@@ -346,6 +353,36 @@ RustResult tantivy_bm25_search_query_with_filter(void *ptr,
                                                  uintptr_t filter_bitset_len,
                                                  RustScoredSearchResult *result);
 
+/// Performs multi-field BM25 search by querying multiple text indexes and aggregating results.
+/// This enables native multi-field text search without requiring HybridSearch + Reranker.
+///
+/// # Arguments
+/// * `readers` - Array of IndexReaderWrapper pointers for each field
+/// * `num_readers` - Number of readers
+/// * `query` - Search query text
+/// * `topk` - Number of top results per field (before aggregation)
+/// * `weights` - Weight for each field (array of floats, length must match num_readers)
+/// * `aggregation` - How to combine scores (0 = WeightedSum, 1 = Max)
+/// * `result` - Output result
+RustResult tantivy_bm25_multi_field_search(void *const *readers,
+                                           uintptr_t num_readers,
+                                           const char *query,
+                                           uintptr_t topk,
+                                           const float *weights,
+                                           BM25AggregationType aggregation,
+                                           RustScoredSearchResult *result);
+
+/// Performs multi-field BM25 search with a filter bitset.
+RustResult tantivy_bm25_multi_field_search_with_filter(void *const *readers,
+                                                       uintptr_t num_readers,
+                                                       const char *query,
+                                                       uintptr_t topk,
+                                                       const float *weights,
+                                                       BM25AggregationType aggregation,
+                                                       const uint8_t *filter_bitset,
+                                                       uintptr_t filter_bitset_len,
+                                                       RustScoredSearchResult *result);
+
 RustResult tantivy_create_index(const char *field_name,
                                 TantivyDataType data_type,
                                 const char *path,