From c4fc5183fe82e6e48fc029e55173c053137ee02d Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Wed, 8 Jul 2026 21:41:09 +0800 Subject: [PATCH 1/8] feat(lumina): support Lumina tag filtering --- .../global_index/global_index_io_meta.h | 15 +- include/paimon/global_index/global_indexer.h | 6 + .../global_index/global_index_scan_impl.cpp | 4 +- .../global_index/global_index_write_task.cpp | 143 +++- .../lumina/lumina_global_index.cpp | 622 +++++++++++++++++- .../global_index/lumina/lumina_global_index.h | 45 ++ .../lumina/lumina_global_index_test.cpp | 386 ++++++++++- test/inte/global_index_test.cpp | 152 +++++ 8 files changed, 1323 insertions(+), 50 deletions(-) diff --git a/include/paimon/global_index/global_index_io_meta.h b/include/paimon/global_index/global_index_io_meta.h index e9da91cba..600d278f6 100644 --- a/include/paimon/global_index/global_index_io_meta.h +++ b/include/paimon/global_index/global_index_io_meta.h @@ -16,8 +16,11 @@ #pragma once +#include #include +#include #include +#include #include "paimon/memory/bytes.h" #include "paimon/utils/range.h" @@ -27,7 +30,15 @@ namespace paimon { struct PAIMON_EXPORT GlobalIndexIOMeta { GlobalIndexIOMeta(const std::string& _file_path, int64_t _file_size, const std::shared_ptr& _metadata) - : file_path(_file_path), file_size(_file_size), metadata(_metadata) {} + : GlobalIndexIOMeta(_file_path, _file_size, _metadata, std::nullopt) {} + + GlobalIndexIOMeta(const std::string& _file_path, int64_t _file_size, + const std::shared_ptr& _metadata, + const std::optional>& _extra_field_ids) + : file_path(_file_path), + file_size(_file_size), + metadata(_metadata), + extra_field_ids(_extra_field_ids) {} std::string file_path; int64_t file_size; @@ -35,6 +46,8 @@ struct PAIMON_EXPORT GlobalIndexIOMeta { /// secondary index structures or inline index bytes. /// May be null if no additional metadata is available. std::shared_ptr metadata; + /// Optional table field ids materialized together with the indexed field. + std::optional> extra_field_ids; }; } // namespace paimon diff --git a/include/paimon/global_index/global_indexer.h b/include/paimon/global_index/global_indexer.h index 5e529fbe8..a5e881366 100644 --- a/include/paimon/global_index/global_indexer.h +++ b/include/paimon/global_index/global_indexer.h @@ -17,6 +17,7 @@ #pragma once #include +#include #include #include @@ -36,6 +37,11 @@ class PAIMON_EXPORT GlobalIndexer { public: virtual ~GlobalIndexer() = default; + /// Returns additional table fields required during index construction. + virtual Result>> GetExtraFieldNames() const { + return std::optional>(std::nullopt); + } + /// Creates a writer for building a global index on a specific field. /// /// @param field_name Name of the field to be indexed. diff --git a/src/paimon/core/global_index/global_index_scan_impl.cpp b/src/paimon/core/global_index/global_index_scan_impl.cpp index 1c24d2f74..79058bd87 100644 --- a/src/paimon/core/global_index/global_index_scan_impl.cpp +++ b/src/paimon/core/global_index/global_index_scan_impl.cpp @@ -20,6 +20,7 @@ #include #include "arrow/c/bridge.h" +#include "arrow/c/helpers.h" #include "paimon/common/global_index/offset_global_index_reader.h" #include "paimon/common/global_index/union_global_index_reader.h" #include "paimon/common/utils/scope_guard.h" @@ -185,7 +186,6 @@ Result>> GlobalIndexScanImpl::Cre if (row_range_index && !row_range_index->Intersects(range.from, range.to)) { continue; } - // TODO(xinyu.lxy): c_arrow_schema may contains additional associated fields. auto arrow_field = DataField::ConvertDataFieldToArrowField(field); auto arrow_schema = arrow::schema({arrow_field}); @@ -223,7 +223,7 @@ GlobalIndexIOMeta GlobalIndexScanImpl::ToGlobalIndexIOMeta( assert(index_meta->GetGlobalIndexMeta()); const auto& global_index_meta = index_meta->GetGlobalIndexMeta().value(); return {index_file_manager_->ToPath(index_meta), index_meta->FileSize(), - global_index_meta.index_meta}; + global_index_meta.index_meta, global_index_meta.extra_field_ids}; } Result> GlobalIndexScanImpl::Scan( diff --git a/src/paimon/core/global_index/global_index_write_task.cpp b/src/paimon/core/global_index/global_index_write_task.cpp index d44840957..f43ec3a2f 100644 --- a/src/paimon/core/global_index/global_index_write_task.cpp +++ b/src/paimon/core/global_index/global_index_write_task.cpp @@ -16,7 +16,10 @@ #include "paimon/global_index/global_index_write_task.h" +#include + #include "arrow/c/bridge.h" +#include "arrow/c/helpers.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" #include "paimon/common/utils/arrow/status_utils.h" @@ -35,6 +38,17 @@ #include "paimon/table/source/table_read.h" namespace paimon { namespace { +Result> CreateGlobalIndexer(const std::string& index_type, + const CoreOptions& core_options) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, + GlobalIndexerFactory::Get(index_type, core_options.ToMap())); + if (!indexer) { + return Status::Invalid( + fmt::format("Unknown index type {}, may not registered", index_type)); + } + return indexer; +} + Result> CreateGlobalIndexFileManager( const std::string& table_path, const std::shared_ptr& table_schema, const CoreOptions& core_options, const std::shared_ptr& pool) { @@ -58,25 +72,76 @@ Result> CreateGlobalIndexFileManager( } Result> CreateGlobalIndexWriter( - const std::string& index_type, const DataField& field, + const GlobalIndexer& indexer, const DataField& field, + const std::vector& extra_fields, const std::shared_ptr& index_file_manager, - const CoreOptions& core_options, const std::shared_ptr& pool) { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, - GlobalIndexerFactory::Get(index_type, core_options.ToMap())); - if (!indexer) { - return Status::Invalid( - fmt::format("Unknown index type {}, may not registered", index_type)); + const std::shared_ptr& pool) { + arrow::FieldVector arrow_fields; + arrow_fields.reserve(extra_fields.size() + 1); + arrow_fields.push_back(DataField::ConvertDataFieldToArrowField(field)); + for (const auto& extra_field : extra_fields) { + arrow_fields.push_back(DataField::ConvertDataFieldToArrowField(extra_field)); } - // TODO(xinyu.lxy): may add additional fields to read for index write - auto arrow_field = DataField::ConvertDataFieldToArrowField(field); - auto arrow_schema = arrow::schema({arrow_field}); + auto arrow_schema = arrow::schema(arrow_fields); ArrowSchema c_arrow_schema; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*arrow_schema, &c_arrow_schema)); - return indexer->CreateWriter(field.Name(), &c_arrow_schema, index_file_manager, pool); + ScopeGuard guard([&]() { ArrowSchemaRelease(&c_arrow_schema); }); + return indexer.CreateWriter(field.Name(), &c_arrow_schema, index_file_manager, pool); +} + +Result> GetExtraFields(const TableSchema& table_schema, + const std::string& field_name, + const std::vector& extra_field_names) { + std::vector extra_fields; + extra_fields.reserve(extra_field_names.size()); + std::set dedup_field_names; + for (const auto& extra_field_name : extra_field_names) { + if (extra_field_name == field_name || !dedup_field_names.insert(extra_field_name).second) { + continue; + } + PAIMON_ASSIGN_OR_RAISE(DataField extra_field, table_schema.GetField(extra_field_name)); + extra_fields.push_back(extra_field); + } + return extra_fields; +} + +std::vector BuildReadFieldNames(const std::string& field_name, + const std::vector& extra_fields) { + std::vector read_field_names; + read_field_names.reserve(extra_fields.size() + 2); + read_field_names.push_back(field_name); + for (const auto& extra_field : extra_fields) { + read_field_names.push_back(extra_field.Name()); + } + read_field_names.push_back(SpecialFields::RowId().Name()); + return read_field_names; +} + +std::vector BuildWriterFieldNames(const std::string& field_name, + const std::vector& extra_fields) { + std::vector writer_field_names; + writer_field_names.reserve(extra_fields.size() + 1); + writer_field_names.push_back(field_name); + for (const auto& extra_field : extra_fields) { + writer_field_names.push_back(extra_field.Name()); + } + return writer_field_names; +} + +std::optional> GetExtraFieldIds(const std::vector& extra_fields) { + if (extra_fields.empty()) { + return std::nullopt; + } + std::vector extra_field_ids; + extra_field_ids.reserve(extra_fields.size()); + for (const auto& extra_field : extra_fields) { + extra_field_ids.push_back(extra_field.Id()); + } + return extra_field_ids; } Result> CreateBatchReader( - const std::string& table_path, const std::string& field_name, + const std::string& table_path, const std::vector& read_field_names, const std::shared_ptr& indexed_split, const CoreOptions& core_options, const std::shared_ptr& pool) { ReadContextBuilder read_context_builder(table_path); @@ -84,7 +149,7 @@ Result> CreateBatchReader( .WithFileSystem(core_options.GetFileSystem()) .EnablePrefetch(true) .WithMemoryPool(pool) - .SetReadFieldNames({field_name, SpecialFields::RowId().Name()}); + .SetReadFieldNames(read_field_names); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_context_builder.Finish()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, @@ -92,9 +157,10 @@ Result> CreateBatchReader( return table_read->CreateReader(indexed_split); } -Result> BuildIndex(const std::string& field_name, const Range& range, - BatchReader* batch_reader, - GlobalIndexWriter* global_index_writer) { +Result> BuildIndex( + const std::string& field_name, const Range& range, + const std::vector& writer_field_names, BatchReader* batch_reader, + GlobalIndexWriter* global_index_writer) { while (true) { PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch read_batch, batch_reader->NextBatch()); if (BatchReader::IsEofBatch(read_batch)) { @@ -131,8 +197,20 @@ Result> BuildIndex(const std::string& field_name, } relative_row_ids.push_back(row_id - range.from); } - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr new_array, - arrow::StructArray::Make({indexed_array}, {field_name})); + std::vector> writer_arrays; + writer_arrays.reserve(writer_field_names.size()); + for (const auto& writer_field_name : writer_field_names) { + auto writer_array = struct_array->GetFieldByName(writer_field_name); + if (!writer_array) { + return Status::Invalid( + fmt::format("read array does not contain {} field in GlobalIndexWriteTask", + writer_field_name)); + } + writer_arrays.push_back(writer_array); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr new_array, + arrow::StructArray::Make(writer_arrays, writer_field_names)); ::ArrowArray c_new_array; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*new_array, &c_new_array)); PAIMON_RETURN_NOT_OK( @@ -157,8 +235,8 @@ Result> ToCommitMessage( index_file_metas.push_back(std::make_shared( index_type, PathUtil::GetName(io_meta.file_path), io_meta.file_size, range.Count(), /*dv_ranges=*/std::nullopt, external_path, - GlobalIndexMeta(range.from, range.to, field_id, - /*extra_field_ids=*/std::nullopt, io_meta.metadata))); + GlobalIndexMeta(range.from, range.to, field_id, io_meta.extra_field_ids, + io_meta.metadata))); } DataIncrement data_increment(std::move(index_file_metas)); return std::make_shared(partition, bucket, @@ -199,6 +277,17 @@ Result> GlobalIndexWriteTask::WriteIndex( } PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(final_options, file_system)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, + CreateGlobalIndexer(index_type, core_options)); + PAIMON_ASSIGN_OR_RAISE(std::optional> extra_field_names, + indexer->GetExtraFieldNames()); + PAIMON_ASSIGN_OR_RAISE(DataField field, table_schema->GetField(field_name)); + PAIMON_ASSIGN_OR_RAISE(std::vector extra_fields, + GetExtraFields(*table_schema, field_name, + extra_field_names.value_or(std::vector()))); + std::optional> extra_field_ids = GetExtraFieldIds(extra_fields); + std::vector writer_field_names = BuildWriterFieldNames(field_name, extra_fields); + std::vector read_field_names = BuildReadFieldNames(field_name, extra_fields); // create index file manager PAIMON_ASSIGN_OR_RAISE( @@ -208,13 +297,12 @@ Result> GlobalIndexWriteTask::WriteIndex( // create batch reader PAIMON_ASSIGN_OR_RAISE( std::unique_ptr batch_reader, - CreateBatchReader(table_path, field_name, indexed_split, core_options, pool)); + CreateBatchReader(table_path, read_field_names, indexed_split, core_options, pool)); // create global index writer - PAIMON_ASSIGN_OR_RAISE(DataField field, table_schema->GetField(field_name)); PAIMON_ASSIGN_OR_RAISE( std::shared_ptr global_index_writer, - CreateGlobalIndexWriter(index_type, field, index_file_manager, core_options, pool)); + CreateGlobalIndexWriter(*indexer, field, extra_fields, index_file_manager, pool)); ScopeGuard guard([&]() { global_index_writer.reset(); @@ -222,9 +310,12 @@ Result> GlobalIndexWriteTask::WriteIndex( }); // read from data split and write to index writer - PAIMON_ASSIGN_OR_RAISE( - std::vector global_index_io_metas, - BuildIndex(field_name, range, batch_reader.get(), global_index_writer.get())); + PAIMON_ASSIGN_OR_RAISE(std::vector global_index_io_metas, + BuildIndex(field_name, range, writer_field_names, batch_reader.get(), + global_index_writer.get())); + for (auto& io_meta : global_index_io_metas) { + io_meta.extra_field_ids = extra_field_ids; + } // generate commit message return ToCommitMessage(index_type, field.Id(), range, global_index_io_metas, diff --git a/src/paimon/global_index/lumina/lumina_global_index.cpp b/src/paimon/global_index/lumina/lumina_global_index.cpp index 8e4af3e49..29ead076b 100644 --- a/src/paimon/global_index/lumina/lumina_global_index.cpp +++ b/src/paimon/global_index/lumina/lumina_global_index.cpp @@ -16,6 +16,9 @@ #include "paimon/global_index/lumina/lumina_global_index.h" +#include +#include +#include #include #include "arrow/c/bridge.h" @@ -27,6 +30,7 @@ #include "lumina/core/Constants.h" #include "lumina/core/Status.h" #include "lumina/core/Types.h" +#include "lumina/extensions/experimental/BuildCombinedExtensionV0.h" #include "paimon/common/global_index/global_index_utils.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/rapidjson_util.h" @@ -35,6 +39,9 @@ #include "paimon/global_index/lumina/lumina_file_reader.h" #include "paimon/global_index/lumina/lumina_file_writer.h" #include "paimon/global_index/lumina/lumina_utils.h" +#include "paimon/predicate/compound_predicate.h" +#include "paimon/predicate/leaf_predicate.h" +#include "rapidjson/document.h" namespace paimon::lumina { #define CHECK_NOT_NULL(pointer, error_msg) \ do { \ @@ -43,6 +50,470 @@ namespace paimon::lumina { } \ } while (0) +namespace { +using TagDimensionData = ::lumina::extensions::experimental::TagDimensionData; +using TagFilter = ::lumina::extensions::experimental::TagFilter; +using TagValue = ::lumina::extensions::experimental::TagValue; +using TagValues = ::lumina::extensions::experimental::TagValues; + +Result GetRequiredStringMember(const rapidjson::Value& obj, + const std::string& field_name, + const std::string& tag_label) { + auto iter = obj.FindMember(field_name.c_str()); + if (iter == obj.MemberEnd()) { + return Status::Invalid( + fmt::format("lumina tag_schema {} missing required field: {}", tag_label, field_name)); + } + if (!iter->value.IsString()) { + return Status::Invalid( + fmt::format("lumina tag_schema {} field {} must be string", tag_label, field_name)); + } + return std::string(iter->value.GetString(), iter->value.GetStringLength()); +} + +Result ParseTagField(const rapidjson::Value& obj, const std::string& tag_label) { + if (!obj.IsObject()) { + return Status::Invalid(fmt::format("lumina tag_schema {} must be object", tag_label)); + } + if (obj.MemberCount() != 3) { + return Status::Invalid(fmt::format( + "lumina tag_schema {} must have exactly 3 fields: key_name, type, value_type", + tag_label)); + } + + PAIMON_ASSIGN_OR_RAISE( + std::string key_name, + GetRequiredStringMember(obj, std::string(::lumina::core::kExtensionTagKName), tag_label)); + PAIMON_ASSIGN_OR_RAISE( + std::string type, + GetRequiredStringMember(obj, std::string(::lumina::core::kExtensionTagType), tag_label)); + PAIMON_ASSIGN_OR_RAISE( + std::string value_type, + GetRequiredStringMember(obj, std::string(::lumina::core::kExtensionTagVType), tag_label)); + + if (key_name.empty()) { + return Status::Invalid( + fmt::format("lumina tag_schema {} key_name must not be empty", tag_label)); + } + if (key_name.size() > ::lumina::core::kMaxTagKNameLength) { + return Status::Invalid(fmt::format("lumina tag_schema {} key_name exceeds max length {}", + tag_label, ::lumina::core::kMaxTagKNameLength)); + } + + LuminaTagField::Type parsed_type; + if (type == std::string(::lumina::core::kExtensionTagTypeEnum)) { + parsed_type = LuminaTagField::Type::ENUM; + } else if (type == std::string(::lumina::core::kExtensionTagTypeRange)) { + parsed_type = LuminaTagField::Type::RANGE; + } else { + return Status::Invalid( + fmt::format("lumina tag_schema {} has unsupported type: {}", tag_label, type)); + } + + LuminaTagField::ValueType parsed_value_type; + if (value_type == std::string(::lumina::core::kExtensionTagVTypeInt64)) { + parsed_value_type = LuminaTagField::ValueType::INT64; + } else if (value_type == std::string(::lumina::core::kExtensionTagVTypeDouble)) { + parsed_value_type = LuminaTagField::ValueType::DOUBLE; + } else if (value_type == std::string(::lumina::core::kExtensionTagVTypeString)) { + parsed_value_type = LuminaTagField::ValueType::STRING; + } else { + return Status::Invalid(fmt::format("lumina tag_schema {} has unsupported value_type: {}", + tag_label, value_type)); + } + if (parsed_type == LuminaTagField::Type::RANGE && + parsed_value_type == LuminaTagField::ValueType::STRING) { + return Status::Invalid(fmt::format( + "lumina tag_schema {} range type does not support string value_type", tag_label)); + } + return LuminaTagField{key_name, parsed_type, parsed_value_type}; +} + +Status ValidateTagArrowType(const LuminaTagField& tag_field, + const std::shared_ptr& field_type) { + auto value_type = field_type; + if (auto list_type = std::dynamic_pointer_cast(field_type)) { + value_type = list_type->value_type(); + } + + bool compatible = false; + switch (tag_field.value_type) { + case LuminaTagField::ValueType::INT64: + compatible = + value_type->id() == arrow::Type::INT8 || value_type->id() == arrow::Type::INT16 || + value_type->id() == arrow::Type::INT32 || value_type->id() == arrow::Type::INT64; + break; + case LuminaTagField::ValueType::DOUBLE: + compatible = + value_type->id() == arrow::Type::FLOAT || value_type->id() == arrow::Type::DOUBLE; + break; + case LuminaTagField::ValueType::STRING: + compatible = value_type->id() == arrow::Type::STRING; + break; + } + if (!compatible) { + return Status::Invalid( + fmt::format("lumina tag field {} type {} is not compatible with tag_schema value_type", + tag_field.name, field_type->ToString())); + } + return Status::OK(); +} + +Status AppendInt64Value(const std::shared_ptr& array, int64_t index, + std::vector* values) { + if (array->IsNull(index)) { + return Status::OK(); + } + switch (array->type_id()) { + case arrow::Type::INT8: + values->push_back(std::dynamic_pointer_cast(array)->Value(index)); + break; + case arrow::Type::INT16: + values->push_back(std::dynamic_pointer_cast(array)->Value(index)); + break; + case arrow::Type::INT32: + values->push_back(std::dynamic_pointer_cast(array)->Value(index)); + break; + case arrow::Type::INT64: + values->push_back(std::dynamic_pointer_cast(array)->Value(index)); + break; + default: + return Status::Invalid(fmt::format( + "lumina int64 tag field has unsupported arrow type {}", array->type()->ToString())); + } + return Status::OK(); +} + +Status AppendDoubleValue(const std::shared_ptr& array, int64_t index, + std::vector* values) { + if (array->IsNull(index)) { + return Status::OK(); + } + switch (array->type_id()) { + case arrow::Type::FLOAT: + values->push_back(std::dynamic_pointer_cast(array)->Value(index)); + break; + case arrow::Type::DOUBLE: + values->push_back(std::dynamic_pointer_cast(array)->Value(index)); + break; + default: + return Status::Invalid( + fmt::format("lumina double tag field has unsupported arrow type {}", + array->type()->ToString())); + } + return Status::OK(); +} + +Status AppendStringValue(const std::shared_ptr& array, int64_t index, + std::vector* values) { + if (array->IsNull(index)) { + return Status::OK(); + } + auto string_array = std::dynamic_pointer_cast(array); + CHECK_NOT_NULL(string_array, + fmt::format("lumina string tag field has unsupported arrow type {}", + array->type()->ToString())); + auto view = string_array->GetView(index); + values->emplace_back(view.data(), view.size()); + return Status::OK(); +} + +template +Status ExtractTagValues(const std::shared_ptr& field_array, int64_t segment_start, + int64_t segment_len, + Status (*append_value)(const std::shared_ptr&, int64_t, + std::vector*), + std::vector>* values) { + values->resize(segment_len); + auto list_array = std::dynamic_pointer_cast(field_array); + if (list_array) { + auto child_values = list_array->values(); + for (int64_t i = 0; i < segment_len; i++) { + int64_t row = segment_start + i; + if (list_array->IsNull(row)) { + continue; + } + auto value_start = list_array->value_offset(row); + auto value_end = list_array->value_offset(row + 1); + auto& row_values = (*values)[i]; + row_values.reserve(value_end - value_start); + for (int64_t value_index = value_start; value_index < value_end; value_index++) { + PAIMON_RETURN_NOT_OK(append_value(child_values, value_index, &row_values)); + } + } + return Status::OK(); + } + + for (int64_t i = 0; i < segment_len; i++) { + PAIMON_RETURN_NOT_OK(append_value(field_array, segment_start + i, &(*values)[i])); + } + return Status::OK(); +} + +Result LiteralToTagValue(const Literal& literal) { + if (literal.IsNull()) { + return Status::Invalid("lumina tag predicate does not support null literal"); + } + switch (literal.GetType()) { + case FieldType::TINYINT: + return TagValue(static_cast(literal.GetValue())); + case FieldType::SMALLINT: + return TagValue(static_cast(literal.GetValue())); + case FieldType::INT: + return TagValue(static_cast(literal.GetValue())); + case FieldType::BIGINT: + return TagValue(literal.GetValue()); + case FieldType::FLOAT: + return TagValue(static_cast(literal.GetValue())); + case FieldType::DOUBLE: + return TagValue(literal.GetValue()); + case FieldType::STRING: + return TagValue(literal.GetValue()); + default: + return Status::Invalid( + fmt::format("lumina tag predicate does not support literal type {}", + static_cast(literal.GetType()))); + } +} + +Result GetSingleLiteral(const std::vector& literals, + const std::string& function_name) { + if (literals.size() != 1) { + return Status::Invalid( + fmt::format("lumina tag {} predicate requires one literal", function_name)); + } + return &literals[0]; +} + +Result LiteralsToTagValues(const std::vector& literals) { + if (literals.empty()) { + return Status::Invalid("lumina tag predicate IN requires at least one literal"); + } + + switch (literals[0].GetType()) { + case FieldType::TINYINT: + case FieldType::SMALLINT: + case FieldType::INT: + case FieldType::BIGINT: { + std::vector values; + values.reserve(literals.size()); + for (const auto& literal : literals) { + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(literal)); + auto typed_value = std::get_if(&value); + CHECK_NOT_NULL(typed_value, + "lumina tag predicate IN literals must have the same value type"); + values.push_back(*typed_value); + } + return TagValues(std::move(values)); + } + case FieldType::FLOAT: + case FieldType::DOUBLE: { + std::vector values; + values.reserve(literals.size()); + for (const auto& literal : literals) { + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(literal)); + auto typed_value = std::get_if(&value); + CHECK_NOT_NULL(typed_value, + "lumina tag predicate IN literals must have the same value type"); + values.push_back(*typed_value); + } + return TagValues(std::move(values)); + } + case FieldType::STRING: { + std::vector values; + values.reserve(literals.size()); + for (const auto& literal : literals) { + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(literal)); + auto typed_value = std::get_if(&value); + CHECK_NOT_NULL(typed_value, + "lumina tag predicate IN literals must have the same value type"); + values.push_back(std::move(*typed_value)); + } + return TagValues(std::move(values)); + } + default: + return Status::Invalid( + fmt::format("lumina tag predicate IN does not support literal type {}", + static_cast(literals[0].GetType()))); + } +} + +} // namespace + +Result> LuminaIndexWriter::ExtractTagDataForSegment( + const std::shared_ptr& struct_array, + const std::vector& tag_fields, int64_t segment_start, int64_t segment_len) { + std::vector tag_dimensions_data; + tag_dimensions_data.reserve(tag_fields.size()); + for (const auto& tag_field : tag_fields) { + auto field_array = struct_array->GetFieldByName(tag_field.name); + CHECK_NOT_NULL(field_array, + fmt::format("lumina tag field {} not in input array", tag_field.name)); + + TagDimensionData tag_dimension_data; + tag_dimension_data.tagkName = tag_field.name; + switch (tag_field.value_type) { + case LuminaTagField::ValueType::INT64: { + std::vector> values; + PAIMON_RETURN_NOT_OK(ExtractTagValues( + field_array, segment_start, segment_len, AppendInt64Value, &values)); + tag_dimension_data.values = std::move(values); + break; + } + case LuminaTagField::ValueType::DOUBLE: { + std::vector> values; + PAIMON_RETURN_NOT_OK(ExtractTagValues( + field_array, segment_start, segment_len, AppendDoubleValue, &values)); + tag_dimension_data.values = std::move(values); + break; + } + case LuminaTagField::ValueType::STRING: { + std::vector> values; + PAIMON_RETURN_NOT_OK(ExtractTagValues( + field_array, segment_start, segment_len, AppendStringValue, &values)); + tag_dimension_data.values = std::move(values); + break; + } + } + tag_dimensions_data.push_back(std::move(tag_dimension_data)); + } + return tag_dimensions_data; +} + +Result> LuminaGlobalIndex::ParseTagSchema( + const std::map& lumina_options) { + auto iter = lumina_options.find(std::string(::lumina::core::kExtensionTagSchema)); + if (iter == lumina_options.end()) { + return std::vector(); + } + + rapidjson::Document document; + document.Parse(iter->second.c_str()); + if (document.HasParseError()) { + return Status::Invalid("lumina tag_schema must be a valid JSON string"); + } + + std::vector tag_fields; + if (document.IsArray()) { + if (document.Empty()) { + return Status::Invalid("lumina tag_schema must contain at least one tag definition"); + } + tag_fields.reserve(document.Size()); + for (rapidjson::SizeType i = 0; i < document.Size(); i++) { + PAIMON_ASSIGN_OR_RAISE(LuminaTagField field, + ParseTagField(document[i], fmt::format("tag[{}]", i))); + tag_fields.push_back(std::move(field)); + } + } else if (document.IsObject()) { + PAIMON_ASSIGN_OR_RAISE(LuminaTagField field, ParseTagField(document, "tag[0]")); + tag_fields.push_back(std::move(field)); + } else { + return Status::Invalid("lumina tag_schema must be an object or array of objects"); + } + + std::unordered_set seen_names; + for (const auto& field : tag_fields) { + if (!seen_names.insert(field.name).second) { + return Status::Invalid( + fmt::format("lumina tag_schema has duplicate key_name: {}", field.name)); + } + } + return tag_fields; +} + +Status LuminaGlobalIndex::ValidateTagFields(const arrow::StructType& struct_type, + const std::vector& tag_fields) { + for (const auto& tag_field : tag_fields) { + auto field = struct_type.GetFieldByName(tag_field.name); + CHECK_NOT_NULL( + field, fmt::format("lumina tag field {} not exist in arrow schema", tag_field.name)); + PAIMON_RETURN_NOT_OK(ValidateTagArrowType(tag_field, field->type())); + } + return Status::OK(); +} + +Result<::lumina::extensions::experimental::TagFilter> LuminaIndexReader::PredicateToTagFilter( + const std::shared_ptr& predicate) { + if (!predicate) { + return Status::Invalid("lumina tag predicate must not be null"); + } + + auto compound_predicate = std::dynamic_pointer_cast(predicate); + if (compound_predicate) { + std::vector<::lumina::extensions::experimental::TagFilter> children; + children.reserve(compound_predicate->Children().size()); + for (const auto& child : compound_predicate->Children()) { + PAIMON_ASSIGN_OR_RAISE(::lumina::extensions::experimental::TagFilter tag_filter, + PredicateToTagFilter(child)); + children.push_back(std::move(tag_filter)); + } + if (children.empty()) { + return Status::Invalid("lumina tag compound predicate must have at least one child"); + } + if (children.size() == 1) { + return std::move(children.front()); + } + switch (compound_predicate->GetFunction().GetType()) { + case Function::Type::AND: + return ::lumina::extensions::experimental::TagFilter::And(std::move(children)); + case Function::Type::OR: + return ::lumina::extensions::experimental::TagFilter::Or(std::move(children)); + default: + return Status::NotImplemented( + fmt::format("lumina tag predicate does not support compound function {}", + compound_predicate->GetFunction().ToString())); + } + } + + auto leaf_predicate = std::dynamic_pointer_cast(predicate); + if (!leaf_predicate) { + return Status::Invalid( + fmt::format("cannot cast predicate {} to CompoundPredicate or LeafPredicate", + predicate->ToString())); + } + + const auto& literals = leaf_predicate->Literals(); + const auto& field_name = leaf_predicate->FieldName(); + switch (leaf_predicate->GetFunction().GetType()) { + case Function::Type::EQUAL: { + PAIMON_ASSIGN_OR_RAISE(const Literal* literal, GetSingleLiteral(literals, "equal")); + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(*literal)); + return ::lumina::extensions::experimental::TagFilter::Eq(field_name, std::move(value)); + } + case Function::Type::GREATER_THAN: { + PAIMON_ASSIGN_OR_RAISE(const Literal* literal, + GetSingleLiteral(literals, "greater than")); + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(*literal)); + return ::lumina::extensions::experimental::TagFilter::Gt(field_name, std::move(value)); + } + case Function::Type::GREATER_OR_EQUAL: { + PAIMON_ASSIGN_OR_RAISE(const Literal* literal, + GetSingleLiteral(literals, "greater or equal")); + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(*literal)); + return ::lumina::extensions::experimental::TagFilter::Gte(field_name, std::move(value)); + } + case Function::Type::LESS_THAN: { + PAIMON_ASSIGN_OR_RAISE(const Literal* literal, GetSingleLiteral(literals, "less than")); + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(*literal)); + return ::lumina::extensions::experimental::TagFilter::Lt(field_name, std::move(value)); + } + case Function::Type::LESS_OR_EQUAL: { + PAIMON_ASSIGN_OR_RAISE(const Literal* literal, + GetSingleLiteral(literals, "less or equal")); + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(*literal)); + return ::lumina::extensions::experimental::TagFilter::Lte(field_name, std::move(value)); + } + case Function::Type::IN: { + PAIMON_ASSIGN_OR_RAISE(TagValues values, LiteralsToTagValues(literals)); + return ::lumina::extensions::experimental::TagFilter::In(field_name, std::move(values)); + } + default: + return Status::NotImplemented( + fmt::format("lumina tag predicate does not support leaf function {}", + leaf_predicate->GetFunction().ToString())); + } +} + Result> LuminaGlobalIndex::CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, @@ -65,6 +536,8 @@ Result> LuminaGlobalIndex::CreateWriter( // check options auto lumina_options = OptionsUtils::FetchOptionsWithPrefix(LuminaDefines::kOptionKeyPrefix, options_); + PAIMON_ASSIGN_OR_RAISE(std::vector tag_fields, ParseTagSchema(lumina_options)); + PAIMON_RETURN_NOT_OK(ValidateTagFields(*struct_type, tag_fields)); PAIMON_ASSIGN_OR_RAISE(uint32_t dimension, OptionsUtils::GetValueFromMap( lumina_options, std::string(::lumina::core::kDimension))); @@ -76,7 +549,7 @@ Result> LuminaGlobalIndex::CreateWriter( auto lumina_pool = std::make_shared(pool); return std::make_shared( field_name, arrow_type, dimension, file_writer, std::move(builder_options), - ::lumina::api::IOOptions(), lumina_options, lumina_pool); + ::lumina::api::IOOptions(), lumina_options, std::move(tag_fields), lumina_pool); } Result LuminaIndexReader::GetIndexInfo( @@ -111,7 +584,9 @@ Result LuminaIndexReader::GetIndexInfo( return Status::Invalid( fmt::format("invalid distance type {} for lumina", distance_type_str)); } - return LuminaIndexReader::IndexInfo({dimension, index_type, distance_type}); + bool has_tag = lumina_write_options.find(std::string(::lumina::core::kExtensionTagSchema)) != + lumina_write_options.end(); + return LuminaIndexReader::IndexInfo({dimension, index_type, distance_type, has_tag}); } Result> LuminaGlobalIndex::CreateReader( @@ -170,8 +645,30 @@ Result> LuminaGlobalIndex::CreateReader( } auto searcher_with_filter = std::make_unique<::lumina::extensions::SearchWithFilterExtension>(); PAIMON_RETURN_NOT_OK_FROM_LUMINA(searcher->Attach(*searcher_with_filter)); + std::unique_ptr<::lumina::extensions::experimental::SearchWithTagExtension> searcher_with_tag; + if (index_info.has_tag) { + searcher_with_tag = + std::make_unique<::lumina::extensions::experimental::SearchWithTagExtension>(); + PAIMON_RETURN_NOT_OK_FROM_LUMINA(searcher->Attach(*searcher_with_tag)); + } return std::make_shared(index_info, std::move(searcher), - std::move(searcher_with_filter), lumina_pool); + std::move(searcher_with_filter), + std::move(searcher_with_tag), lumina_pool); +} + +Result>> LuminaGlobalIndex::GetExtraFieldNames() const { + auto lumina_options = + OptionsUtils::FetchOptionsWithPrefix(LuminaDefines::kOptionKeyPrefix, options_); + PAIMON_ASSIGN_OR_RAISE(std::vector tag_fields, ParseTagSchema(lumina_options)); + if (tag_fields.empty()) { + return std::optional>(std::nullopt); + } + std::vector field_names; + field_names.reserve(tag_fields.size()); + for (const auto& tag_field : tag_fields) { + field_names.push_back(tag_field.name); + } + return std::optional>(std::move(field_names)); } class LuminaDataset : public ::lumina::api::Dataset { @@ -221,14 +718,62 @@ class LuminaDataset : public ::lumina::api::Dataset { size_t cursor_ = 0; }; -LuminaIndexWriter::LuminaIndexWriter(const std::string& field_name, - const std::shared_ptr& arrow_type, - uint32_t dimension, - const std::shared_ptr& file_manager, - ::lumina::api::BuilderOptions&& builder_options, - ::lumina::api::IOOptions&& io_options, - const std::map& lumina_options, - const std::shared_ptr& pool) +class LuminaDatasetWithTag : public ::lumina::extensions::experimental::DatasetWithTag { + public: + LuminaDatasetWithTag(int64_t element_count, uint32_t dimension, + const std::vector>& array_vec, + const std::vector& start_ids, + const std::vector>& tag_data_vec) + : element_count_(element_count), + dimension_(dimension), + array_vec_(array_vec), + start_ids_(start_ids), + tag_data_vec_(tag_data_vec) {} + + uint32_t Dim() const noexcept override { + return dimension_; + } + uint64_t TotalSize() const noexcept override { + return element_count_; + } + + ::lumina::core::Result GetNextBatch( + std::vector& vector_buffer, std::vector<::lumina::core::vector_id_t>& id_buffer, + std::vector& tag_dimensions_data) noexcept override { + if (cursor_ >= array_vec_.size()) { + return ::lumina::core::Result::Ok(0); + } + auto& value_array = array_vec_[cursor_]; + int64_t value_array_length = value_array->length(); + int64_t element_count = value_array_length / dimension_; + const float* value_ptr = value_array->raw_values(); + vector_buffer.resize(value_array_length); + memcpy(vector_buffer.data(), value_ptr, sizeof(float) * value_array_length); + id_buffer.resize(element_count); + std::iota(id_buffer.begin(), id_buffer.end(), + static_cast<::lumina::core::vector_id_t>(start_ids_[cursor_])); + tag_dimensions_data = std::move(tag_data_vec_[cursor_]); + + value_array.reset(); + cursor_++; + return ::lumina::core::Result::Ok(static_cast(element_count)); + } + + private: + int64_t element_count_; + uint32_t dimension_; + std::vector> array_vec_; + std::vector start_ids_; + std::vector> tag_data_vec_; + size_t cursor_ = 0; +}; + +LuminaIndexWriter::LuminaIndexWriter( + const std::string& field_name, const std::shared_ptr& arrow_type, + uint32_t dimension, const std::shared_ptr& file_manager, + ::lumina::api::BuilderOptions&& builder_options, ::lumina::api::IOOptions&& io_options, + const std::map& lumina_options, + std::vector&& tag_fields, const std::shared_ptr& pool) : pool_(pool), field_name_(field_name), arrow_type_(arrow_type), @@ -236,7 +781,8 @@ LuminaIndexWriter::LuminaIndexWriter(const std::string& field_name, file_manager_(file_manager), builder_options_(std::move(builder_options)), io_options_(std::move(io_options)), - lumina_options_(lumina_options) {} + lumina_options_(lumina_options), + tag_fields_(std::move(tag_fields)) {} Status LuminaIndexWriter::AddBatch(::ArrowArray* arrow_array, std::vector&& relative_row_ids) { @@ -289,8 +835,17 @@ Status LuminaIndexWriter::AddBatch(::ArrowArray* arrow_array, "multiplied dimension [{}] must match length of field value array [{}]", segment_len, dimension_, sliced_values->length())); } + std::vector tag_data; + if (!tag_fields_.empty()) { + PAIMON_ASSIGN_OR_RAISE( + tag_data, ExtractTagDataForSegment(struct_array, tag_fields_, segment_start, + segment_len)); + } array_vec_.push_back(std::move(sliced_values)); array_start_ids_.push_back(count_ + segment_start); + if (!tag_fields_.empty()) { + tag_data_vec_.push_back(std::move(tag_data)); + } indexed_count_ += segment_len; segment_start = -1; } @@ -313,9 +868,19 @@ Result> LuminaIndexWriter::Finish() { PAIMON_RETURN_NOT_OK_FROM_LUMINA(builder.PretrainFrom(dataset1)); // insert data - LuminaDataset dataset2(indexed_count_, dimension_, array_vec_, array_start_ids_); - std::vector>().swap(array_vec_); - PAIMON_RETURN_NOT_OK_FROM_LUMINA(builder.InsertFrom(dataset2)); + if (tag_fields_.empty()) { + LuminaDataset dataset2(indexed_count_, dimension_, array_vec_, array_start_ids_); + std::vector>().swap(array_vec_); + PAIMON_RETURN_NOT_OK_FROM_LUMINA(builder.InsertFrom(dataset2)); + } else { + ::lumina::extensions::experimental::BuildWithTagExtension tag_extension; + PAIMON_RETURN_NOT_OK_FROM_LUMINA(builder.Attach(tag_extension)); + LuminaDatasetWithTag dataset2(indexed_count_, dimension_, array_vec_, array_start_ids_, + tag_data_vec_); + std::vector>().swap(array_vec_); + std::vector>().swap(tag_data_vec_); + PAIMON_RETURN_NOT_OK_FROM_LUMINA(tag_extension.InsertFromWithTag(dataset2)); + } // dump index PAIMON_ASSIGN_OR_RAISE(std::string index_file_name, @@ -338,17 +903,16 @@ LuminaIndexReader::LuminaIndexReader( const LuminaIndexReader::IndexInfo& index_info, std::unique_ptr<::lumina::api::LuminaSearcher>&& searcher, std::unique_ptr<::lumina::extensions::SearchWithFilterExtension>&& searcher_with_filter, + std::unique_ptr<::lumina::extensions::experimental::SearchWithTagExtension>&& searcher_with_tag, const std::shared_ptr& pool) : index_info_(index_info), pool_(pool), searcher_(std::move(searcher)), - searcher_with_filter_(std::move(searcher_with_filter)) {} + searcher_with_filter_(std::move(searcher_with_filter)), + searcher_with_tag_(std::move(searcher_with_tag)) {} Result> LuminaIndexReader::VisitVectorSearch( const std::shared_ptr& vector_search) { - if (vector_search->predicate) { - return Status::NotImplemented("lumina index not support predicate in VisitVectorSearch"); - } if (vector_search->distance_type && vector_search->distance_type.value() != index_info_.distance_type) { return Status::Invalid("distance type for index and search not match"); @@ -375,7 +939,25 @@ Result> LuminaIndexReader::VisitVectorS ::lumina::api::Query lumina_query(vector_search->query.data(), vector_search->query.size()); ::lumina::api::LuminaSearcher::SearchResult search_result; - if (!vector_search->pre_filter) { + if (vector_search->predicate) { + if (!searcher_with_tag_) { + return Status::Invalid("lumina index was not built with tag"); + } + PAIMON_ASSIGN_OR_RAISE(::lumina::extensions::experimental::TagFilter tag_filter, + PredicateToTagFilter(vector_search->predicate)); + if (!vector_search->pre_filter) { + PAIMON_ASSIGN_OR_RAISE_FROM_LUMINA( + search_result, searcher_with_tag_->SearchWithTag(lumina_query, tag_filter, + search_options, *pool_)); + } else { + auto lumina_filter = [filter = vector_search->pre_filter]( + ::lumina::core::vector_id_t id) -> bool { return filter(id); }; + PAIMON_ASSIGN_OR_RAISE_FROM_LUMINA( + search_result, + searcher_with_tag_->SearchWithTagAndFilter(lumina_query, tag_filter, lumina_filter, + search_options, *pool_)); + } + } else if (!vector_search->pre_filter) { PAIMON_ASSIGN_OR_RAISE_FROM_LUMINA(search_result, searcher_->Search(lumina_query, search_options, *pool_)); } else { diff --git a/src/paimon/global_index/lumina/lumina_global_index.h b/src/paimon/global_index/lumina/lumina_global_index.h index 81d092600..9ce03edfc 100644 --- a/src/paimon/global_index/lumina/lumina_global_index.h +++ b/src/paimon/global_index/lumina/lumina_global_index.h @@ -18,20 +18,42 @@ #include #include +#include #include #include +#include #include #include "arrow/api.h" #include "lumina/api/LuminaSearcher.h" #include "lumina/api/Options.h" #include "lumina/extensions/SearchWithFilterExtension.h" +#include "lumina/extensions/experimental/DatasetWithTag.h" +#include "lumina/extensions/experimental/SearchWithTagExtension.h" +#include "lumina/extensions/experimental/TagFilter.h" #include "paimon/global_index/bitmap_global_index_result.h" #include "paimon/global_index/global_indexer.h" #include "paimon/global_index/lumina/lumina_memory_pool.h" #include "paimon/global_index/lumina/lumina_utils.h" namespace paimon::lumina { +struct LuminaTagField { + enum class Type { + ENUM, + RANGE, + }; + + enum class ValueType { + INT64, + DOUBLE, + STRING, + }; + + std::string name; + Type type; + ValueType value_type; +}; + /// @note When enabling the lumina global index in `paimon-cpp`, all configuration parameters /// specific to Lumina **must be prefixed with `lumina.`**. /// See `docs/reference/OptionsReference.md` in the Lumina release package for more options. @@ -61,6 +83,8 @@ class LuminaGlobalIndex : public GlobalIndexer { explicit LuminaGlobalIndex(const std::map& options) : options_(options) {} + Result>> GetExtraFieldNames() const override; + Result> CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, @@ -72,6 +96,12 @@ class LuminaGlobalIndex : public GlobalIndexer { const std::shared_ptr& pool) const override; private: + static Result> ParseTagSchema( + const std::map& lumina_options); + + static Status ValidateTagFields(const arrow::StructType& struct_type, + const std::vector& tag_fields); + std::map options_; }; @@ -83,6 +113,7 @@ class LuminaIndexWriter : public GlobalIndexWriter { ::lumina::api::BuilderOptions&& builder_options, ::lumina::api::IOOptions&& io_options, const std::map& lumina_options, + std::vector&& tag_fields, const std::shared_ptr& pool); Status AddBatch(::ArrowArray* arrow_array, std::vector&& relative_row_ids) override; @@ -90,6 +121,11 @@ class LuminaIndexWriter : public GlobalIndexWriter { Result> Finish() override; private: + static Result> + ExtractTagDataForSegment(const std::shared_ptr& struct_array, + const std::vector& tag_fields, int64_t segment_start, + int64_t segment_len); + int64_t count_ = 0; int64_t indexed_count_ = 0; std::shared_ptr pool_; @@ -100,8 +136,10 @@ class LuminaIndexWriter : public GlobalIndexWriter { ::lumina::api::BuilderOptions builder_options_; ::lumina::api::IOOptions io_options_; std::map lumina_options_; + std::vector tag_fields_; std::vector> array_vec_; std::vector array_start_ids_; + std::vector> tag_data_vec_; }; class LuminaIndexReader : public GlobalIndexReader { @@ -110,11 +148,14 @@ class LuminaIndexReader : public GlobalIndexReader { uint32_t dimension; std::string index_type; VectorSearch::DistanceType distance_type; + bool has_tag; }; LuminaIndexReader( const IndexInfo& index_info, std::unique_ptr<::lumina::api::LuminaSearcher>&& searcher, std::unique_ptr<::lumina::extensions::SearchWithFilterExtension>&& searcher_with_filter, + std::unique_ptr<::lumina::extensions::experimental::SearchWithTagExtension>&& + searcher_with_tag, const std::shared_ptr& pool); ~LuminaIndexReader() override { @@ -201,9 +242,13 @@ class LuminaIndexReader : public GlobalIndexReader { static Result GetIndexInfo(const GlobalIndexIOMeta& io_meta); private: + static Result<::lumina::extensions::experimental::TagFilter> PredicateToTagFilter( + const std::shared_ptr& predicate); + LuminaIndexReader::IndexInfo index_info_; std::shared_ptr pool_; std::unique_ptr<::lumina::api::LuminaSearcher> searcher_; std::unique_ptr<::lumina::extensions::SearchWithFilterExtension> searcher_with_filter_; + std::unique_ptr<::lumina::extensions::experimental::SearchWithTagExtension> searcher_with_tag_; }; } // namespace paimon::lumina diff --git a/src/paimon/global_index/lumina/lumina_global_index_test.cpp b/src/paimon/global_index/lumina/lumina_global_index_test.cpp index e5aeca71a..d762c2a9b 100644 --- a/src/paimon/global_index/lumina/lumina_global_index_test.cpp +++ b/src/paimon/global_index/lumina/lumina_global_index_test.cpp @@ -250,6 +250,390 @@ TEST_F(LuminaGlobalIndexTest, TestWithFilter) { } } +TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithTagFilter) { + auto test_root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(test_root_dir); + std::string test_root = test_root_dir->Str(); + + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"color","type":"enum","value_type":"string"}])"; + + std::shared_ptr tag_data_type = arrow::struct_( + {arrow::field("f0", arrow::list(arrow::float32())), arrow::field("color", arrow::utf8())}); + std::shared_ptr tag_array = + arrow::ipc::internal::json::ArrayFromJSON(tag_data_type, + R"([ + [[0.0, 0.0, 0.0, 0.0], "cold"], + [[0.0, 1.0, 0.0, 1.0], "warm"], + [[1.0, 0.0, 1.0, 0.0], "cold"], + [[1.0, 1.0, 1.0, 1.0], "warm"] + ])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN( + GlobalIndexIOMeta meta, + WriteGlobalIndex(test_root, tag_data_type, tag_options, tag_array, Range(0, 3))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr reader, + CreateGlobalIndexReader(test_root, data_type_, tag_options, meta)); + + std::shared_ptr predicate = PredicateBuilder::Equal( + /*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "warm", 4)); + { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr scored_result, + reader->VisitVectorSearch(std::make_shared( + /*field_name=*/"f0", /*limit=*/4, query_, /*filter=*/nullptr, predicate, + /*distance_type=*/std::nullopt, /*options=*/tag_options))); + CheckResult(scored_result, {3l, 1l}, {0.01f, 2.01f}); + } + { + auto pre_filter = [](int64_t id) -> bool { return id < 3; }; + ASSERT_OK_AND_ASSIGN(std::shared_ptr filtered_scored_result, + reader->VisitVectorSearch(std::make_shared( + /*field_name=*/"f0", /*limit=*/4, query_, pre_filter, predicate, + /*distance_type=*/std::nullopt, /*options=*/tag_options))); + CheckResult(filtered_scored_result, {1l}, {2.01f}); + } +} + +TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithMixedTagPredicates) { + auto test_root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(test_root_dir); + std::string test_root = test_root_dir->Str(); + + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"color","type":"enum","value_type":"string"},)" + R"({"key_name":"price","type":"range","value_type":"double"}])"; + + std::shared_ptr tag_data_type = arrow::struct_( + {arrow::field("f0", arrow::list(arrow::float32())), arrow::field("color", arrow::utf8()), + arrow::field("price", arrow::float64())}); + std::shared_ptr tag_array = + arrow::ipc::internal::json::ArrayFromJSON(tag_data_type, + R"([ + [[0.0, 0.0, 0.0, 0.0], "cold", 5.0], + [[0.0, 1.0, 0.0, 1.0], "warm", 10.0], + [[1.0, 0.0, 1.0, 0.0], "warm", 20.0], + [[1.0, 1.0, 1.0, 1.0], "warm", 30.0] + ])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN( + GlobalIndexIOMeta meta, + WriteGlobalIndex(test_root, tag_data_type, tag_options, tag_array, Range(0, 3))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr reader, + CreateGlobalIndexReader(test_root, data_type_, tag_options, meta)); + + std::shared_ptr color_predicate = PredicateBuilder::Equal( + /*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "warm", 4)); + std::shared_ptr low_price_predicate = PredicateBuilder::LessOrEqual( + /*field_index=*/2, /*field_name=*/"price", FieldType::DOUBLE, Literal(10.0)); + std::shared_ptr high_price_predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/2, /*field_name=*/"price", FieldType::DOUBLE, Literal(30.0)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr price_predicate, + PredicateBuilder::Or({low_price_predicate, high_price_predicate})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::And({color_predicate, price_predicate})); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr scored_result, + reader->VisitVectorSearch(std::make_shared( + /*field_name=*/"f0", /*limit=*/4, query_, /*filter=*/nullptr, predicate, + /*distance_type=*/std::nullopt, /*options=*/tag_options))); + CheckResult(scored_result, {3l, 1l}, {0.01f, 2.01f}); +} + +TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithIntegerListTagFilter) { + auto test_root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(test_root_dir); + std::string test_root = test_root_dir->Str(); + + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"category_ids","type":"enum","value_type":"int64"}])"; + + std::shared_ptr tag_data_type = + arrow::struct_({arrow::field("f0", arrow::list(arrow::float32())), + arrow::field("category_ids", arrow::list(arrow::int64()))}); + std::shared_ptr tag_array = + arrow::ipc::internal::json::ArrayFromJSON(tag_data_type, + R"([ + [[0.0, 0.0, 0.0, 0.0], [1, 2]], + [[0.0, 1.0, 0.0, 1.0], [3, 8]], + [[1.0, 0.0, 1.0, 0.0], [4, 5]], + [[1.0, 1.0, 1.0, 1.0], [6, 9]] + ])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN( + GlobalIndexIOMeta meta, + WriteGlobalIndex(test_root, tag_data_type, tag_options, tag_array, Range(0, 3))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr reader, + CreateGlobalIndexReader(test_root, data_type_, tag_options, meta)); + + std::shared_ptr predicate = PredicateBuilder::In( + /*field_index=*/1, /*field_name=*/"category_ids", FieldType::BIGINT, + {Literal(8l), Literal(9l)}); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr scored_result, + reader->VisitVectorSearch(std::make_shared( + /*field_name=*/"f0", /*limit=*/4, query_, /*filter=*/nullptr, predicate, + /*distance_type=*/std::nullopt, /*options=*/tag_options))); + CheckResult(scored_result, {3l, 1l}, {0.01f, 2.01f}); +} + +TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithCompatibleTagArrowTypes) { + auto test_root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(test_root_dir); + std::string test_root = test_root_dir->Str(); + + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"tag_i8","type":"enum","value_type":"int64"},)" + R"({"key_name":"tag_i16","type":"enum","value_type":"int64"},)" + R"({"key_name":"tag_i32","type":"enum","value_type":"int64"},)" + R"({"key_name":"tag_f32","type":"range","value_type":"double"}])"; + + std::shared_ptr tag_data_type = arrow::struct_( + {arrow::field("f0", arrow::list(arrow::float32())), arrow::field("tag_i8", arrow::int8()), + arrow::field("tag_i16", arrow::int16()), arrow::field("tag_i32", arrow::int32()), + arrow::field("tag_f32", arrow::float32())}); + std::shared_ptr tag_array = + arrow::ipc::internal::json::ArrayFromJSON(tag_data_type, + R"([ + [[0.0, 0.0, 0.0, 0.0], 1, 10, 100, 1.5], + [[0.0, 1.0, 0.0, 1.0], 2, 20, 200, 2.5], + [[1.0, 0.0, 1.0, 0.0], 3, 30, 300, 3.5], + [[1.0, 1.0, 1.0, 1.0], 4, 40, 400, 4.5] + ])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN( + GlobalIndexIOMeta meta, + WriteGlobalIndex(test_root, tag_data_type, tag_options, tag_array, Range(0, 3))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr reader, + CreateGlobalIndexReader(test_root, data_type_, tag_options, meta)); + + auto search_and_check = [&](const std::shared_ptr& predicate, + const std::vector& expected_ids, + const std::vector& expected_scores) { + ASSERT_OK_AND_ASSIGN( + std::shared_ptr scored_result, + reader->VisitVectorSearch(std::make_shared( + /*field_name=*/"f0", /*limit=*/4, query_, /*filter=*/nullptr, predicate, + /*distance_type=*/std::nullopt, /*options=*/tag_options))); + CheckResult(scored_result, expected_ids, expected_scores); + }; + + search_and_check(PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"tag_i8", + FieldType::TINYINT, Literal(static_cast(2))), + {1l}, {2.01f}); + search_and_check( + PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"tag_i16", FieldType::SMALLINT, + Literal(static_cast(30))), + {2l}, {2.21f}); + search_and_check(PredicateBuilder::Equal(/*field_index=*/3, /*field_name=*/"tag_i32", + FieldType::INT, Literal(400)), + {3l}, {0.01f}); + search_and_check(PredicateBuilder::GreaterThan(/*field_index=*/4, /*field_name=*/"tag_f32", + FieldType::FLOAT, Literal(4.0f)), + {3l}, {0.01f}); +} + +TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithTagNullAndEmptyValues) { + auto test_root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(test_root_dir); + std::string test_root = test_root_dir->Str(); + + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"color","type":"enum","value_type":"string"},)" + R"({"key_name":"labels","type":"enum","value_type":"string"},)" + R"({"key_name":"price","type":"range","value_type":"double"},)" + R"({"key_name":"scores","type":"enum","value_type":"double"},)" + R"({"key_name":"category","type":"enum","value_type":"int64"},)" + R"({"key_name":"category_ids","type":"enum","value_type":"int64"}])"; + + std::shared_ptr tag_data_type = arrow::struct_( + {arrow::field("f0", arrow::list(arrow::float32())), arrow::field("color", arrow::utf8()), + arrow::field("labels", arrow::list(arrow::utf8())), + arrow::field("price", arrow::float64()), + arrow::field("scores", arrow::list(arrow::float64())), + arrow::field("category", arrow::int64()), + arrow::field("category_ids", arrow::list(arrow::int64()))}); + std::shared_ptr tag_array = + arrow::ipc::internal::json::ArrayFromJSON(tag_data_type, + R"([ + [[0.0, 0.0, 0.0, 0.0], "red", ["hot"], 5.0, [0.25], 7, [1, 2]], + [null, "red", ["vip"], 8.0, [0.8], 7, [9]], + [[0.0, 1.0, 0.0, 1.0], null, null, null, null, null, null], + [[1.0, 0.0, 1.0, 0.0], " ", [], 20.0, [], 0, []], + [[1.0, 1.0, 1.0, 1.0], "blue", ["vip", null], 30.0, [null, 0.75], 3, [null, 9]] + ])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN( + GlobalIndexIOMeta meta, + WriteGlobalIndex(test_root, tag_data_type, tag_options, tag_array, Range(0, 4))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr reader, + CreateGlobalIndexReader(test_root, data_type_, tag_options, meta)); + auto search_and_check_with_filter = + [&](VectorSearch::PreFilter pre_filter, const std::shared_ptr& predicate, + const std::vector& expected_ids, const std::vector& expected_scores) { + std::shared_ptr vector_search = std::make_shared( + /*field_name=*/"f0", /*limit=*/5, query_, pre_filter, predicate, + /*distance_type=*/std::nullopt, /*options=*/tag_options); + ASSERT_OK_AND_ASSIGN(std::shared_ptr scored_result, + reader->VisitVectorSearch(vector_search)); + CheckResult(scored_result, expected_ids, expected_scores); + }; + auto search_and_check = [&](const std::shared_ptr& predicate, + const std::vector& expected_ids, + const std::vector& expected_scores) { + search_and_check_with_filter(/*pre_filter=*/nullptr, predicate, expected_ids, + expected_scores); + }; + auto search_and_check_error = [&](const std::shared_ptr& predicate, + const std::string& expected_message) { + ASSERT_NOK_WITH_MSG( + reader->VisitVectorSearch(std::make_shared( + /*field_name=*/"f0", /*limit=*/5, query_, /*filter=*/nullptr, predicate, + /*distance_type=*/std::nullopt, /*options=*/tag_options)), + expected_message); + }; + + search_and_check(/*predicate=*/nullptr, {4l, 2l, 3l, 0l}, {0.01f, 2.01f, 2.21f, 4.21f}); + search_and_check( + PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "red", 3)), + {0l}, {4.21f}); + search_and_check(PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"color", + FieldType::STRING, Literal(FieldType::STRING, " ", 1)), + {3l}, {2.21f}); + search_and_check( + PredicateBuilder::In(/*field_index=*/2, /*field_name=*/"labels", FieldType::STRING, + {Literal(FieldType::STRING, "vip", 3)}), + {4l}, {0.01f}); + search_and_check(PredicateBuilder::LessOrEqual(/*field_index=*/3, /*field_name=*/"price", + FieldType::DOUBLE, Literal(10.0)), + {0l}, {4.21f}); + search_and_check(PredicateBuilder::LessThan(/*field_index=*/3, /*field_name=*/"price", + FieldType::DOUBLE, Literal(10.0)), + {0l}, {4.21f}); + search_and_check(PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"price", + FieldType::DOUBLE, Literal(10.0)), + {4l, 3l}, {0.01f, 2.21f}); + search_and_check(PredicateBuilder::In(/*field_index=*/4, /*field_name=*/"scores", + FieldType::DOUBLE, {Literal(0.25), Literal(0.75)}), + {4l, 0l}, {0.01f, 4.21f}); + search_and_check(PredicateBuilder::Equal(/*field_index=*/5, /*field_name=*/"category", + FieldType::BIGINT, Literal(7l)), + {0l}, {4.21f}); + search_and_check(PredicateBuilder::In(/*field_index=*/6, /*field_name=*/"category_ids", + FieldType::BIGINT, {Literal(9l)}), + {4l}, {0.01f}); + search_and_check( + PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "green", 5)), + {}, {}); + search_and_check_error( + PredicateBuilder::NotIn(/*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + {Literal(FieldType::STRING, "red", 3)}), + "lumina tag predicate does not support leaf function NotIn"); + search_and_check_error( + PredicateBuilder::NotEqual(/*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "red", 3)), + "lumina tag predicate does not support leaf function NotEqual"); + search_and_check_error( + PredicateBuilder::Equal(/*field_index=*/7, /*field_name=*/"unknown", FieldType::STRING, + Literal(FieldType::STRING, "red", 3)), + "unknown tag key 'unknown' in label filter"); + search_and_check_error(PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"color", + FieldType::BIGINT, Literal(1l)), + "tag value type mismatch for key 'color'"); + search_and_check_error( + PredicateBuilder::Equal(/*field_index=*/3, /*field_name=*/"price", FieldType::STRING, + Literal(FieldType::STRING, "x", 1)), + "tag value type mismatch for key 'price'"); + { + std::shared_ptr predicate = PredicateBuilder::Equal( + /*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "red", 3)); + search_and_check_with_filter([](int64_t id) { return id == 0 || id == 4; }, predicate, {0l}, + {4.21f}); + search_and_check_with_filter([](int64_t id) { return id == 4; }, predicate, {}, {}); + } + { + std::shared_ptr red_predicate = PredicateBuilder::Equal( + /*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "red", 3)); + std::shared_ptr blue_predicate = PredicateBuilder::Equal( + /*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "blue", 4)); + std::shared_ptr high_price_predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/3, /*field_name=*/"price", FieldType::DOUBLE, Literal(30.0)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr blue_high_price_predicate, + PredicateBuilder::And({blue_predicate, high_price_predicate})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr compound_predicate, + PredicateBuilder::Or({red_predicate, blue_high_price_predicate})); + search_and_check(compound_predicate, {0l, 4l}, {4.21f, 0.01f}); + search_and_check_with_filter([](int64_t id) { return id == 4; }, compound_predicate, {4l}, + {0.01f}); + } +} + +TEST_F(LuminaGlobalIndexTest, TestTagSchemaValidation) { + auto test_root_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(test_root_dir); + std::string index_root = test_root_dir->Str(); + + std::shared_ptr tag_data_type = arrow::struct_( + {arrow::field("f0", arrow::list(arrow::float32())), arrow::field("color", arrow::utf8())}); + + { + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"color","type":"range","value_type":"string"}])"; + ASSERT_NOK_WITH_MSG( + WriteGlobalIndex(index_root, tag_data_type, tag_options, array_, Range(0, 3)), + "lumina tag_schema tag[0] range type does not support string value_type"); + } + { + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"color","type":"enum","value_type":"int64"}])"; + ASSERT_NOK_WITH_MSG( + WriteGlobalIndex(index_root, tag_data_type, tag_options, array_, Range(0, 3)), + "lumina tag field color type string is not compatible with tag_schema value_type"); + } +} + +TEST_F(LuminaGlobalIndexTest, TestGetExtraFieldNames) { + { + LuminaGlobalIndex global_index(options_); + ASSERT_OK_AND_ASSIGN(std::optional> field_names, + global_index.GetExtraFieldNames()); + ASSERT_FALSE(field_names); + } + { + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"color","type":"enum","value_type":"string"},)" + R"({"key_name":"price","type":"range","value_type":"double"},)" + R"({"key_name":"category_ids","type":"enum","value_type":"int64"}])"; + LuminaGlobalIndex global_index(tag_options); + ASSERT_OK_AND_ASSIGN(std::optional> field_names, + global_index.GetExtraFieldNames()); + ASSERT_TRUE(field_names); + ASSERT_EQ(field_names.value(), + std::vector({"color", "price", "category_ids"})); + } +} + TEST_F(LuminaGlobalIndexTest, TestInvalidInputs) { auto test_root_dir = paimon::test::UniqueTestDirectory::Create(); ASSERT_TRUE(test_root_dir); @@ -375,7 +759,7 @@ TEST_F(LuminaGlobalIndexTest, TestInvalidInputs) { FieldType::BIGINT, Literal(5l)), /*distance_type=*/std::nullopt, /*options=*/std::map())), - "lumina index not support predicate in VisitVectorSearch"); + "lumina index was not built with tag"); } { ASSERT_OK_AND_ASSIGN(auto reader, diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp index d59c11089..ab3a03d10 100644 --- a/test/inte/global_index_test.cpp +++ b/test/inte/global_index_test.cpp @@ -1133,6 +1133,158 @@ TEST_P(GlobalIndexTest, TestWriteCommitScanReadIndexWithScore) { ASSERT_FALSE(typed_result->bitmap_.Contains(7)); } } + +TEST_P(GlobalIndexTest, TestWriteAndQueryLuminaIndexWithTagNullAndEmptyValues) { + arrow::FieldVector fields = {arrow::field("name", arrow::utf8()), + arrow::field("embedding", arrow::list(arrow::float32())), + arrow::field("color", arrow::utf8()), + arrow::field("labels", arrow::list(arrow::utf8())), + arrow::field("price", arrow::float64()), + arrow::field("scores", arrow::list(arrow::float64())), + arrow::field("category", arrow::int64()), + arrow::field("category_ids", arrow::list(arrow::int64()))}; + auto schema = arrow::schema(fields); + std::map lumina_options = { + {"lumina.index.dimension", "4"}, + {"lumina.index.type", "bruteforce"}, + {"lumina.distance.metric", "l2"}, + {"lumina.encoding.type", "rawf32"}, + {"lumina.search.parallel_number", "10"}, + {"lumina.extension.build.tag.tag_schema", + R"([{"key_name":"color","type":"enum","value_type":"string"},)" + R"({"key_name":"labels","type":"enum","value_type":"string"},)" + R"({"key_name":"price","type":"range","value_type":"double"},)" + R"({"key_name":"scores","type":"range","value_type":"double"},)" + R"({"key_name":"category","type":"enum","value_type":"int64"},)" + R"({"key_name":"category_ids","type":"enum","value_type":"int64"}])"}}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, file_format_}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(/*partition_keys=*/{}, schema, options); + + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + std::vector write_cols = schema->field_names(); + auto src_array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ +["row0", [0.0, 0.0, 0.0, 0.0], "red", ["hot"], 5.0, [0.25], 7, [1, 2]], +["row1", null, "red", ["vip"], 8.0, [0.8], 7, [9]], +["row2", [0.0, 1.0, 0.0, 1.0], null, null, null, null, null, null], +["row3", [1.0, 0.0, 1.0, 0.0], " ", [], 20.0, [], 0, []], +["row4", [1.0, 1.0, 1.0, 1.0], "blue", ["vip", null], 30.0, [null, 0.75], 3, [null, 9]] + ])") + .ValueOrDie(); + + ASSERT_OK_AND_ASSIGN(auto commit_msgs, WriteArray(table_path, write_cols, src_array)); + ASSERT_OK(Commit(table_path, commit_msgs)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr split, + ScanData(table_path, /*partition_filters=*/{})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr index_commit_msg, + GlobalIndexWriteTask::WriteIndex( + table_path, "embedding", "lumina", + std::make_shared(split, std::vector({Range(0, 4)})), + /*options=*/lumina_options, pool_, fs_)); + + std::shared_ptr index_commit_msg_impl = + std::dynamic_pointer_cast(index_commit_msg); + ASSERT_TRUE(index_commit_msg_impl); + const auto& new_index_files = index_commit_msg_impl->GetNewFilesIncrement().NewIndexFiles(); + ASSERT_EQ(new_index_files.size(), 1u); + ASSERT_EQ(new_index_files[0]->IndexType(), "lumina"); + ASSERT_EQ(new_index_files[0]->RowCount(), 5); + const std::optional& global_index_meta = + new_index_files[0]->GetGlobalIndexMeta(); + ASSERT_TRUE(global_index_meta); + std::string expected_index_meta_json = + R"({"distance.metric":"l2","encoding.type":"rawf32","extension.build.tag.tag_schema":"[{\"key_name\":\"color\",\"type\":\"enum\",\"value_type\":\"string\"},{\"key_name\":\"labels\",\"type\":\"enum\",\"value_type\":\"string\"},{\"key_name\":\"price\",\"type\":\"range\",\"value_type\":\"double\"},{\"key_name\":\"scores\",\"type\":\"range\",\"value_type\":\"double\"},{\"key_name\":\"category\",\"type\":\"enum\",\"value_type\":\"int64\"},{\"key_name\":\"category_ids\",\"type\":\"enum\",\"value_type\":\"int64\"}]","index.dimension":"4","index.type":"bruteforce","search.parallel_number":"10"})"; + GlobalIndexMeta expected_global_index_meta( + /*row_range_start=*/0, /*row_range_end=*/4, /*index_field_id=*/1, + /*extra_field_ids=*/std::optional>({2, 3, 4, 5, 6, 7}), + std::make_shared(expected_index_meta_json, pool_.get())); + ASSERT_EQ(global_index_meta.value(), expected_global_index_meta); + + ASSERT_OK(Commit(table_path, {index_commit_msg})); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr global_index_scan, + GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, + /*partitions=*/std::nullopt, lumina_options, fs_, + /*executor=*/nullptr, pool_)); + ASSERT_OK_AND_ASSIGN(auto lumina_readers, + global_index_scan->CreateReaders("embedding", std::nullopt)); + ASSERT_EQ(lumina_readers.size(), 1u); + + std::vector query = {1.0f, 1.0f, 1.0f, 1.1f}; + auto search_and_check_with_filter = [&](VectorSearch::PreFilter pre_filter, + const std::shared_ptr& predicate, + const std::string& expected) { + std::shared_ptr vector_search = std::make_shared( + "embedding", /*limit=*/5, query, pre_filter, predicate, + /*distance_type=*/std::nullopt, /*options=*/lumina_options); + ASSERT_OK_AND_ASSIGN(std::shared_ptr scored_result, + lumina_readers[0]->VisitVectorSearch(vector_search)); + ASSERT_EQ(scored_result->ToString(), expected); + }; + auto search_and_check = [&](const std::shared_ptr& predicate, + const std::string& expected) { + search_and_check_with_filter(/*pre_filter=*/nullptr, predicate, expected); + }; + + search_and_check(/*predicate=*/nullptr, "row ids: {0,2,3,4}, scores: {4.21,2.01,2.21,0.01}"); + search_and_check( + PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "red", 3)), + "row ids: {0}, scores: {4.21}"); + search_and_check(PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"color", + FieldType::STRING, Literal(FieldType::STRING, " ", 1)), + "row ids: {3}, scores: {2.21}"); + search_and_check( + PredicateBuilder::In(/*field_index=*/3, /*field_name=*/"labels", FieldType::STRING, + {Literal(FieldType::STRING, "vip", 3)}), + "row ids: {4}, scores: {0.01}"); + search_and_check(PredicateBuilder::LessOrEqual(/*field_index=*/4, /*field_name=*/"price", + FieldType::DOUBLE, Literal(10.0)), + "row ids: {0}, scores: {4.21}"); + search_and_check(PredicateBuilder::GreaterOrEqual(/*field_index=*/5, /*field_name=*/"scores", + FieldType::DOUBLE, Literal(0.5)), + "row ids: {4}, scores: {0.01}"); + search_and_check(PredicateBuilder::Equal(/*field_index=*/6, /*field_name=*/"category", + FieldType::BIGINT, Literal(7l)), + "row ids: {0}, scores: {4.21}"); + search_and_check(PredicateBuilder::In(/*field_index=*/7, /*field_name=*/"category_ids", + FieldType::BIGINT, {Literal(9l)}), + "row ids: {4}, scores: {0.01}"); + search_and_check( + PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "green", 5)), + "row ids: {}, scores: {}"); + { + std::shared_ptr predicate = PredicateBuilder::Equal( + /*field_index=*/2, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "red", 3)); + search_and_check_with_filter([](int64_t id) { return id == 0 || id == 4; }, predicate, + "row ids: {0}, scores: {4.21}"); + search_and_check_with_filter([](int64_t id) { return id == 4; }, predicate, + "row ids: {}, scores: {}"); + } + { + std::shared_ptr red_predicate = PredicateBuilder::Equal( + /*field_index=*/2, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "red", 3)); + std::shared_ptr blue_predicate = PredicateBuilder::Equal( + /*field_index=*/2, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "blue", 4)); + std::shared_ptr high_price_predicate = PredicateBuilder::GreaterOrEqual( + /*field_index=*/4, /*field_name=*/"price", FieldType::DOUBLE, Literal(30.0)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr blue_high_price_predicate, + PredicateBuilder::And({blue_predicate, high_price_predicate})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr compound_predicate, + PredicateBuilder::Or({red_predicate, blue_high_price_predicate})); + search_and_check(compound_predicate, "row ids: {0,4}, scores: {4.21,0.01}"); + search_and_check_with_filter([](int64_t id) { return id == 4; }, compound_predicate, + "row ids: {4}, scores: {0.01}"); + } +} #endif TEST_P(GlobalIndexTest, TestDataEvolutionBatchScan) { From d4d2b385426e1fdfec8bb880aef86539b3450c9e Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Wed, 8 Jul 2026 22:00:40 +0800 Subject: [PATCH 2/8] fix little --- include/paimon/global_index/global_index_io_meta.h | 4 ---- .../global_index/btree/btree_compatibility_test.cpp | 3 ++- .../btree/btree_file_meta_selector_test.cpp | 12 ++++++------ .../btree/btree_global_index_integration_test.cpp | 2 +- .../global_index/btree/btree_global_index_writer.cpp | 3 ++- .../global_index/wrap/file_index_writer_wrapper.h | 2 +- .../lucene/lucene_global_index_writer.cpp | 2 +- .../global_index/lumina/lumina_global_index.cpp | 2 +- .../tantivy/tantivy_equivalence_test.cpp | 4 ++-- .../tantivy/tantivy_global_index_writer.cpp | 2 +- .../tantivy/tantivy_java_compat_test.cpp | 6 ++++-- 11 files changed, 21 insertions(+), 21 deletions(-) diff --git a/include/paimon/global_index/global_index_io_meta.h b/include/paimon/global_index/global_index_io_meta.h index 600d278f6..409983e94 100644 --- a/include/paimon/global_index/global_index_io_meta.h +++ b/include/paimon/global_index/global_index_io_meta.h @@ -28,10 +28,6 @@ namespace paimon { /// Metadata describing a single file entry in a global index. struct PAIMON_EXPORT GlobalIndexIOMeta { - GlobalIndexIOMeta(const std::string& _file_path, int64_t _file_size, - const std::shared_ptr& _metadata) - : GlobalIndexIOMeta(_file_path, _file_size, _metadata, std::nullopt) {} - GlobalIndexIOMeta(const std::string& _file_path, int64_t _file_size, const std::shared_ptr& _metadata, const std::optional>& _extra_field_ids) diff --git a/src/paimon/common/global_index/btree/btree_compatibility_test.cpp b/src/paimon/common/global_index/btree/btree_compatibility_test.cpp index 420d16ed1..af349a4de 100644 --- a/src/paimon/common/global_index/btree/btree_compatibility_test.cpp +++ b/src/paimon/common/global_index/btree/btree_compatibility_test.cpp @@ -115,7 +115,8 @@ class BTreeCompatibilityTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE(auto file_status, fs_->GetFileStatus(bin_path)); auto file_size = file_status->GetLen(); - GlobalIndexIOMeta io_meta(bin_path, file_size, meta_bytes); + GlobalIndexIOMeta io_meta(bin_path, file_size, meta_bytes, + /*extra_field_ids=*/std::nullopt); std::vector metas = {io_meta}; auto schema = arrow::schema({arrow::field("testField", arrow_type)}); diff --git a/src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp b/src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp index 31e45f642..ca126009d 100644 --- a/src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp +++ b/src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp @@ -48,12 +48,12 @@ class BTreeFileMetaSelectorTest : public ::testing::Test { // file6: only-nulls file (no keys, has_nulls=true) auto meta6 = std::make_shared(nullptr, nullptr, true); files_ = { - GlobalIndexIOMeta("file1", 1, meta1->Serialize(pool_.get())), - GlobalIndexIOMeta("file2", 1, meta2->Serialize(pool_.get())), - GlobalIndexIOMeta("file3", 1, meta3->Serialize(pool_.get())), - GlobalIndexIOMeta("file4", 1, meta4->Serialize(pool_.get())), - GlobalIndexIOMeta("file5", 1, meta5->Serialize(pool_.get())), - GlobalIndexIOMeta("file6", 1, meta6->Serialize(pool_.get())), + GlobalIndexIOMeta("file1", 1, meta1->Serialize(pool_.get()), std::nullopt), + GlobalIndexIOMeta("file2", 1, meta2->Serialize(pool_.get()), std::nullopt), + GlobalIndexIOMeta("file3", 1, meta3->Serialize(pool_.get()), std::nullopt), + GlobalIndexIOMeta("file4", 1, meta4->Serialize(pool_.get()), std::nullopt), + GlobalIndexIOMeta("file5", 1, meta5->Serialize(pool_.get()), std::nullopt), + GlobalIndexIOMeta("file6", 1, meta6->Serialize(pool_.get()), std::nullopt), }; } diff --git a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp index 3d0751a18..1f577d420 100644 --- a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp +++ b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp @@ -1703,7 +1703,7 @@ TEST_P(BTreeGlobalIndexIntegrationTest, CreateReaderWithMultiFieldSchema) { {BtreeDefs::kBtreeIndexCompression, GetParam()}}; ASSERT_OK_AND_ASSIGN(auto indexer, BTreeGlobalIndexer::Create(options)); - GlobalIndexIOMeta meta("fake_path", 100, nullptr); + GlobalIndexIOMeta meta("fake_path", 100, nullptr, /*extra_field_ids=*/std::nullopt); std::vector metas = {meta}; ASSERT_NOK_WITH_MSG(indexer->CreateReader(c_schema.get(), file_reader, metas, pool_), diff --git a/src/paimon/common/global_index/btree/btree_global_index_writer.cpp b/src/paimon/common/global_index/btree/btree_global_index_writer.cpp index bfb99e8b6..5abf6fe52 100644 --- a/src/paimon/common/global_index/btree/btree_global_index_writer.cpp +++ b/src/paimon/common/global_index/btree/btree_global_index_writer.cpp @@ -199,7 +199,8 @@ Result> BTreeGlobalIndexWriter::Finish() { // Create GlobalIndexIOMeta std::string file_path = file_writer_->ToPath(index_file_name_); PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_writer_->GetFileSize(index_file_name_)); - GlobalIndexIOMeta io_meta(file_path, file_size, meta_bytes); + GlobalIndexIOMeta io_meta(file_path, file_size, meta_bytes, + /*extra_field_ids=*/std::nullopt); return std::vector{io_meta}; } diff --git a/src/paimon/common/global_index/wrap/file_index_writer_wrapper.h b/src/paimon/common/global_index/wrap/file_index_writer_wrapper.h index 02e40b49d..c9c891a29 100644 --- a/src/paimon/common/global_index/wrap/file_index_writer_wrapper.h +++ b/src/paimon/common/global_index/wrap/file_index_writer_wrapper.h @@ -66,7 +66,7 @@ class FileIndexWriterWrapper : public GlobalIndexWriter { PAIMON_RETURN_NOT_OK(out->Flush()); PAIMON_RETURN_NOT_OK(out->Close()); GlobalIndexIOMeta meta(file_manager_->ToPath(file_name), /*file_size=*/bytes->size(), - /*metadata=*/nullptr); + /*metadata=*/nullptr, /*extra_field_ids=*/std::nullopt); return std::vector({meta}); } diff --git a/src/paimon/global_index/lucene/lucene_global_index_writer.cpp b/src/paimon/global_index/lucene/lucene_global_index_writer.cpp index 4290565c4..936c536bd 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_writer.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index_writer.cpp @@ -237,7 +237,7 @@ Result> LuceneGlobalIndexWriter::Finish() { PAIMON_RETURN_NOT_OK(RapidJsonUtil::ToJsonString(options_, &options_json)); auto meta_bytes = std::make_shared(options_json, pool_.get()); GlobalIndexIOMeta meta(file_writer_->ToPath(index_file_name), file_size, - /*metadata=*/meta_bytes); + /*metadata=*/meta_bytes, /*extra_field_ids=*/std::nullopt); return std::vector({meta}); } diff --git a/src/paimon/global_index/lumina/lumina_global_index.cpp b/src/paimon/global_index/lumina/lumina_global_index.cpp index 29ead076b..dc5c96aa3 100644 --- a/src/paimon/global_index/lumina/lumina_global_index.cpp +++ b/src/paimon/global_index/lumina/lumina_global_index.cpp @@ -895,7 +895,7 @@ Result> LuminaIndexWriter::Finish() { PAIMON_RETURN_NOT_OK(RapidJsonUtil::ToJsonString(lumina_options_, &options_json)); auto meta_bytes = std::make_shared(options_json, pool_->GetPaimonPool().get()); GlobalIndexIOMeta meta(file_manager_->ToPath(index_file_name), file_size, - /*metadata=*/meta_bytes); + /*metadata=*/meta_bytes, /*extra_field_ids=*/std::nullopt); return std::vector({meta}); } diff --git a/src/paimon/global_index/tantivy/tantivy_equivalence_test.cpp b/src/paimon/global_index/tantivy/tantivy_equivalence_test.cpp index e51ee60b3..7c534f1ee 100644 --- a/src/paimon/global_index/tantivy/tantivy_equivalence_test.cpp +++ b/src/paimon/global_index/tantivy/tantivy_equivalence_test.cpp @@ -354,7 +354,7 @@ TEST_F(TantivyEquivalenceTest, BenchmarkBuildAndQuery) { // -------- Lucene: write + open + queries -------- auto lroot = paimon::test::UniqueTestDirectory::Create(); std::map lopt = {{"lucene-fts.write.tmp.directory", lroot->Str()}}; - GlobalIndexIOMeta lmeta{"", 0, nullptr}; + GlobalIndexIOMeta lmeta{"", 0, nullptr, std::nullopt}; auto lwrite_ms = time_ms([&] { lmeta = WriteOne("lucene-fts", data_type, lopt, array, lroot->Str()); }); auto lreader = OpenOne("lucene-fts", data_type, lopt, lmeta, lroot->Str()); @@ -371,7 +371,7 @@ TEST_F(TantivyEquivalenceTest, BenchmarkBuildAndQuery) { // -------- Tantivy: write + open + queries -------- auto troot = paimon::test::UniqueTestDirectory::Create(); - GlobalIndexIOMeta tmeta{"", 0, nullptr}; + GlobalIndexIOMeta tmeta{"", 0, nullptr, std::nullopt}; auto twrite_ms = time_ms([&] { tmeta = WriteOne("tantivy-fulltext", data_type, {}, array, troot->Str()); }); auto treader = OpenOne("tantivy-fulltext", data_type, {}, tmeta, troot->Str()); diff --git a/src/paimon/global_index/tantivy/tantivy_global_index_writer.cpp b/src/paimon/global_index/tantivy/tantivy_global_index_writer.cpp index 748043190..71a3902e7 100644 --- a/src/paimon/global_index/tantivy/tantivy_global_index_writer.cpp +++ b/src/paimon/global_index/tantivy/tantivy_global_index_writer.cpp @@ -162,7 +162,7 @@ Result> TantivyGlobalIndexWriter::Finish() { PAIMON_RETURN_NOT_OK(RapidJsonUtil::ToJsonString(options_, &options_json)); auto meta_bytes = std::make_shared(options_json, pool_.get()); GlobalIndexIOMeta meta(file_writer_->ToPath(index_file_name), file_size, - /*metadata=*/meta_bytes); + /*metadata=*/meta_bytes, /*extra_field_ids=*/std::nullopt); return std::vector({meta}); } diff --git a/src/paimon/global_index/tantivy/tantivy_java_compat_test.cpp b/src/paimon/global_index/tantivy/tantivy_java_compat_test.cpp index f54fc7a33..e4abaf944 100644 --- a/src/paimon/global_index/tantivy/tantivy_java_compat_test.cpp +++ b/src/paimon/global_index/tantivy/tantivy_java_compat_test.cpp @@ -107,7 +107,8 @@ class JavaCompatTest : public ::testing::Test { std::string metadata_json = "{}"; auto meta_bytes = std::make_shared(metadata_json, pool_.get()); - GlobalIndexIOMeta io_meta(archive_path, file_size, meta_bytes); + GlobalIndexIOMeta io_meta(archive_path, file_size, meta_bytes, + /*extra_field_ids=*/std::nullopt); std::map options; auto global_index = std::make_shared(options); @@ -430,7 +431,8 @@ TEST_F(JavaCompatTest, CppWriteDefaultTokenizerForJavaCrossRead) { ASSERT_OK_AND_ASSIGN(auto file_status, fs_->GetFileStatus(archive_path)); int64_t file_size = file_status->GetLen(); auto meta_bytes = std::make_shared(std::string("{}"), pool_.get()); - GlobalIndexIOMeta io_meta(archive_path, file_size, meta_bytes); + GlobalIndexIOMeta io_meta(archive_path, file_size, meta_bytes, + /*extra_field_ids=*/std::nullopt); auto reader_factory = std::make_shared(std::map{}); auto reader_path_factory = std::make_shared(out_dir); From bc3ae4fdb2e043503c011459464f1d1bd3343f47 Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Thu, 9 Jul 2026 13:07:37 +0800 Subject: [PATCH 3/8] fix little --- .../global_index/global_index_io_meta.h | 13 +- .../btree/btree_compatibility_test.cpp | 3 +- .../btree/btree_file_meta_selector_test.cpp | 12 +- .../btree_global_index_integration_test.cpp | 2 +- .../btree/btree_global_index_writer.cpp | 3 +- .../wrap/file_index_writer_wrapper.h | 2 +- .../global_index/global_index_scan_impl.cpp | 2 +- .../global_index/global_index_write_task.cpp | 26 ++- .../lucene/lucene_global_index_writer.cpp | 2 +- .../lumina/lumina_global_index.cpp | 160 ++++++++---------- .../lumina/lumina_global_index_test.cpp | 59 ++++--- .../tantivy/tantivy_equivalence_test.cpp | 4 +- .../tantivy/tantivy_global_index_writer.cpp | 2 +- .../tantivy/tantivy_java_compat_test.cpp | 6 +- test/inte/global_index_test.cpp | 18 +- 15 files changed, 153 insertions(+), 161 deletions(-) diff --git a/include/paimon/global_index/global_index_io_meta.h b/include/paimon/global_index/global_index_io_meta.h index 409983e94..522762668 100644 --- a/include/paimon/global_index/global_index_io_meta.h +++ b/include/paimon/global_index/global_index_io_meta.h @@ -18,23 +18,16 @@ #include #include -#include #include -#include #include "paimon/memory/bytes.h" -#include "paimon/utils/range.h" namespace paimon { /// Metadata describing a single file entry in a global index. struct PAIMON_EXPORT GlobalIndexIOMeta { GlobalIndexIOMeta(const std::string& _file_path, int64_t _file_size, - const std::shared_ptr& _metadata, - const std::optional>& _extra_field_ids) - : file_path(_file_path), - file_size(_file_size), - metadata(_metadata), - extra_field_ids(_extra_field_ids) {} + const std::shared_ptr& _metadata) + : file_path(_file_path), file_size(_file_size), metadata(_metadata) {} std::string file_path; int64_t file_size; @@ -42,8 +35,6 @@ struct PAIMON_EXPORT GlobalIndexIOMeta { /// secondary index structures or inline index bytes. /// May be null if no additional metadata is available. std::shared_ptr metadata; - /// Optional table field ids materialized together with the indexed field. - std::optional> extra_field_ids; }; } // namespace paimon diff --git a/src/paimon/common/global_index/btree/btree_compatibility_test.cpp b/src/paimon/common/global_index/btree/btree_compatibility_test.cpp index af349a4de..420d16ed1 100644 --- a/src/paimon/common/global_index/btree/btree_compatibility_test.cpp +++ b/src/paimon/common/global_index/btree/btree_compatibility_test.cpp @@ -115,8 +115,7 @@ class BTreeCompatibilityTest : public ::testing::Test { PAIMON_ASSIGN_OR_RAISE(auto file_status, fs_->GetFileStatus(bin_path)); auto file_size = file_status->GetLen(); - GlobalIndexIOMeta io_meta(bin_path, file_size, meta_bytes, - /*extra_field_ids=*/std::nullopt); + GlobalIndexIOMeta io_meta(bin_path, file_size, meta_bytes); std::vector metas = {io_meta}; auto schema = arrow::schema({arrow::field("testField", arrow_type)}); diff --git a/src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp b/src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp index ca126009d..31e45f642 100644 --- a/src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp +++ b/src/paimon/common/global_index/btree/btree_file_meta_selector_test.cpp @@ -48,12 +48,12 @@ class BTreeFileMetaSelectorTest : public ::testing::Test { // file6: only-nulls file (no keys, has_nulls=true) auto meta6 = std::make_shared(nullptr, nullptr, true); files_ = { - GlobalIndexIOMeta("file1", 1, meta1->Serialize(pool_.get()), std::nullopt), - GlobalIndexIOMeta("file2", 1, meta2->Serialize(pool_.get()), std::nullopt), - GlobalIndexIOMeta("file3", 1, meta3->Serialize(pool_.get()), std::nullopt), - GlobalIndexIOMeta("file4", 1, meta4->Serialize(pool_.get()), std::nullopt), - GlobalIndexIOMeta("file5", 1, meta5->Serialize(pool_.get()), std::nullopt), - GlobalIndexIOMeta("file6", 1, meta6->Serialize(pool_.get()), std::nullopt), + GlobalIndexIOMeta("file1", 1, meta1->Serialize(pool_.get())), + GlobalIndexIOMeta("file2", 1, meta2->Serialize(pool_.get())), + GlobalIndexIOMeta("file3", 1, meta3->Serialize(pool_.get())), + GlobalIndexIOMeta("file4", 1, meta4->Serialize(pool_.get())), + GlobalIndexIOMeta("file5", 1, meta5->Serialize(pool_.get())), + GlobalIndexIOMeta("file6", 1, meta6->Serialize(pool_.get())), }; } diff --git a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp index 1f577d420..3d0751a18 100644 --- a/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp +++ b/src/paimon/common/global_index/btree/btree_global_index_integration_test.cpp @@ -1703,7 +1703,7 @@ TEST_P(BTreeGlobalIndexIntegrationTest, CreateReaderWithMultiFieldSchema) { {BtreeDefs::kBtreeIndexCompression, GetParam()}}; ASSERT_OK_AND_ASSIGN(auto indexer, BTreeGlobalIndexer::Create(options)); - GlobalIndexIOMeta meta("fake_path", 100, nullptr, /*extra_field_ids=*/std::nullopt); + GlobalIndexIOMeta meta("fake_path", 100, nullptr); std::vector metas = {meta}; ASSERT_NOK_WITH_MSG(indexer->CreateReader(c_schema.get(), file_reader, metas, pool_), diff --git a/src/paimon/common/global_index/btree/btree_global_index_writer.cpp b/src/paimon/common/global_index/btree/btree_global_index_writer.cpp index 5abf6fe52..bfb99e8b6 100644 --- a/src/paimon/common/global_index/btree/btree_global_index_writer.cpp +++ b/src/paimon/common/global_index/btree/btree_global_index_writer.cpp @@ -199,8 +199,7 @@ Result> BTreeGlobalIndexWriter::Finish() { // Create GlobalIndexIOMeta std::string file_path = file_writer_->ToPath(index_file_name_); PAIMON_ASSIGN_OR_RAISE(int64_t file_size, file_writer_->GetFileSize(index_file_name_)); - GlobalIndexIOMeta io_meta(file_path, file_size, meta_bytes, - /*extra_field_ids=*/std::nullopt); + GlobalIndexIOMeta io_meta(file_path, file_size, meta_bytes); return std::vector{io_meta}; } diff --git a/src/paimon/common/global_index/wrap/file_index_writer_wrapper.h b/src/paimon/common/global_index/wrap/file_index_writer_wrapper.h index c9c891a29..02e40b49d 100644 --- a/src/paimon/common/global_index/wrap/file_index_writer_wrapper.h +++ b/src/paimon/common/global_index/wrap/file_index_writer_wrapper.h @@ -66,7 +66,7 @@ class FileIndexWriterWrapper : public GlobalIndexWriter { PAIMON_RETURN_NOT_OK(out->Flush()); PAIMON_RETURN_NOT_OK(out->Close()); GlobalIndexIOMeta meta(file_manager_->ToPath(file_name), /*file_size=*/bytes->size(), - /*metadata=*/nullptr, /*extra_field_ids=*/std::nullopt); + /*metadata=*/nullptr); return std::vector({meta}); } diff --git a/src/paimon/core/global_index/global_index_scan_impl.cpp b/src/paimon/core/global_index/global_index_scan_impl.cpp index 79058bd87..393d524de 100644 --- a/src/paimon/core/global_index/global_index_scan_impl.cpp +++ b/src/paimon/core/global_index/global_index_scan_impl.cpp @@ -223,7 +223,7 @@ GlobalIndexIOMeta GlobalIndexScanImpl::ToGlobalIndexIOMeta( assert(index_meta->GetGlobalIndexMeta()); const auto& global_index_meta = index_meta->GetGlobalIndexMeta().value(); return {index_file_manager_->ToPath(index_meta), index_meta->FileSize(), - global_index_meta.index_meta, global_index_meta.extra_field_ids}; + global_index_meta.index_meta}; } Result> GlobalIndexScanImpl::Scan( diff --git a/src/paimon/core/global_index/global_index_write_task.cpp b/src/paimon/core/global_index/global_index_write_task.cpp index f43ec3a2f..00c4f73e9 100644 --- a/src/paimon/core/global_index/global_index_write_task.cpp +++ b/src/paimon/core/global_index/global_index_write_task.cpp @@ -96,8 +96,13 @@ Result> GetExtraFields(const TableSchema& table_schema, extra_fields.reserve(extra_field_names.size()); std::set dedup_field_names; for (const auto& extra_field_name : extra_field_names) { - if (extra_field_name == field_name || !dedup_field_names.insert(extra_field_name).second) { - continue; + if (extra_field_name == field_name) { + return Status::Invalid(fmt::format( + "global index extra field {} must not be the indexed field", extra_field_name)); + } + if (!dedup_field_names.insert(extra_field_name).second) { + return Status::Invalid(fmt::format("global index extra field {} must not be duplicated", + extra_field_name)); } PAIMON_ASSIGN_OR_RAISE(DataField extra_field, table_schema.GetField(extra_field_name)); extra_fields.push_back(extra_field); @@ -174,11 +179,6 @@ Result> BuildIndex( return Status::Invalid( "array read from batch reader is not a struct array in GlobalIndexWriteTask"); } - auto indexed_array = struct_array->GetFieldByName(field_name); - if (!indexed_array) { - return Status::Invalid(fmt::format( - "read array does not contain {} field in GlobalIndexWriteTask", field_name)); - } auto row_id_array = struct_array->GetFieldByName(SpecialFields::RowId().Name()); auto typed_row_id_array = std::dynamic_pointer_cast(row_id_array); if (!typed_row_id_array) { @@ -222,7 +222,8 @@ Result> BuildIndex( Result> ToCommitMessage( const std::string& index_type, int32_t field_id, const Range& range, const std::vector& global_index_io_metas, const BinaryRow& partition, - int32_t bucket, const std::shared_ptr& file_manager) { + int32_t bucket, const std::shared_ptr& file_manager, + const std::optional>& extra_field_ids) { std::vector> index_file_metas; index_file_metas.reserve(global_index_io_metas.size()); bool is_external_path = file_manager->IsExternalPath(); @@ -235,8 +236,7 @@ Result> ToCommitMessage( index_file_metas.push_back(std::make_shared( index_type, PathUtil::GetName(io_meta.file_path), io_meta.file_size, range.Count(), /*dv_ranges=*/std::nullopt, external_path, - GlobalIndexMeta(range.from, range.to, field_id, io_meta.extra_field_ids, - io_meta.metadata))); + GlobalIndexMeta(range.from, range.to, field_id, extra_field_ids, io_meta.metadata))); } DataIncrement data_increment(std::move(index_file_metas)); return std::make_shared(partition, bucket, @@ -313,13 +313,11 @@ Result> GlobalIndexWriteTask::WriteIndex( PAIMON_ASSIGN_OR_RAISE(std::vector global_index_io_metas, BuildIndex(field_name, range, writer_field_names, batch_reader.get(), global_index_writer.get())); - for (auto& io_meta : global_index_io_metas) { - io_meta.extra_field_ids = extra_field_ids; - } // generate commit message return ToCommitMessage(index_type, field.Id(), range, global_index_io_metas, - data_split->Partition(), data_split->Bucket(), index_file_manager); + data_split->Partition(), data_split->Bucket(), index_file_manager, + extra_field_ids); } } // namespace paimon diff --git a/src/paimon/global_index/lucene/lucene_global_index_writer.cpp b/src/paimon/global_index/lucene/lucene_global_index_writer.cpp index 936c536bd..4290565c4 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_writer.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index_writer.cpp @@ -237,7 +237,7 @@ Result> LuceneGlobalIndexWriter::Finish() { PAIMON_RETURN_NOT_OK(RapidJsonUtil::ToJsonString(options_, &options_json)); auto meta_bytes = std::make_shared(options_json, pool_.get()); GlobalIndexIOMeta meta(file_writer_->ToPath(index_file_name), file_size, - /*metadata=*/meta_bytes, /*extra_field_ids=*/std::nullopt); + /*metadata=*/meta_bytes); return std::vector({meta}); } diff --git a/src/paimon/global_index/lumina/lumina_global_index.cpp b/src/paimon/global_index/lumina/lumina_global_index.cpp index dc5c96aa3..00bac8747 100644 --- a/src/paimon/global_index/lumina/lumina_global_index.cpp +++ b/src/paimon/global_index/lumina/lumina_global_index.cpp @@ -18,6 +18,7 @@ #include #include +#include #include #include @@ -136,16 +137,17 @@ Status ValidateTagArrowType(const LuminaTagField& tag_field, value_type = list_type->value_type(); } + // Lumina currently downcasts range tag values to 32-bit precision internally. + // Reject Arrow int64/double inputs to avoid silent precision loss. bool compatible = false; switch (tag_field.value_type) { case LuminaTagField::ValueType::INT64: - compatible = - value_type->id() == arrow::Type::INT8 || value_type->id() == arrow::Type::INT16 || - value_type->id() == arrow::Type::INT32 || value_type->id() == arrow::Type::INT64; + compatible = value_type->id() == arrow::Type::INT8 || + value_type->id() == arrow::Type::INT16 || + value_type->id() == arrow::Type::INT32; break; case LuminaTagField::ValueType::DOUBLE: - compatible = - value_type->id() == arrow::Type::FLOAT || value_type->id() == arrow::Type::DOUBLE; + compatible = value_type->id() == arrow::Type::FLOAT; break; case LuminaTagField::ValueType::STRING: compatible = value_type->id() == arrow::Type::STRING; @@ -159,71 +161,64 @@ Status ValidateTagArrowType(const LuminaTagField& tag_field, return Status::OK(); } -Status AppendInt64Value(const std::shared_ptr& array, int64_t index, - std::vector* values) { - if (array->IsNull(index)) { - return Status::OK(); - } - switch (array->type_id()) { - case arrow::Type::INT8: - values->push_back(std::dynamic_pointer_cast(array)->Value(index)); - break; - case arrow::Type::INT16: - values->push_back(std::dynamic_pointer_cast(array)->Value(index)); - break; - case arrow::Type::INT32: - values->push_back(std::dynamic_pointer_cast(array)->Value(index)); - break; - case arrow::Type::INT64: - values->push_back(std::dynamic_pointer_cast(array)->Value(index)); - break; - default: - return Status::Invalid(fmt::format( - "lumina int64 tag field has unsupported arrow type {}", array->type()->ToString())); - } - return Status::OK(); +template +void AppendPrimitiveTagValue(const std::shared_ptr& array, int64_t index, + std::vector* values) { + values->push_back( + static_cast(static_cast(array.get())->Value(index))); } -Status AppendDoubleValue(const std::shared_ptr& array, int64_t index, - std::vector* values) { +template +Status AppendTagValue(const std::shared_ptr& array, int64_t index, + std::vector* values) { if (array->IsNull(index)) { return Status::OK(); } - switch (array->type_id()) { - case arrow::Type::FLOAT: - values->push_back(std::dynamic_pointer_cast(array)->Value(index)); - break; - case arrow::Type::DOUBLE: - values->push_back(std::dynamic_pointer_cast(array)->Value(index)); - break; - default: + + if constexpr (std::is_same_v) { + switch (array->type_id()) { + case arrow::Type::INT8: + AppendPrimitiveTagValue(array, index, values); + break; + case arrow::Type::INT16: + AppendPrimitiveTagValue(array, index, values); + break; + case arrow::Type::INT32: + AppendPrimitiveTagValue(array, index, values); + break; + default: + return Status::Invalid( + fmt::format("lumina integer tag field has unsupported arrow type {}", + array->type()->ToString())); + } + } else if constexpr (std::is_same_v) { + switch (array->type_id()) { + case arrow::Type::FLOAT: + AppendPrimitiveTagValue(array, index, values); + break; + default: + return Status::Invalid( + fmt::format("lumina floating tag field has unsupported arrow type {}", + array->type()->ToString())); + } + } else if constexpr (std::is_same_v) { + if (array->type_id() != arrow::Type::STRING) { return Status::Invalid( - fmt::format("lumina double tag field has unsupported arrow type {}", + fmt::format("lumina string tag field has unsupported arrow type {}", array->type()->ToString())); + } + auto string_array = static_cast(array.get()); + auto view = string_array->GetView(index); + values->emplace_back(view.data(), view.size()); + } else { + return Status::Invalid("lumina tag field has unsupported value type"); } return Status::OK(); } -Status AppendStringValue(const std::shared_ptr& array, int64_t index, - std::vector* values) { - if (array->IsNull(index)) { - return Status::OK(); - } - auto string_array = std::dynamic_pointer_cast(array); - CHECK_NOT_NULL(string_array, - fmt::format("lumina string tag field has unsupported arrow type {}", - array->type()->ToString())); - auto view = string_array->GetView(index); - values->emplace_back(view.data(), view.size()); - return Status::OK(); -} - template Status ExtractTagValues(const std::shared_ptr& field_array, int64_t segment_start, - int64_t segment_len, - Status (*append_value)(const std::shared_ptr&, int64_t, - std::vector*), - std::vector>* values) { + int64_t segment_len, std::vector>* values) { values->resize(segment_len); auto list_array = std::dynamic_pointer_cast(field_array); if (list_array) { @@ -238,14 +233,14 @@ Status ExtractTagValues(const std::shared_ptr& field_array, int64_ auto& row_values = (*values)[i]; row_values.reserve(value_end - value_start); for (int64_t value_index = value_start; value_index < value_end; value_index++) { - PAIMON_RETURN_NOT_OK(append_value(child_values, value_index, &row_values)); + PAIMON_RETURN_NOT_OK(AppendTagValue(child_values, value_index, &row_values)); } } return Status::OK(); } for (int64_t i = 0; i < segment_len; i++) { - PAIMON_RETURN_NOT_OK(append_value(field_array, segment_start + i, &(*values)[i])); + PAIMON_RETURN_NOT_OK(AppendTagValue(field_array, segment_start + i, &(*values)[i])); } return Status::OK(); } @@ -261,12 +256,8 @@ Result LiteralToTagValue(const Literal& literal) { return TagValue(static_cast(literal.GetValue())); case FieldType::INT: return TagValue(static_cast(literal.GetValue())); - case FieldType::BIGINT: - return TagValue(literal.GetValue()); case FieldType::FLOAT: return TagValue(static_cast(literal.GetValue())); - case FieldType::DOUBLE: - return TagValue(literal.GetValue()); case FieldType::STRING: return TagValue(literal.GetValue()); default: @@ -293,8 +284,7 @@ Result LiteralsToTagValues(const std::vector& literals) { switch (literals[0].GetType()) { case FieldType::TINYINT: case FieldType::SMALLINT: - case FieldType::INT: - case FieldType::BIGINT: { + case FieldType::INT: { std::vector values; values.reserve(literals.size()); for (const auto& literal : literals) { @@ -306,8 +296,7 @@ Result LiteralsToTagValues(const std::vector& literals) { } return TagValues(std::move(values)); } - case FieldType::FLOAT: - case FieldType::DOUBLE: { + case FieldType::FLOAT: { std::vector values; values.reserve(literals.size()); for (const auto& literal : literals) { @@ -355,22 +344,22 @@ Result> LuminaIndexWriter::ExtractTagDataForSegmen switch (tag_field.value_type) { case LuminaTagField::ValueType::INT64: { std::vector> values; - PAIMON_RETURN_NOT_OK(ExtractTagValues( - field_array, segment_start, segment_len, AppendInt64Value, &values)); + PAIMON_RETURN_NOT_OK( + ExtractTagValues(field_array, segment_start, segment_len, &values)); tag_dimension_data.values = std::move(values); break; } case LuminaTagField::ValueType::DOUBLE: { std::vector> values; - PAIMON_RETURN_NOT_OK(ExtractTagValues( - field_array, segment_start, segment_len, AppendDoubleValue, &values)); + PAIMON_RETURN_NOT_OK( + ExtractTagValues(field_array, segment_start, segment_len, &values)); tag_dimension_data.values = std::move(values); break; } case LuminaTagField::ValueType::STRING: { std::vector> values; - PAIMON_RETURN_NOT_OK(ExtractTagValues( - field_array, segment_start, segment_len, AppendStringValue, &values)); + PAIMON_RETURN_NOT_OK(ExtractTagValues(field_array, segment_start, + segment_len, &values)); tag_dimension_data.values = std::move(values); break; } @@ -696,18 +685,18 @@ class LuminaDataset : public ::lumina::api::Dataset { } auto& value_array = array_vec_[cursor_]; int64_t value_array_length = value_array->length(); - int64_t element_count = value_array_length / dimension_; + int64_t batch_element_count = value_array_length / dimension_; const float* value_ptr = value_array->raw_values(); vector_buffer.resize(value_array_length); memcpy(vector_buffer.data(), value_ptr, sizeof(float) * value_array_length); - id_buffer.resize(element_count); + id_buffer.resize(batch_element_count); std::iota(id_buffer.begin(), id_buffer.end(), static_cast<::lumina::core::vector_id_t>(start_ids_[cursor_])); // release the array when copy to vector_buffer value_array.reset(); cursor_++; - return ::lumina::core::Result::Ok(static_cast(element_count)); + return ::lumina::core::Result::Ok(static_cast(batch_element_count)); } private: @@ -745,18 +734,18 @@ class LuminaDatasetWithTag : public ::lumina::extensions::experimental::DatasetW } auto& value_array = array_vec_[cursor_]; int64_t value_array_length = value_array->length(); - int64_t element_count = value_array_length / dimension_; + int64_t batch_element_count = value_array_length / dimension_; const float* value_ptr = value_array->raw_values(); vector_buffer.resize(value_array_length); memcpy(vector_buffer.data(), value_ptr, sizeof(float) * value_array_length); - id_buffer.resize(element_count); + id_buffer.resize(batch_element_count); std::iota(id_buffer.begin(), id_buffer.end(), static_cast<::lumina::core::vector_id_t>(start_ids_[cursor_])); tag_dimensions_data = std::move(tag_data_vec_[cursor_]); value_array.reset(); cursor_++; - return ::lumina::core::Result::Ok(static_cast(element_count)); + return ::lumina::core::Result::Ok(static_cast(batch_element_count)); } private: @@ -835,17 +824,14 @@ Status LuminaIndexWriter::AddBatch(::ArrowArray* arrow_array, "multiplied dimension [{}] must match length of field value array [{}]", segment_len, dimension_, sliced_values->length())); } - std::vector tag_data; if (!tag_fields_.empty()) { - PAIMON_ASSIGN_OR_RAISE( - tag_data, ExtractTagDataForSegment(struct_array, tag_fields_, segment_start, - segment_len)); + PAIMON_ASSIGN_OR_RAISE(std::vector tag_data, + ExtractTagDataForSegment(struct_array, tag_fields_, + segment_start, segment_len)); + tag_data_vec_.push_back(std::move(tag_data)); } array_vec_.push_back(std::move(sliced_values)); array_start_ids_.push_back(count_ + segment_start); - if (!tag_fields_.empty()) { - tag_data_vec_.push_back(std::move(tag_data)); - } indexed_count_ += segment_len; segment_start = -1; } @@ -895,7 +881,7 @@ Result> LuminaIndexWriter::Finish() { PAIMON_RETURN_NOT_OK(RapidJsonUtil::ToJsonString(lumina_options_, &options_json)); auto meta_bytes = std::make_shared(options_json, pool_->GetPaimonPool().get()); GlobalIndexIOMeta meta(file_manager_->ToPath(index_file_name), file_size, - /*metadata=*/meta_bytes, /*extra_field_ids=*/std::nullopt); + /*metadata=*/meta_bytes); return std::vector({meta}); } diff --git a/src/paimon/global_index/lumina/lumina_global_index_test.cpp b/src/paimon/global_index/lumina/lumina_global_index_test.cpp index d762c2a9b..ddd853770 100644 --- a/src/paimon/global_index/lumina/lumina_global_index_test.cpp +++ b/src/paimon/global_index/lumina/lumina_global_index_test.cpp @@ -310,7 +310,7 @@ TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithMixedTagPredicates) { std::shared_ptr tag_data_type = arrow::struct_( {arrow::field("f0", arrow::list(arrow::float32())), arrow::field("color", arrow::utf8()), - arrow::field("price", arrow::float64())}); + arrow::field("price", arrow::float32())}); std::shared_ptr tag_array = arrow::ipc::internal::json::ArrayFromJSON(tag_data_type, R"([ @@ -331,9 +331,9 @@ TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithMixedTagPredicates) { /*field_index=*/1, /*field_name=*/"color", FieldType::STRING, Literal(FieldType::STRING, "warm", 4)); std::shared_ptr low_price_predicate = PredicateBuilder::LessOrEqual( - /*field_index=*/2, /*field_name=*/"price", FieldType::DOUBLE, Literal(10.0)); + /*field_index=*/2, /*field_name=*/"price", FieldType::FLOAT, Literal(10.0f)); std::shared_ptr high_price_predicate = PredicateBuilder::GreaterOrEqual( - /*field_index=*/2, /*field_name=*/"price", FieldType::DOUBLE, Literal(30.0)); + /*field_index=*/2, /*field_name=*/"price", FieldType::FLOAT, Literal(30.0f)); ASSERT_OK_AND_ASSIGN(std::shared_ptr price_predicate, PredicateBuilder::Or({low_price_predicate, high_price_predicate})); ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, @@ -358,7 +358,7 @@ TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithIntegerListTagFilter) { std::shared_ptr tag_data_type = arrow::struct_({arrow::field("f0", arrow::list(arrow::float32())), - arrow::field("category_ids", arrow::list(arrow::int64()))}); + arrow::field("category_ids", arrow::list(arrow::int32()))}); std::shared_ptr tag_array = arrow::ipc::internal::json::ArrayFromJSON(tag_data_type, R"([ @@ -376,8 +376,7 @@ TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithIntegerListTagFilter) { CreateGlobalIndexReader(test_root, data_type_, tag_options, meta)); std::shared_ptr predicate = PredicateBuilder::In( - /*field_index=*/1, /*field_name=*/"category_ids", FieldType::BIGINT, - {Literal(8l), Literal(9l)}); + /*field_index=*/1, /*field_name=*/"category_ids", FieldType::INT, {Literal(8), Literal(9)}); ASSERT_OK_AND_ASSIGN( std::shared_ptr scored_result, reader->VisitVectorSearch(std::make_shared( @@ -461,10 +460,10 @@ TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithTagNullAndEmptyValues) { std::shared_ptr tag_data_type = arrow::struct_( {arrow::field("f0", arrow::list(arrow::float32())), arrow::field("color", arrow::utf8()), arrow::field("labels", arrow::list(arrow::utf8())), - arrow::field("price", arrow::float64()), - arrow::field("scores", arrow::list(arrow::float64())), - arrow::field("category", arrow::int64()), - arrow::field("category_ids", arrow::list(arrow::int64()))}); + arrow::field("price", arrow::float32()), + arrow::field("scores", arrow::list(arrow::float32())), + arrow::field("category", arrow::int32()), + arrow::field("category_ids", arrow::list(arrow::int32()))}); std::shared_ptr tag_array = arrow::ipc::internal::json::ArrayFromJSON(tag_data_type, R"([ @@ -519,22 +518,22 @@ TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithTagNullAndEmptyValues) { {Literal(FieldType::STRING, "vip", 3)}), {4l}, {0.01f}); search_and_check(PredicateBuilder::LessOrEqual(/*field_index=*/3, /*field_name=*/"price", - FieldType::DOUBLE, Literal(10.0)), + FieldType::FLOAT, Literal(10.0f)), {0l}, {4.21f}); search_and_check(PredicateBuilder::LessThan(/*field_index=*/3, /*field_name=*/"price", - FieldType::DOUBLE, Literal(10.0)), + FieldType::FLOAT, Literal(10.0f)), {0l}, {4.21f}); search_and_check(PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"price", - FieldType::DOUBLE, Literal(10.0)), + FieldType::FLOAT, Literal(10.0f)), {4l, 3l}, {0.01f, 2.21f}); search_and_check(PredicateBuilder::In(/*field_index=*/4, /*field_name=*/"scores", - FieldType::DOUBLE, {Literal(0.25), Literal(0.75)}), + FieldType::FLOAT, {Literal(0.25f), Literal(0.75f)}), {4l, 0l}, {0.01f, 4.21f}); search_and_check(PredicateBuilder::Equal(/*field_index=*/5, /*field_name=*/"category", - FieldType::BIGINT, Literal(7l)), + FieldType::INT, Literal(7)), {0l}, {4.21f}); search_and_check(PredicateBuilder::In(/*field_index=*/6, /*field_name=*/"category_ids", - FieldType::BIGINT, {Literal(9l)}), + FieldType::INT, {Literal(9)}), {4l}, {0.01f}); search_and_check( PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"color", FieldType::STRING, @@ -553,7 +552,7 @@ TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithTagNullAndEmptyValues) { Literal(FieldType::STRING, "red", 3)), "unknown tag key 'unknown' in label filter"); search_and_check_error(PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"color", - FieldType::BIGINT, Literal(1l)), + FieldType::INT, Literal(1)), "tag value type mismatch for key 'color'"); search_and_check_error( PredicateBuilder::Equal(/*field_index=*/3, /*field_name=*/"price", FieldType::STRING, @@ -575,7 +574,7 @@ TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithTagNullAndEmptyValues) { /*field_index=*/1, /*field_name=*/"color", FieldType::STRING, Literal(FieldType::STRING, "blue", 4)); std::shared_ptr high_price_predicate = PredicateBuilder::GreaterOrEqual( - /*field_index=*/3, /*field_name=*/"price", FieldType::DOUBLE, Literal(30.0)); + /*field_index=*/3, /*field_name=*/"price", FieldType::FLOAT, Literal(30.0f)); ASSERT_OK_AND_ASSIGN(std::shared_ptr blue_high_price_predicate, PredicateBuilder::And({blue_predicate, high_price_predicate})); ASSERT_OK_AND_ASSIGN(std::shared_ptr compound_predicate, @@ -610,6 +609,28 @@ TEST_F(LuminaGlobalIndexTest, TestTagSchemaValidation) { WriteGlobalIndex(index_root, tag_data_type, tag_options, array_, Range(0, 3)), "lumina tag field color type string is not compatible with tag_schema value_type"); } + { + std::shared_ptr int64_tag_data_type = + arrow::struct_({arrow::field("f0", arrow::list(arrow::float32())), + arrow::field("category", arrow::int64())}); + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"category","type":"enum","value_type":"int64"}])"; + ASSERT_NOK_WITH_MSG( + WriteGlobalIndex(index_root, int64_tag_data_type, tag_options, array_, Range(0, 3)), + "lumina tag field category type int64 is not compatible with tag_schema value_type"); + } + { + std::shared_ptr double_tag_data_type = + arrow::struct_({arrow::field("f0", arrow::list(arrow::float32())), + arrow::field("price", arrow::float64())}); + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"price","type":"range","value_type":"double"}])"; + ASSERT_NOK_WITH_MSG( + WriteGlobalIndex(index_root, double_tag_data_type, tag_options, array_, Range(0, 3)), + "lumina tag field price type double is not compatible with tag_schema value_type"); + } } TEST_F(LuminaGlobalIndexTest, TestGetExtraFieldNames) { @@ -756,7 +777,7 @@ TEST_F(LuminaGlobalIndexTest, TestInvalidInputs) { "f1", /*limit=*/2, query_, /*filter=*/nullptr, PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"f0", - FieldType::BIGINT, Literal(5l)), + FieldType::INT, Literal(5)), /*distance_type=*/std::nullopt, /*options=*/std::map())), "lumina index was not built with tag"); diff --git a/src/paimon/global_index/tantivy/tantivy_equivalence_test.cpp b/src/paimon/global_index/tantivy/tantivy_equivalence_test.cpp index 7c534f1ee..e51ee60b3 100644 --- a/src/paimon/global_index/tantivy/tantivy_equivalence_test.cpp +++ b/src/paimon/global_index/tantivy/tantivy_equivalence_test.cpp @@ -354,7 +354,7 @@ TEST_F(TantivyEquivalenceTest, BenchmarkBuildAndQuery) { // -------- Lucene: write + open + queries -------- auto lroot = paimon::test::UniqueTestDirectory::Create(); std::map lopt = {{"lucene-fts.write.tmp.directory", lroot->Str()}}; - GlobalIndexIOMeta lmeta{"", 0, nullptr, std::nullopt}; + GlobalIndexIOMeta lmeta{"", 0, nullptr}; auto lwrite_ms = time_ms([&] { lmeta = WriteOne("lucene-fts", data_type, lopt, array, lroot->Str()); }); auto lreader = OpenOne("lucene-fts", data_type, lopt, lmeta, lroot->Str()); @@ -371,7 +371,7 @@ TEST_F(TantivyEquivalenceTest, BenchmarkBuildAndQuery) { // -------- Tantivy: write + open + queries -------- auto troot = paimon::test::UniqueTestDirectory::Create(); - GlobalIndexIOMeta tmeta{"", 0, nullptr, std::nullopt}; + GlobalIndexIOMeta tmeta{"", 0, nullptr}; auto twrite_ms = time_ms([&] { tmeta = WriteOne("tantivy-fulltext", data_type, {}, array, troot->Str()); }); auto treader = OpenOne("tantivy-fulltext", data_type, {}, tmeta, troot->Str()); diff --git a/src/paimon/global_index/tantivy/tantivy_global_index_writer.cpp b/src/paimon/global_index/tantivy/tantivy_global_index_writer.cpp index 71a3902e7..748043190 100644 --- a/src/paimon/global_index/tantivy/tantivy_global_index_writer.cpp +++ b/src/paimon/global_index/tantivy/tantivy_global_index_writer.cpp @@ -162,7 +162,7 @@ Result> TantivyGlobalIndexWriter::Finish() { PAIMON_RETURN_NOT_OK(RapidJsonUtil::ToJsonString(options_, &options_json)); auto meta_bytes = std::make_shared(options_json, pool_.get()); GlobalIndexIOMeta meta(file_writer_->ToPath(index_file_name), file_size, - /*metadata=*/meta_bytes, /*extra_field_ids=*/std::nullopt); + /*metadata=*/meta_bytes); return std::vector({meta}); } diff --git a/src/paimon/global_index/tantivy/tantivy_java_compat_test.cpp b/src/paimon/global_index/tantivy/tantivy_java_compat_test.cpp index e4abaf944..f54fc7a33 100644 --- a/src/paimon/global_index/tantivy/tantivy_java_compat_test.cpp +++ b/src/paimon/global_index/tantivy/tantivy_java_compat_test.cpp @@ -107,8 +107,7 @@ class JavaCompatTest : public ::testing::Test { std::string metadata_json = "{}"; auto meta_bytes = std::make_shared(metadata_json, pool_.get()); - GlobalIndexIOMeta io_meta(archive_path, file_size, meta_bytes, - /*extra_field_ids=*/std::nullopt); + GlobalIndexIOMeta io_meta(archive_path, file_size, meta_bytes); std::map options; auto global_index = std::make_shared(options); @@ -431,8 +430,7 @@ TEST_F(JavaCompatTest, CppWriteDefaultTokenizerForJavaCrossRead) { ASSERT_OK_AND_ASSIGN(auto file_status, fs_->GetFileStatus(archive_path)); int64_t file_size = file_status->GetLen(); auto meta_bytes = std::make_shared(std::string("{}"), pool_.get()); - GlobalIndexIOMeta io_meta(archive_path, file_size, meta_bytes, - /*extra_field_ids=*/std::nullopt); + GlobalIndexIOMeta io_meta(archive_path, file_size, meta_bytes); auto reader_factory = std::make_shared(std::map{}); auto reader_path_factory = std::make_shared(out_dir); diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp index ab3a03d10..978d2255f 100644 --- a/test/inte/global_index_test.cpp +++ b/test/inte/global_index_test.cpp @@ -1139,10 +1139,10 @@ TEST_P(GlobalIndexTest, TestWriteAndQueryLuminaIndexWithTagNullAndEmptyValues) { arrow::field("embedding", arrow::list(arrow::float32())), arrow::field("color", arrow::utf8()), arrow::field("labels", arrow::list(arrow::utf8())), - arrow::field("price", arrow::float64()), - arrow::field("scores", arrow::list(arrow::float64())), - arrow::field("category", arrow::int64()), - arrow::field("category_ids", arrow::list(arrow::int64()))}; + arrow::field("price", arrow::float32()), + arrow::field("scores", arrow::list(arrow::float32())), + arrow::field("category", arrow::int32()), + arrow::field("category_ids", arrow::list(arrow::int32()))}; auto schema = arrow::schema(fields); std::map lumina_options = { {"lumina.index.dimension", "4"}, @@ -1243,16 +1243,16 @@ TEST_P(GlobalIndexTest, TestWriteAndQueryLuminaIndexWithTagNullAndEmptyValues) { {Literal(FieldType::STRING, "vip", 3)}), "row ids: {4}, scores: {0.01}"); search_and_check(PredicateBuilder::LessOrEqual(/*field_index=*/4, /*field_name=*/"price", - FieldType::DOUBLE, Literal(10.0)), + FieldType::FLOAT, Literal(10.0f)), "row ids: {0}, scores: {4.21}"); search_and_check(PredicateBuilder::GreaterOrEqual(/*field_index=*/5, /*field_name=*/"scores", - FieldType::DOUBLE, Literal(0.5)), + FieldType::FLOAT, Literal(0.5f)), "row ids: {4}, scores: {0.01}"); search_and_check(PredicateBuilder::Equal(/*field_index=*/6, /*field_name=*/"category", - FieldType::BIGINT, Literal(7l)), + FieldType::INT, Literal(7)), "row ids: {0}, scores: {4.21}"); search_and_check(PredicateBuilder::In(/*field_index=*/7, /*field_name=*/"category_ids", - FieldType::BIGINT, {Literal(9l)}), + FieldType::INT, {Literal(9)}), "row ids: {4}, scores: {0.01}"); search_and_check( PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"color", FieldType::STRING, @@ -1275,7 +1275,7 @@ TEST_P(GlobalIndexTest, TestWriteAndQueryLuminaIndexWithTagNullAndEmptyValues) { /*field_index=*/2, /*field_name=*/"color", FieldType::STRING, Literal(FieldType::STRING, "blue", 4)); std::shared_ptr high_price_predicate = PredicateBuilder::GreaterOrEqual( - /*field_index=*/4, /*field_name=*/"price", FieldType::DOUBLE, Literal(30.0)); + /*field_index=*/4, /*field_name=*/"price", FieldType::FLOAT, Literal(30.0f)); ASSERT_OK_AND_ASSIGN(std::shared_ptr blue_high_price_predicate, PredicateBuilder::And({blue_predicate, high_price_predicate})); ASSERT_OK_AND_ASSIGN(std::shared_ptr compound_predicate, From 6650c2fe808009ffe75ea02886df9f5e82c1fa79 Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Thu, 9 Jul 2026 13:14:08 +0800 Subject: [PATCH 4/8] add comments --- include/paimon/predicate/vector_search.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/paimon/predicate/vector_search.h b/include/paimon/predicate/vector_search.h index 1d9a7beea..9c6e564be 100644 --- a/include/paimon/predicate/vector_search.h +++ b/include/paimon/predicate/vector_search.h @@ -74,6 +74,10 @@ struct PAIMON_EXPORT VectorSearch { /// context-aware filtering at query time. /// @note All fields referenced in the predicate must have been materialized /// in the index during build to ensure availability. + /// @note For tag-based vector indexes, fields referenced by the predicate + /// must keep the same names and types as the fields used during index + /// construction. Indexes built with tag fields must not be reused across + /// schema evolution that renames or changes those tag fields. std::shared_ptr predicate; /// The distance metric to use for this query, if explicitly specified. /// If set, this value must match the distance type used by the index (e.g., EUCLIDEAN, COSINE). From 4c78d4dc8e250c430eb380215ddbe510a3ff19e1 Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Sat, 18 Jul 2026 10:09:20 +0800 Subject: [PATCH 5/8] update lumina to 0.3.1 --- .../lumina/lumina_global_index.cpp | 100 ++++++++++++++---- .../global_index/lumina/lumina_global_index.h | 2 + .../lumina/lumina_global_index_test.cpp | 76 +++++++++---- test/inte/global_index_test.cpp | 10 +- third_party/versions.txt | 4 +- 5 files changed, 140 insertions(+), 52 deletions(-) diff --git a/src/paimon/global_index/lumina/lumina_global_index.cpp b/src/paimon/global_index/lumina/lumina_global_index.cpp index 00bac8747..e6f1c0fe2 100644 --- a/src/paimon/global_index/lumina/lumina_global_index.cpp +++ b/src/paimon/global_index/lumina/lumina_global_index.cpp @@ -112,8 +112,12 @@ Result ParseTagField(const rapidjson::Value& obj, const std::str } LuminaTagField::ValueType parsed_value_type; - if (value_type == std::string(::lumina::core::kExtensionTagVTypeInt64)) { + if (value_type == std::string(::lumina::core::kExtensionTagVTypeInt32)) { + parsed_value_type = LuminaTagField::ValueType::INT32; + } else if (value_type == std::string(::lumina::core::kExtensionTagVTypeInt64)) { parsed_value_type = LuminaTagField::ValueType::INT64; + } else if (value_type == std::string(::lumina::core::kExtensionTagVTypeFloat)) { + parsed_value_type = LuminaTagField::ValueType::FLOAT; } else if (value_type == std::string(::lumina::core::kExtensionTagVTypeDouble)) { parsed_value_type = LuminaTagField::ValueType::DOUBLE; } else if (value_type == std::string(::lumina::core::kExtensionTagVTypeString)) { @@ -122,11 +126,6 @@ Result ParseTagField(const rapidjson::Value& obj, const std::str return Status::Invalid(fmt::format("lumina tag_schema {} has unsupported value_type: {}", tag_label, value_type)); } - if (parsed_type == LuminaTagField::Type::RANGE && - parsed_value_type == LuminaTagField::ValueType::STRING) { - return Status::Invalid(fmt::format( - "lumina tag_schema {} range type does not support string value_type", tag_label)); - } return LuminaTagField{key_name, parsed_type, parsed_value_type}; } @@ -137,18 +136,22 @@ Status ValidateTagArrowType(const LuminaTagField& tag_field, value_type = list_type->value_type(); } - // Lumina currently downcasts range tag values to 32-bit precision internally. - // Reject Arrow int64/double inputs to avoid silent precision loss. bool compatible = false; switch (tag_field.value_type) { - case LuminaTagField::ValueType::INT64: + case LuminaTagField::ValueType::INT32: compatible = value_type->id() == arrow::Type::INT8 || value_type->id() == arrow::Type::INT16 || value_type->id() == arrow::Type::INT32; break; - case LuminaTagField::ValueType::DOUBLE: + case LuminaTagField::ValueType::INT64: + compatible = value_type->id() == arrow::Type::INT64; + break; + case LuminaTagField::ValueType::FLOAT: compatible = value_type->id() == arrow::Type::FLOAT; break; + case LuminaTagField::ValueType::DOUBLE: + compatible = value_type->id() == arrow::Type::DOUBLE; + break; case LuminaTagField::ValueType::STRING: compatible = value_type->id() == arrow::Type::STRING; break; @@ -175,7 +178,7 @@ Status AppendTagValue(const std::shared_ptr& array, int64_t index, return Status::OK(); } - if constexpr (std::is_same_v) { + if constexpr (std::is_same_v) { switch (array->type_id()) { case arrow::Type::INT8: AppendPrimitiveTagValue(array, index, values); @@ -191,16 +194,25 @@ Status AppendTagValue(const std::shared_ptr& array, int64_t index, fmt::format("lumina integer tag field has unsupported arrow type {}", array->type()->ToString())); } + } else if constexpr (std::is_same_v) { + if (array->type_id() != arrow::Type::INT64) { + return Status::Invalid(fmt::format( + "lumina int64 tag field has unsupported arrow type {}", array->type()->ToString())); + } + AppendPrimitiveTagValue(array, index, values); + } else if constexpr (std::is_same_v) { + if (array->type_id() != arrow::Type::FLOAT) { + return Status::Invalid(fmt::format( + "lumina float tag field has unsupported arrow type {}", array->type()->ToString())); + } + AppendPrimitiveTagValue(array, index, values); } else if constexpr (std::is_same_v) { - switch (array->type_id()) { - case arrow::Type::FLOAT: - AppendPrimitiveTagValue(array, index, values); - break; - default: - return Status::Invalid( - fmt::format("lumina floating tag field has unsupported arrow type {}", - array->type()->ToString())); + if (array->type_id() != arrow::Type::DOUBLE) { + return Status::Invalid( + fmt::format("lumina double tag field has unsupported arrow type {}", + array->type()->ToString())); } + AppendPrimitiveTagValue(array, index, values); } else if constexpr (std::is_same_v) { if (array->type_id() != arrow::Type::STRING) { return Status::Invalid( @@ -251,13 +263,17 @@ Result LiteralToTagValue(const Literal& literal) { } switch (literal.GetType()) { case FieldType::TINYINT: - return TagValue(static_cast(literal.GetValue())); + return TagValue(static_cast(literal.GetValue())); case FieldType::SMALLINT: - return TagValue(static_cast(literal.GetValue())); + return TagValue(static_cast(literal.GetValue())); case FieldType::INT: - return TagValue(static_cast(literal.GetValue())); + return TagValue(literal.GetValue()); + case FieldType::BIGINT: + return TagValue(literal.GetValue()); case FieldType::FLOAT: - return TagValue(static_cast(literal.GetValue())); + return TagValue(literal.GetValue()); + case FieldType::DOUBLE: + return TagValue(literal.GetValue()); case FieldType::STRING: return TagValue(literal.GetValue()); default: @@ -285,6 +301,18 @@ Result LiteralsToTagValues(const std::vector& literals) { case FieldType::TINYINT: case FieldType::SMALLINT: case FieldType::INT: { + std::vector values; + values.reserve(literals.size()); + for (const auto& literal : literals) { + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(literal)); + auto typed_value = std::get_if(&value); + CHECK_NOT_NULL(typed_value, + "lumina tag predicate IN literals must have the same value type"); + values.push_back(*typed_value); + } + return TagValues(std::move(values)); + } + case FieldType::BIGINT: { std::vector values; values.reserve(literals.size()); for (const auto& literal : literals) { @@ -297,6 +325,18 @@ Result LiteralsToTagValues(const std::vector& literals) { return TagValues(std::move(values)); } case FieldType::FLOAT: { + std::vector values; + values.reserve(literals.size()); + for (const auto& literal : literals) { + PAIMON_ASSIGN_OR_RAISE(TagValue value, LiteralToTagValue(literal)); + auto typed_value = std::get_if(&value); + CHECK_NOT_NULL(typed_value, + "lumina tag predicate IN literals must have the same value type"); + values.push_back(*typed_value); + } + return TagValues(std::move(values)); + } + case FieldType::DOUBLE: { std::vector values; values.reserve(literals.size()); for (const auto& literal : literals) { @@ -342,6 +382,13 @@ Result> LuminaIndexWriter::ExtractTagDataForSegmen TagDimensionData tag_dimension_data; tag_dimension_data.tagkName = tag_field.name; switch (tag_field.value_type) { + case LuminaTagField::ValueType::INT32: { + std::vector> values; + PAIMON_RETURN_NOT_OK( + ExtractTagValues(field_array, segment_start, segment_len, &values)); + tag_dimension_data.values = std::move(values); + break; + } case LuminaTagField::ValueType::INT64: { std::vector> values; PAIMON_RETURN_NOT_OK( @@ -349,6 +396,13 @@ Result> LuminaIndexWriter::ExtractTagDataForSegmen tag_dimension_data.values = std::move(values); break; } + case LuminaTagField::ValueType::FLOAT: { + std::vector> values; + PAIMON_RETURN_NOT_OK( + ExtractTagValues(field_array, segment_start, segment_len, &values)); + tag_dimension_data.values = std::move(values); + break; + } case LuminaTagField::ValueType::DOUBLE: { std::vector> values; PAIMON_RETURN_NOT_OK( diff --git a/src/paimon/global_index/lumina/lumina_global_index.h b/src/paimon/global_index/lumina/lumina_global_index.h index 9ce03edfc..196cdddc4 100644 --- a/src/paimon/global_index/lumina/lumina_global_index.h +++ b/src/paimon/global_index/lumina/lumina_global_index.h @@ -44,7 +44,9 @@ struct LuminaTagField { }; enum class ValueType { + INT32, INT64, + FLOAT, DOUBLE, STRING, }; diff --git a/src/paimon/global_index/lumina/lumina_global_index_test.cpp b/src/paimon/global_index/lumina/lumina_global_index_test.cpp index ddd853770..e2ee54ba0 100644 --- a/src/paimon/global_index/lumina/lumina_global_index_test.cpp +++ b/src/paimon/global_index/lumina/lumina_global_index_test.cpp @@ -306,7 +306,7 @@ TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithMixedTagPredicates) { std::map tag_options = options_; tag_options["lumina.extension.build.tag.tag_schema"] = R"([{"key_name":"color","type":"enum","value_type":"string"},)" - R"({"key_name":"price","type":"range","value_type":"double"}])"; + R"({"key_name":"price","type":"range","value_type":"float"}])"; std::shared_ptr tag_data_type = arrow::struct_( {arrow::field("f0", arrow::list(arrow::float32())), arrow::field("color", arrow::utf8()), @@ -354,7 +354,7 @@ TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithIntegerListTagFilter) { std::map tag_options = options_; tag_options["lumina.extension.build.tag.tag_schema"] = - R"([{"key_name":"category_ids","type":"enum","value_type":"int64"}])"; + R"([{"key_name":"category_ids","type":"enum","value_type":"int32"}])"; std::shared_ptr tag_data_type = arrow::struct_({arrow::field("f0", arrow::list(arrow::float32())), @@ -392,22 +392,25 @@ TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithCompatibleTagArrowTypes) { std::map tag_options = options_; tag_options["lumina.extension.build.tag.tag_schema"] = - R"([{"key_name":"tag_i8","type":"enum","value_type":"int64"},)" - R"({"key_name":"tag_i16","type":"enum","value_type":"int64"},)" - R"({"key_name":"tag_i32","type":"enum","value_type":"int64"},)" - R"({"key_name":"tag_f32","type":"range","value_type":"double"}])"; + R"([{"key_name":"tag_i8","type":"enum","value_type":"int32"},)" + R"({"key_name":"tag_i16","type":"enum","value_type":"int32"},)" + R"({"key_name":"tag_i32","type":"enum","value_type":"int32"},)" + R"({"key_name":"tag_i64","type":"enum","value_type":"int64"},)" + R"({"key_name":"tag_f32","type":"range","value_type":"float"},)" + R"({"key_name":"tag_f64","type":"enum","value_type":"double"}])"; std::shared_ptr tag_data_type = arrow::struct_( {arrow::field("f0", arrow::list(arrow::float32())), arrow::field("tag_i8", arrow::int8()), arrow::field("tag_i16", arrow::int16()), arrow::field("tag_i32", arrow::int32()), - arrow::field("tag_f32", arrow::float32())}); + arrow::field("tag_i64", arrow::int64()), arrow::field("tag_f32", arrow::float32()), + arrow::field("tag_f64", arrow::float64())}); std::shared_ptr tag_array = arrow::ipc::internal::json::ArrayFromJSON(tag_data_type, R"([ - [[0.0, 0.0, 0.0, 0.0], 1, 10, 100, 1.5], - [[0.0, 1.0, 0.0, 1.0], 2, 20, 200, 2.5], - [[1.0, 0.0, 1.0, 0.0], 3, 30, 300, 3.5], - [[1.0, 1.0, 1.0, 1.0], 4, 40, 400, 4.5] + [[0.0, 0.0, 0.0, 0.0], 1, 10, 100, 10000000001, 1.5, 1.25], + [[0.0, 1.0, 0.0, 1.0], 2, 20, 200, 10000000002, 2.5, 2.25], + [[1.0, 0.0, 1.0, 0.0], 3, 30, 300, 10000000003, 3.5, 3.25], + [[1.0, 1.0, 1.0, 1.0], 4, 40, 400, 10000000004, 4.5, 4.25] ])") .ValueOrDie(); @@ -438,9 +441,24 @@ TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithCompatibleTagArrowTypes) { search_and_check(PredicateBuilder::Equal(/*field_index=*/3, /*field_name=*/"tag_i32", FieldType::INT, Literal(400)), {3l}, {0.01f}); - search_and_check(PredicateBuilder::GreaterThan(/*field_index=*/4, /*field_name=*/"tag_f32", + search_and_check(PredicateBuilder::Equal( + /*field_index=*/4, /*field_name=*/"tag_i64", FieldType::BIGINT, + Literal(static_cast(10000000003))), + {2l}, {2.21f}); + search_and_check(PredicateBuilder::In( + /*field_index=*/4, /*field_name=*/"tag_i64", FieldType::BIGINT, + {Literal(static_cast(10000000001)), + Literal(static_cast(10000000004))}), + {3l, 0l}, {0.01f, 4.21f}); + search_and_check(PredicateBuilder::GreaterThan(/*field_index=*/5, /*field_name=*/"tag_f32", FieldType::FLOAT, Literal(4.0f)), {3l}, {0.01f}); + search_and_check(PredicateBuilder::Equal(/*field_index=*/6, /*field_name=*/"tag_f64", + FieldType::DOUBLE, Literal(3.25)), + {2l}, {2.21f}); + search_and_check(PredicateBuilder::In(/*field_index=*/6, /*field_name=*/"tag_f64", + FieldType::DOUBLE, {Literal(2.25), Literal(4.25)}), + {3l, 1l}, {0.01f, 2.01f}); } TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithTagNullAndEmptyValues) { @@ -452,10 +470,10 @@ TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithTagNullAndEmptyValues) { tag_options["lumina.extension.build.tag.tag_schema"] = R"([{"key_name":"color","type":"enum","value_type":"string"},)" R"({"key_name":"labels","type":"enum","value_type":"string"},)" - R"({"key_name":"price","type":"range","value_type":"double"},)" - R"({"key_name":"scores","type":"enum","value_type":"double"},)" - R"({"key_name":"category","type":"enum","value_type":"int64"},)" - R"({"key_name":"category_ids","type":"enum","value_type":"int64"}])"; + R"({"key_name":"price","type":"range","value_type":"float"},)" + R"({"key_name":"scores","type":"enum","value_type":"float"},)" + R"({"key_name":"category","type":"enum","value_type":"int32"},)" + R"({"key_name":"category_ids","type":"enum","value_type":"int32"}])"; std::shared_ptr tag_data_type = arrow::struct_( {arrow::field("f0", arrow::list(arrow::float32())), arrow::field("color", arrow::utf8()), @@ -599,12 +617,13 @@ TEST_F(LuminaGlobalIndexTest, TestTagSchemaValidation) { R"([{"key_name":"color","type":"range","value_type":"string"}])"; ASSERT_NOK_WITH_MSG( WriteGlobalIndex(index_root, tag_data_type, tag_options, array_, Range(0, 3)), - "lumina tag_schema tag[0] range type does not support string value_type"); + "Option extension.build.tag.tag_schema tag[0] range type does not support value_type " + "'string'"); } { std::map tag_options = options_; tag_options["lumina.extension.build.tag.tag_schema"] = - R"([{"key_name":"color","type":"enum","value_type":"int64"}])"; + R"([{"key_name":"color","type":"enum","value_type":"int32"}])"; ASSERT_NOK_WITH_MSG( WriteGlobalIndex(index_root, tag_data_type, tag_options, array_, Range(0, 3)), "lumina tag field color type string is not compatible with tag_schema value_type"); @@ -615,7 +634,7 @@ TEST_F(LuminaGlobalIndexTest, TestTagSchemaValidation) { arrow::field("category", arrow::int64())}); std::map tag_options = options_; tag_options["lumina.extension.build.tag.tag_schema"] = - R"([{"key_name":"category","type":"enum","value_type":"int64"}])"; + R"([{"key_name":"category","type":"enum","value_type":"int32"}])"; ASSERT_NOK_WITH_MSG( WriteGlobalIndex(index_root, int64_tag_data_type, tag_options, array_, Range(0, 3)), "lumina tag field category type int64 is not compatible with tag_schema value_type"); @@ -629,7 +648,20 @@ TEST_F(LuminaGlobalIndexTest, TestTagSchemaValidation) { R"([{"key_name":"price","type":"range","value_type":"double"}])"; ASSERT_NOK_WITH_MSG( WriteGlobalIndex(index_root, double_tag_data_type, tag_options, array_, Range(0, 3)), - "lumina tag field price type double is not compatible with tag_schema value_type"); + "Option extension.build.tag.tag_schema tag[0] range type does not support value_type " + "'double'"); + } + { + std::shared_ptr int64_tag_data_type = + arrow::struct_({arrow::field("f0", arrow::list(arrow::float32())), + arrow::field("price", arrow::int64())}); + std::map tag_options = options_; + tag_options["lumina.extension.build.tag.tag_schema"] = + R"([{"key_name":"price","type":"range","value_type":"int64"}])"; + ASSERT_NOK_WITH_MSG( + WriteGlobalIndex(index_root, int64_tag_data_type, tag_options, array_, Range(0, 3)), + "Option extension.build.tag.tag_schema tag[0] range type does not support value_type " + "'int64'"); } } @@ -644,8 +676,8 @@ TEST_F(LuminaGlobalIndexTest, TestGetExtraFieldNames) { std::map tag_options = options_; tag_options["lumina.extension.build.tag.tag_schema"] = R"([{"key_name":"color","type":"enum","value_type":"string"},)" - R"({"key_name":"price","type":"range","value_type":"double"},)" - R"({"key_name":"category_ids","type":"enum","value_type":"int64"}])"; + R"({"key_name":"price","type":"range","value_type":"float"},)" + R"({"key_name":"category_ids","type":"enum","value_type":"int32"}])"; LuminaGlobalIndex global_index(tag_options); ASSERT_OK_AND_ASSIGN(std::optional> field_names, global_index.GetExtraFieldNames()); diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp index 978d2255f..3c559ba4c 100644 --- a/test/inte/global_index_test.cpp +++ b/test/inte/global_index_test.cpp @@ -1153,10 +1153,10 @@ TEST_P(GlobalIndexTest, TestWriteAndQueryLuminaIndexWithTagNullAndEmptyValues) { {"lumina.extension.build.tag.tag_schema", R"([{"key_name":"color","type":"enum","value_type":"string"},)" R"({"key_name":"labels","type":"enum","value_type":"string"},)" - R"({"key_name":"price","type":"range","value_type":"double"},)" - R"({"key_name":"scores","type":"range","value_type":"double"},)" - R"({"key_name":"category","type":"enum","value_type":"int64"},)" - R"({"key_name":"category_ids","type":"enum","value_type":"int64"}])"}}; + R"({"key_name":"price","type":"range","value_type":"float"},)" + R"({"key_name":"scores","type":"range","value_type":"float"},)" + R"({"key_name":"category","type":"enum","value_type":"int32"},)" + R"({"key_name":"category_ids","type":"enum","value_type":"int32"}])"}}; std::map options = {{Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format_}, {Options::FILE_SYSTEM, "local"}, @@ -1197,7 +1197,7 @@ TEST_P(GlobalIndexTest, TestWriteAndQueryLuminaIndexWithTagNullAndEmptyValues) { new_index_files[0]->GetGlobalIndexMeta(); ASSERT_TRUE(global_index_meta); std::string expected_index_meta_json = - R"({"distance.metric":"l2","encoding.type":"rawf32","extension.build.tag.tag_schema":"[{\"key_name\":\"color\",\"type\":\"enum\",\"value_type\":\"string\"},{\"key_name\":\"labels\",\"type\":\"enum\",\"value_type\":\"string\"},{\"key_name\":\"price\",\"type\":\"range\",\"value_type\":\"double\"},{\"key_name\":\"scores\",\"type\":\"range\",\"value_type\":\"double\"},{\"key_name\":\"category\",\"type\":\"enum\",\"value_type\":\"int64\"},{\"key_name\":\"category_ids\",\"type\":\"enum\",\"value_type\":\"int64\"}]","index.dimension":"4","index.type":"bruteforce","search.parallel_number":"10"})"; + R"({"distance.metric":"l2","encoding.type":"rawf32","extension.build.tag.tag_schema":"[{\"key_name\":\"color\",\"type\":\"enum\",\"value_type\":\"string\"},{\"key_name\":\"labels\",\"type\":\"enum\",\"value_type\":\"string\"},{\"key_name\":\"price\",\"type\":\"range\",\"value_type\":\"float\"},{\"key_name\":\"scores\",\"type\":\"range\",\"value_type\":\"float\"},{\"key_name\":\"category\",\"type\":\"enum\",\"value_type\":\"int32\"},{\"key_name\":\"category_ids\",\"type\":\"enum\",\"value_type\":\"int32\"}]","index.dimension":"4","index.type":"bruteforce","search.parallel_number":"10"})"; GlobalIndexMeta expected_global_index_meta( /*row_range_start=*/0, /*row_range_end=*/4, /*index_field_id=*/1, /*extra_field_ids=*/std::optional>({2, 3, 4, 5, 6, 7}), diff --git a/third_party/versions.txt b/third_party/versions.txt index 20d9e9d27..a2d3ae0df 100644 --- a/third_party/versions.txt +++ b/third_party/versions.txt @@ -88,8 +88,8 @@ PAIMON_RAPIDJSON_BUILD_VERSION=232389d4f1012dddec4ef84861face2d2ba85709 PAIMON_RAPIDJSON_BUILD_SHA256_CHECKSUM=b9290a9a6d444c8e049bd589ab804e0ccf2b05dc5984a19ed5ae75d090064806 PAIMON_RAPIDJSON_PKG_NAME=rapidjson-${PAIMON_RAPIDJSON_BUILD_VERSION}.tar.gz -PAIMON_LUMINA_BUILD_VERSION=0.3.0-rc1 -PAIMON_LUMINA_BUILD_SHA256_CHECKSUM=6bdb9eeeeb0c6192e480ea0523712df6f07ea58521ca5ca1abc023673dfa1fa5 +PAIMON_LUMINA_BUILD_VERSION=0.3.1 +PAIMON_LUMINA_BUILD_SHA256_CHECKSUM=2aed1ae238c866c12ee01fad17d52651a9029cba1b972b23cd640959310e2149 PAIMON_LUMINA_PKG_NAME=lumina_release-${PAIMON_LUMINA_BUILD_VERSION}.tar.gz PAIMON_JINDOSDK_C_BUILD_VERSION=6.10.2 From b793b920b8e448879819ca490804efa9f95621cc Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Sat, 18 Jul 2026 11:56:50 +0800 Subject: [PATCH 6/8] fix little --- include/paimon/global_index/global_indexer.h | 4 +-- .../bitmap/bitmap_global_index.cpp | 4 +++ .../global_index/bitmap/bitmap_global_index.h | 3 ++ .../btree/btree_global_indexer.cpp | 4 +++ .../global_index/btree/btree_global_indexer.h | 2 ++ .../rangebitmap/range_bitmap_global_index.cpp | 4 +++ .../rangebitmap/range_bitmap_global_index.h | 3 ++ .../lucene/lucene_global_index.cpp | 4 +++ .../global_index/lucene/lucene_global_index.h | 3 ++ .../lumina/lumina_global_index.cpp | 36 +++++++------------ .../lumina/lumina_global_index_test.cpp | 8 ++--- .../tantivy/tantivy_global_index.cpp | 4 +++ .../tantivy/tantivy_global_index.h | 3 ++ 13 files changed, 52 insertions(+), 30 deletions(-) diff --git a/include/paimon/global_index/global_indexer.h b/include/paimon/global_index/global_indexer.h index a5e881366..5145627bf 100644 --- a/include/paimon/global_index/global_indexer.h +++ b/include/paimon/global_index/global_indexer.h @@ -38,9 +38,7 @@ class PAIMON_EXPORT GlobalIndexer { virtual ~GlobalIndexer() = default; /// Returns additional table fields required during index construction. - virtual Result>> GetExtraFieldNames() const { - return std::optional>(std::nullopt); - } + virtual Result>> GetExtraFieldNames() const = 0; /// Creates a writer for building a global index on a specific field. /// diff --git a/src/paimon/common/global_index/bitmap/bitmap_global_index.cpp b/src/paimon/common/global_index/bitmap/bitmap_global_index.cpp index 3e8d75346..98e909cbb 100644 --- a/src/paimon/common/global_index/bitmap/bitmap_global_index.cpp +++ b/src/paimon/common/global_index/bitmap/bitmap_global_index.cpp @@ -19,6 +19,10 @@ #include "paimon/common/global_index/wrap/file_index_writer_wrapper.h" namespace paimon { +Result>> BitmapGlobalIndex::GetExtraFieldNames() const { + return std::optional>(std::nullopt); +} + Result> BitmapGlobalIndex::CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, diff --git a/src/paimon/common/global_index/bitmap/bitmap_global_index.h b/src/paimon/common/global_index/bitmap/bitmap_global_index.h index 2d2f9c140..29f974f05 100644 --- a/src/paimon/common/global_index/bitmap/bitmap_global_index.h +++ b/src/paimon/common/global_index/bitmap/bitmap_global_index.h @@ -17,6 +17,7 @@ #pragma once #include +#include #include #include @@ -29,6 +30,8 @@ class BitmapGlobalIndex : public GlobalIndexer { public: explicit BitmapGlobalIndex(const std::shared_ptr& index) : index_(index) {} + Result>> GetExtraFieldNames() const override; + Result> CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, diff --git a/src/paimon/common/global_index/btree/btree_global_indexer.cpp b/src/paimon/common/global_index/btree/btree_global_indexer.cpp index fdad901b7..aff8e6cf8 100644 --- a/src/paimon/common/global_index/btree/btree_global_indexer.cpp +++ b/src/paimon/common/global_index/btree/btree_global_indexer.cpp @@ -54,6 +54,10 @@ Result> BTreeGlobalIndexer::Create( return std::unique_ptr(new BTreeGlobalIndexer(cache_manager, options)); } +Result>> BTreeGlobalIndexer::GetExtraFieldNames() const { + return std::optional>(std::nullopt); +} + Result> BTreeGlobalIndexer::CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, diff --git a/src/paimon/common/global_index/btree/btree_global_indexer.h b/src/paimon/common/global_index/btree/btree_global_indexer.h index bf05e704d..9bd5f5c7d 100644 --- a/src/paimon/common/global_index/btree/btree_global_indexer.h +++ b/src/paimon/common/global_index/btree/btree_global_indexer.h @@ -53,6 +53,8 @@ class BTreeGlobalIndexer : public GlobalIndexer { static Result> Create( const std::map& options); + Result>> GetExtraFieldNames() const override; + Result> CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, diff --git a/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.cpp b/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.cpp index 8773e412f..5ac55ec3e 100644 --- a/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.cpp +++ b/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.cpp @@ -19,6 +19,10 @@ #include "paimon/common/global_index/wrap/file_index_writer_wrapper.h" namespace paimon { +Result>> RangeBitmapGlobalIndex::GetExtraFieldNames() const { + return std::optional>(std::nullopt); +} + Result> RangeBitmapGlobalIndex::CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, diff --git a/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.h b/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.h index 2e0458428..5c527f0cc 100644 --- a/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.h +++ b/src/paimon/common/global_index/rangebitmap/range_bitmap_global_index.h @@ -17,6 +17,7 @@ #pragma once #include +#include #include #include @@ -30,6 +31,8 @@ class RangeBitmapGlobalIndex : public GlobalIndexer { explicit RangeBitmapGlobalIndex(const std::shared_ptr& index) : index_(index) {} + Result>> GetExtraFieldNames() const override; + Result> CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, diff --git a/src/paimon/global_index/lucene/lucene_global_index.cpp b/src/paimon/global_index/lucene/lucene_global_index.cpp index bef013698..a652e7984 100644 --- a/src/paimon/global_index/lucene/lucene_global_index.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index.cpp @@ -35,6 +35,10 @@ namespace paimon::lucene { LuceneGlobalIndex::LuceneGlobalIndex(const std::map& options) : options_(OptionsUtils::FetchOptionsWithPrefix(kOptionKeyPrefix, options)) {} +Result>> LuceneGlobalIndex::GetExtraFieldNames() const { + return std::optional>(std::nullopt); +} + Result> LuceneGlobalIndex::CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, diff --git a/src/paimon/global_index/lucene/lucene_global_index.h b/src/paimon/global_index/lucene/lucene_global_index.h index 61864c345..1e52f3463 100644 --- a/src/paimon/global_index/lucene/lucene_global_index.h +++ b/src/paimon/global_index/lucene/lucene_global_index.h @@ -17,6 +17,7 @@ #pragma once #include +#include #include #include @@ -31,6 +32,8 @@ class LuceneGlobalIndex : public GlobalIndexer { public: explicit LuceneGlobalIndex(const std::map& options); + Result>> GetExtraFieldNames() const override; + Result> CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, diff --git a/src/paimon/global_index/lumina/lumina_global_index.cpp b/src/paimon/global_index/lumina/lumina_global_index.cpp index e6f1c0fe2..dbf692360 100644 --- a/src/paimon/global_index/lumina/lumina_global_index.cpp +++ b/src/paimon/global_index/lumina/lumina_global_index.cpp @@ -96,11 +96,6 @@ Result ParseTagField(const rapidjson::Value& obj, const std::str return Status::Invalid( fmt::format("lumina tag_schema {} key_name must not be empty", tag_label)); } - if (key_name.size() > ::lumina::core::kMaxTagKNameLength) { - return Status::Invalid(fmt::format("lumina tag_schema {} key_name exceeds max length {}", - tag_label, ::lumina::core::kMaxTagKNameLength)); - } - LuminaTagField::Type parsed_type; if (type == std::string(::lumina::core::kExtensionTagTypeEnum)) { parsed_type = LuminaTagField::Type::ENUM; @@ -178,6 +173,15 @@ Status AppendTagValue(const std::shared_ptr& array, int64_t index, return Status::OK(); } + auto validate_array_type = [&](arrow::Type::type expected_type, + const char* value_type_name) -> Status { + if (array->type_id() != expected_type) { + return Status::Invalid(fmt::format("lumina {} tag field has unsupported arrow type {}", + value_type_name, array->type()->ToString())); + } + return Status::OK(); + }; + if constexpr (std::is_same_v) { switch (array->type_id()) { case arrow::Type::INT8: @@ -195,30 +199,16 @@ Status AppendTagValue(const std::shared_ptr& array, int64_t index, array->type()->ToString())); } } else if constexpr (std::is_same_v) { - if (array->type_id() != arrow::Type::INT64) { - return Status::Invalid(fmt::format( - "lumina int64 tag field has unsupported arrow type {}", array->type()->ToString())); - } + PAIMON_RETURN_NOT_OK(validate_array_type(arrow::Type::INT64, "int64")); AppendPrimitiveTagValue(array, index, values); } else if constexpr (std::is_same_v) { - if (array->type_id() != arrow::Type::FLOAT) { - return Status::Invalid(fmt::format( - "lumina float tag field has unsupported arrow type {}", array->type()->ToString())); - } + PAIMON_RETURN_NOT_OK(validate_array_type(arrow::Type::FLOAT, "float")); AppendPrimitiveTagValue(array, index, values); } else if constexpr (std::is_same_v) { - if (array->type_id() != arrow::Type::DOUBLE) { - return Status::Invalid( - fmt::format("lumina double tag field has unsupported arrow type {}", - array->type()->ToString())); - } + PAIMON_RETURN_NOT_OK(validate_array_type(arrow::Type::DOUBLE, "double")); AppendPrimitiveTagValue(array, index, values); } else if constexpr (std::is_same_v) { - if (array->type_id() != arrow::Type::STRING) { - return Status::Invalid( - fmt::format("lumina string tag field has unsupported arrow type {}", - array->type()->ToString())); - } + PAIMON_RETURN_NOT_OK(validate_array_type(arrow::Type::STRING, "string")); auto string_array = static_cast(array.get()); auto view = string_array->GetView(index); values->emplace_back(view.data(), view.size()); diff --git a/src/paimon/global_index/lumina/lumina_global_index_test.cpp b/src/paimon/global_index/lumina/lumina_global_index_test.cpp index e2ee54ba0..a01830c83 100644 --- a/src/paimon/global_index/lumina/lumina_global_index_test.cpp +++ b/src/paimon/global_index/lumina/lumina_global_index_test.cpp @@ -394,7 +394,7 @@ TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithCompatibleTagArrowTypes) { tag_options["lumina.extension.build.tag.tag_schema"] = R"([{"key_name":"tag_i8","type":"enum","value_type":"int32"},)" R"({"key_name":"tag_i16","type":"enum","value_type":"int32"},)" - R"({"key_name":"tag_i32","type":"enum","value_type":"int32"},)" + R"({"key_name":"tag_i32","type":"range","value_type":"int32"},)" R"({"key_name":"tag_i64","type":"enum","value_type":"int64"},)" R"({"key_name":"tag_f32","type":"range","value_type":"float"},)" R"({"key_name":"tag_f64","type":"enum","value_type":"double"}])"; @@ -438,9 +438,9 @@ TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithCompatibleTagArrowTypes) { PredicateBuilder::Equal(/*field_index=*/2, /*field_name=*/"tag_i16", FieldType::SMALLINT, Literal(static_cast(30))), {2l}, {2.21f}); - search_and_check(PredicateBuilder::Equal(/*field_index=*/3, /*field_name=*/"tag_i32", - FieldType::INT, Literal(400)), - {3l}, {0.01f}); + search_and_check(PredicateBuilder::GreaterThan(/*field_index=*/3, /*field_name=*/"tag_i32", + FieldType::INT, Literal(250)), + {3l, 2l}, {0.01f, 2.21f}); search_and_check(PredicateBuilder::Equal( /*field_index=*/4, /*field_name=*/"tag_i64", FieldType::BIGINT, Literal(static_cast(10000000003))), diff --git a/src/paimon/global_index/tantivy/tantivy_global_index.cpp b/src/paimon/global_index/tantivy/tantivy_global_index.cpp index 7255e6d41..0ad8f5070 100644 --- a/src/paimon/global_index/tantivy/tantivy_global_index.cpp +++ b/src/paimon/global_index/tantivy/tantivy_global_index.cpp @@ -34,6 +34,10 @@ namespace paimon::tantivy { TantivyGlobalIndex::TantivyGlobalIndex(const std::map& options) : options_(OptionsUtils::FetchOptionsWithPrefix(kOptionKeyPrefix, options)) {} +Result>> TantivyGlobalIndex::GetExtraFieldNames() const { + return std::optional>(std::nullopt); +} + Result> TantivyGlobalIndex::CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, diff --git a/src/paimon/global_index/tantivy/tantivy_global_index.h b/src/paimon/global_index/tantivy/tantivy_global_index.h index dea45229b..928bfb04f 100644 --- a/src/paimon/global_index/tantivy/tantivy_global_index.h +++ b/src/paimon/global_index/tantivy/tantivy_global_index.h @@ -18,6 +18,7 @@ #include #include +#include #include #include @@ -34,6 +35,8 @@ class TantivyGlobalIndex : public GlobalIndexer { public: explicit TantivyGlobalIndex(const std::map& options); + Result>> GetExtraFieldNames() const override; + Result> CreateWriter( const std::string& field_name, ::ArrowSchema* arrow_schema, const std::shared_ptr& file_writer, From e9a92f05f5fd7b54e993db7789021aa066b416c5 Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Tue, 21 Jul 2026 16:27:14 +0800 Subject: [PATCH 7/8] fix review --- src/paimon/global_index/lumina/lumina_global_index_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/paimon/global_index/lumina/lumina_global_index_test.cpp b/src/paimon/global_index/lumina/lumina_global_index_test.cpp index a01830c83..4aea7cc9d 100644 --- a/src/paimon/global_index/lumina/lumina_global_index_test.cpp +++ b/src/paimon/global_index/lumina/lumina_global_index_test.cpp @@ -257,7 +257,7 @@ TEST_F(LuminaGlobalIndexTest, TestWriteAndReadWithTagFilter) { std::map tag_options = options_; tag_options["lumina.extension.build.tag.tag_schema"] = - R"([{"key_name":"color","type":"enum","value_type":"string"}])"; + R"({"key_name":"color","type":"enum","value_type":"string"})"; std::shared_ptr tag_data_type = arrow::struct_( {arrow::field("f0", arrow::list(arrow::float32())), arrow::field("color", arrow::utf8())}); From d806d4582cbc0cf0482ec0b001fd29a3b98bde9b Mon Sep 17 00:00:00 2001 From: lxy264173 Date: Tue, 21 Jul 2026 21:59:04 +0800 Subject: [PATCH 8/8] add dict tests --- .../reader/data_evolution_file_reader.cpp | 6 +- .../data_evolution_file_reader_test.cpp | 3 +- .../global_index/global_index_write_task.cpp | 102 +++++++++++++++++- .../lumina/lumina_global_index.cpp | 14 ++- .../lumina/lumina_global_index_test.cpp | 16 ++- test/inte/global_index_test.cpp | 72 ++++++++++++- 6 files changed, 197 insertions(+), 16 deletions(-) diff --git a/src/paimon/common/reader/data_evolution_file_reader.cpp b/src/paimon/common/reader/data_evolution_file_reader.cpp index 019ef7824..eefe900cc 100644 --- a/src/paimon/common/reader/data_evolution_file_reader.cpp +++ b/src/paimon/common/reader/data_evolution_file_reader.cpp @@ -23,6 +23,7 @@ #include "paimon/common/reader/reader_utils.h" #include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" + namespace paimon { Result> DataEvolutionFileReader::Create( std::vector>&& readers, @@ -165,11 +166,10 @@ Result> DataEvolutionFileReader::NextBatchForSingl if (concat_array_vec.empty()) { return std::shared_ptr(); } - if (concat_array_vec.size() == 1) { - // avoid data copy + if (concat_array_vec.size() == 1 && concat_array_vec[0]->offset() == 0) { + // Avoid data copy when the array is already normalized. return concat_array_vec[0]; } - // TODO(xinyu.lxy) remove data copy for efficiency PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr concat_array, arrow::Concatenate(concat_array_vec, arrow_pool_.get())); assert(concat_array->length() == total_array_length); diff --git a/src/paimon/common/reader/data_evolution_file_reader_test.cpp b/src/paimon/common/reader/data_evolution_file_reader_test.cpp index 51f8f7bfd..4eccca7ab 100644 --- a/src/paimon/common/reader/data_evolution_file_reader_test.cpp +++ b/src/paimon/common/reader/data_evolution_file_reader_test.cpp @@ -30,8 +30,8 @@ #include "paimon/testing/mock/mock_file_batch_reader.h" #include "paimon/testing/utils/read_result_collector.h" #include "paimon/testing/utils/testharness.h" -namespace paimon::test { +namespace paimon::test { class DataEvolutionFileReaderTest : public ::testing::Test, public ::testing::WithParamInterface { public: @@ -117,6 +117,7 @@ class DataEvolutionFileReaderTest : public ::testing::Test, if (result_array == nullptr) { break; } + ASSERT_EQ(result_array->offset(), 0); result_array_vec.push_back(result_array); } ASSERT_EQ(result_array_vec.size(), diff --git a/src/paimon/core/global_index/global_index_write_task.cpp b/src/paimon/core/global_index/global_index_write_task.cpp index 00c4f73e9..04c67ae87 100644 --- a/src/paimon/core/global_index/global_index_write_task.cpp +++ b/src/paimon/core/global_index/global_index_write_task.cpp @@ -18,12 +18,16 @@ #include +#include "arrow/array/array_nested.h" #include "arrow/c/bridge.h" #include "arrow/c/helpers.h" +#include "arrow/type.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/utils/arrow/mem_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/core/casting/casting_utils.h" #include "paimon/core/core_options.h" #include "paimon/core/global_index/global_index_file_manager.h" #include "paimon/core/io/data_increment.h" @@ -162,10 +166,99 @@ Result> CreateBatchReader( return table_read->CreateReader(indexed_split); } +Result> CastDictionaryArrayToString( + const std::shared_ptr& array, arrow::MemoryPool* pool) { + arrow::Type::type type_id = array->type_id(); + if (type_id == arrow::Type::DICTIONARY) { + const auto* dictionary_type = + static_cast(array->type().get()); + arrow::Type::type value_type = dictionary_type->value_type()->id(); + if (value_type != arrow::Type::STRING && value_type != arrow::Type::LARGE_STRING) { + return Status::Invalid(fmt::format( + "GlobalIndexWriteTask cannot decode dictionary array with value type {}", + dictionary_type->value_type()->ToString())); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr casted_array, + CastingUtils::Cast(array, arrow::utf8(), arrow::compute::CastOptions::Safe(), pool)); + return casted_array; + } + if (type_id != arrow::Type::STRUCT && type_id != arrow::Type::MAP && + type_id != arrow::Type::LIST) { + return array; + } + + if (type_id == arrow::Type::STRUCT) { + std::shared_ptr struct_array = + std::static_pointer_cast(array); + arrow::ArrayVector children; + for (int32_t i = 0; i < struct_array->num_fields(); i++) { + std::shared_ptr child = struct_array->field(i); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr casted_child, + CastDictionaryArrayToString(child, pool)); + if (casted_child != child && children.empty()) { + children = struct_array->fields(); + } + if (!children.empty()) { + children[i] = std::move(casted_child); + } + } + if (children.empty()) { + return array; + } + std::vector field_names; + field_names.reserve(struct_array->num_fields()); + for (int32_t i = 0; i < struct_array->num_fields(); i++) { + field_names.push_back(struct_array->struct_type()->field(i)->name()); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr casted_array, + arrow::StructArray::Make(children, field_names, struct_array->null_bitmap(), + struct_array->null_count(), struct_array->offset())); + return casted_array; + } + + if (type_id == arrow::Type::MAP) { + std::shared_ptr map_array = + std::static_pointer_cast(array); + std::shared_ptr original_keys = map_array->keys(); + std::shared_ptr original_items = map_array->items(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr keys, + CastDictionaryArrayToString(original_keys, pool)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr items, + CastDictionaryArrayToString(original_items, pool)); + if (keys == original_keys && items == original_items) { + return array; + } + const auto* map_type = static_cast(map_array->type().get()); + std::shared_ptr casted_type = std::make_shared( + map_type->key_field()->WithType(keys->type()), + map_type->item_field()->WithType(items->type()), map_type->keys_sorted()); + return std::make_shared( + casted_type, map_array->length(), map_array->value_offsets(), keys, items, + map_array->null_bitmap(), map_array->null_count(), map_array->offset()); + } + + std::shared_ptr list_array = + std::static_pointer_cast(array); + std::shared_ptr original_values = list_array->values(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr values, + CastDictionaryArrayToString(original_values, pool)); + if (values == original_values) { + return array; + } + const auto* list_type = static_cast(list_array->type().get()); + std::shared_ptr casted_type = + arrow::list(list_type->value_field()->WithType(values->type())); + return std::make_shared( + casted_type, list_array->length(), list_array->value_offsets(), values, + list_array->null_bitmap(), list_array->null_count(), list_array->offset()); +} + Result> BuildIndex( const std::string& field_name, const Range& range, const std::vector& writer_field_names, BatchReader* batch_reader, - GlobalIndexWriter* global_index_writer) { + GlobalIndexWriter* global_index_writer, arrow::MemoryPool* arrow_pool) { while (true) { PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch read_batch, batch_reader->NextBatch()); if (BatchReader::IsEofBatch(read_batch)) { @@ -206,7 +299,9 @@ Result> BuildIndex( fmt::format("read array does not contain {} field in GlobalIndexWriteTask", writer_field_name)); } - writer_arrays.push_back(writer_array); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr decoded_writer_array, + CastDictionaryArrayToString(writer_array, arrow_pool)); + writer_arrays.push_back(std::move(decoded_writer_array)); } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( std::shared_ptr new_array, @@ -260,6 +355,7 @@ Result> GlobalIndexWriteTask::WriteIndex( } const auto& range = ranges[0]; std::shared_ptr pool = memory_pool ? memory_pool : GetDefaultPool(); + std::unique_ptr arrow_pool = GetArrowPool(pool); // load schema PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, CoreOptions::FromMap(options, file_system)); @@ -312,7 +408,7 @@ Result> GlobalIndexWriteTask::WriteIndex( // read from data split and write to index writer PAIMON_ASSIGN_OR_RAISE(std::vector global_index_io_metas, BuildIndex(field_name, range, writer_field_names, batch_reader.get(), - global_index_writer.get())); + global_index_writer.get(), arrow_pool.get())); // generate commit message return ToCommitMessage(index_type, field.Id(), range, global_index_io_metas, diff --git a/src/paimon/global_index/lumina/lumina_global_index.cpp b/src/paimon/global_index/lumina/lumina_global_index.cpp index dbf692360..68ad416a5 100644 --- a/src/paimon/global_index/lumina/lumina_global_index.cpp +++ b/src/paimon/global_index/lumina/lumina_global_index.cpp @@ -862,11 +862,15 @@ Status LuminaIndexWriter::AddBatch(::ArrowArray* arrow_array, return Status::Invalid( "field value array in LuminaIndexWriter is invalid, must not null"); } - if (sliced_values->length() != segment_len * static_cast(dimension_)) { - return Status::Invalid(fmt::format( - "invalid input array in LuminaIndexWriter, length of field array [{}] " - "multiplied dimension [{}] must match length of field value array [{}]", - segment_len, dimension_, sliced_values->length())); + for (int64_t row = segment_start; row < segment_start + segment_len; row++) { + int64_t vector_length = + list_field_array->value_offset(row + 1) - list_field_array->value_offset(row); + if (vector_length != static_cast(dimension_)) { + return Status::Invalid(fmt::format( + "invalid input array in LuminaIndexWriter, vector at row [{}] has length " + "[{}], expected dimension [{}]", + row, vector_length, dimension_)); + } } if (!tag_fields_.empty()) { PAIMON_ASSIGN_OR_RAISE(std::vector tag_data, diff --git a/src/paimon/global_index/lumina/lumina_global_index_test.cpp b/src/paimon/global_index/lumina/lumina_global_index_test.cpp index 4aea7cc9d..2e0fd51cf 100644 --- a/src/paimon/global_index/lumina/lumina_global_index_test.cpp +++ b/src/paimon/global_index/lumina/lumina_global_index_test.cpp @@ -741,8 +741,20 @@ TEST_F(LuminaGlobalIndexTest, TestInvalidInputs) { .ValueOrDie(); ASSERT_NOK_WITH_MSG( WriteGlobalIndex(index_root, data_type_, options_, array, Range(0, 2)), - "invalid input array in LuminaIndexWriter, length of field array [2] multiplied " - "dimension [4] must match length of field value array [7]"); + "invalid input array in LuminaIndexWriter, vector at row [1] has length [3], " + "expected dimension [4]"); + } + { + std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(data_type_, + R"([ + [[0.0, 0.0, 0.0]], + [[0.0, 1.0, 0.0, 1.0, 0.0]] + ])") + .ValueOrDie(); + ASSERT_NOK_WITH_MSG( + WriteGlobalIndex(index_root, data_type_, options_, array, Range(0, 2)), + "invalid input array in LuminaIndexWriter, vector at row [0] has length [3], " + "expected dimension [4]"); } { diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp index 3c559ba4c..7b36daf03 100644 --- a/test/inte/global_index_test.cpp +++ b/test/inte/global_index_test.cpp @@ -325,8 +325,76 @@ TEST_P(GlobalIndexTest, TestWriteLuminaIndexWithMismatchedDimension) { ASSERT_NOK_WITH_MSG( WriteIndex(table_path, /*partition_filters=*/{}, "f1", "lumina", /*options=*/lumina_options, Range(0, 1)), - "invalid input array in LuminaIndexWriter, length of field array [1] multiplied " - "dimension [3] must match length of field value array [4]"); + "invalid input array in LuminaIndexWriter, vector at row [0] has length [4], " + "expected dimension [3]"); +} + +TEST_P(GlobalIndexTest, TestWriteAndQueryLuminaIndexWithOrcDictionaryStringTags) { + if (file_format_ != "orc") { + GTEST_SKIP() << "ORC-only dictionary encoding case"; + } + + arrow::FieldVector fields = {arrow::field("embedding", arrow::list(arrow::float32())), + arrow::field("color", arrow::utf8()), + arrow::field("labels", arrow::list(arrow::utf8()))}; + std::shared_ptr schema = arrow::schema(fields); + std::map lumina_options = { + {"lumina.index.dimension", "4"}, + {"lumina.index.type", "bruteforce"}, + {"lumina.distance.metric", "l2"}, + {"lumina.encoding.type", "rawf32"}, + {"lumina.search.parallel_number", "10"}, + {"lumina.extension.build.tag.tag_schema", + R"([{"key_name":"color","type":"enum","value_type":"string"},)" + R"({"key_name":"labels","type":"enum","value_type":"string"}])"}}; + std::map options = { + {Options::MANIFEST_FORMAT, "orc"}, {Options::FILE_FORMAT, file_format_}, + {Options::FILE_SYSTEM, "local"}, {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, {"orc.dictionary-key-size-threshold", "1"}, + {"orc.read.enable-lazy-decoding", "true"}}; + CreateTable(/*partition_keys=*/{}, schema, options); + + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + std::shared_ptr src_array = + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ +[[0.0, 0.0, 0.0, 0.0], "red", ["hot", "common"]], +[[0.0, 1.0, 0.0, 1.0], "blue", ["cold", "common"]], +[[1.0, 0.0, 1.0, 0.0], "red", ["cold"]], +[[1.0, 1.0, 1.0, 1.0], "blue", ["hot"]] + ])") + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(std::vector> commit_msgs, + WriteArray(table_path, schema->field_names(), src_array)); + ASSERT_OK(Commit(table_path, commit_msgs)); + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{}, "embedding", "lumina", + lumina_options, Range(0, 3))); + + ASSERT_OK_AND_ASSIGN(std::unique_ptr global_index_scan, + GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, + /*partitions=*/std::nullopt, lumina_options, fs_, + /*executor=*/nullptr, pool_)); + ASSERT_OK_AND_ASSIGN(std::vector> lumina_readers, + global_index_scan->CreateReaders("embedding", std::nullopt)); + ASSERT_EQ(lumina_readers.size(), 1u); + + std::vector query = {1.0f, 1.0f, 1.0f, 1.1f}; + auto search_and_check = [&](const std::shared_ptr& predicate, + const std::string& expected) { + std::shared_ptr vector_search = std::make_shared( + "embedding", /*limit=*/4, query, /*filter=*/nullptr, predicate, + /*distance_type=*/std::nullopt, /*options=*/lumina_options); + ASSERT_OK_AND_ASSIGN(std::shared_ptr scored_result, + lumina_readers[0]->VisitVectorSearch(vector_search)); + ASSERT_EQ(scored_result->ToString(), expected); + }; + search_and_check( + PredicateBuilder::Equal(/*field_index=*/1, /*field_name=*/"color", FieldType::STRING, + Literal(FieldType::STRING, "red", 3)), + "row ids: {0,2}, scores: {4.21,2.21}"); + search_and_check( + PredicateBuilder::In(/*field_index=*/2, /*field_name=*/"labels", FieldType::STRING, + {Literal(FieldType::STRING, "hot", 3)}), + "row ids: {0,3}, scores: {4.21,0.01}"); } TEST_P(GlobalIndexTest, TestWriteIndex) {