From ab8663a8df0b78a6e7118923ce057c542639662a Mon Sep 17 00:00:00 2001 From: Luis Faria Date: Wed, 29 Jul 2026 12:03:00 +0100 Subject: [PATCH 1/2] Add semantic (vector) search support via Solr's text-to-vector module Adds a knn_vector Solr field type and query-time vectorization via Solr's language-models module, so free-text queries can be matched by meaning rather than only by keyword overlap. - New TextToVectorFilterParameter FilterParameter subtype (field, query, model, topK) - vectorizes the query at search time via Solr's {!knn_text_to_vector} query parser and restricts results to the topK nearest neighbours of a knn_vector field. Composes with every other filter type the same way (AND/OR/nesting). - AIP/File/Representation collections each get an embedding_vector field (knn_vector, not stored - vectors are large and only useful for the KNN search itself) and a vectorized_b flag (boolean, default false) that an external enrichment service can poll (fq=vectorized_b:false) to find documents still awaiting vectorization. RODA itself never populates embedding_vector - the enrichment pass is a separate service by design, since running embedding calls synchronously on every index write would be far too slow for ingest. - SolrBootstrapUtils registers the embedding model with Solr's text-to-vector-model-store from new core.index.embedding.* config (enabled flag, base URL, model name, dimensions, Solr model name, default top-K) - an OpenAI-compatible /embeddings endpoint, the same contract any external enrichment service should call so both sides of the vector space stay consistent. - appendTextToVector passes the query text through the quoted `v=` local param rather than as trailing text after the local params block. Trailing text is only unambiguous for a single word - a multi-word query gets combined with other filters into one larger boolean query string, and the outer parser can split on the whitespace and try to resolve the extra words against Solr's default search field, which RODA doesn't define. Verified live against a real embedding server with a 12-word natural-language query correctly ranking the semantically relevant document first. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01263fsB7QYevDFgwAY8okcq --- .../roda/core/data/common/RodaConstants.java | 11 ++ .../data/v2/index/filter/FilterParameter.java | 10 +- .../filter/TextToVectorFilterParameter.java | 133 ++++++++++++++++++ .../org/roda/core/index/SolrUtilsTest.java | 35 +++++ .../org/roda/core/index/schema/Field.java | 1 + .../core/index/schema/SolrBootstrapUtils.java | 73 ++++++++++ .../schema/collections/AIPCollection.java | 6 + .../schema/collections/FileCollection.java | 6 + .../collections/RepresentationCollection.java | 6 + .../org/roda/core/index/utils/SolrUtils.java | 28 ++++ .../index/common/conf/managed-schema.xml | 10 ++ .../config/index/common/conf/solrconfig.xml | 12 ++ .../resources/config/roda-core.properties | 32 +++++ 13 files changed, 360 insertions(+), 3 deletions(-) create mode 100644 roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/index/filter/TextToVectorFilterParameter.java diff --git a/roda-common/roda-common-data/src/main/java/org/roda/core/data/common/RodaConstants.java b/roda-common/roda-common-data/src/main/java/org/roda/core/data/common/RodaConstants.java index b39f3640d4..86e85eba1b 100644 --- a/roda-common/roda-common-data/src/main/java/org/roda/core/data/common/RodaConstants.java +++ b/roda-common/roda-common-data/src/main/java/org/roda/core/data/common/RodaConstants.java @@ -48,6 +48,14 @@ public final class RodaConstants { public static final String CORE_SOLR_CLOUD_URLS = "core.solr.cloud.urls"; public static final String CORE_SOLR_STEMMING_LANGUAGE = "core.solr.stemming.language"; + public static final String CORE_INDEX_EMBEDDING_ENABLED = "core.index.embedding.enabled"; + public static final String CORE_INDEX_EMBEDDING_BASE_URL = "core.index.embedding.base_url"; + public static final String CORE_INDEX_EMBEDDING_API_KEY = "core.index.embedding.api_key"; + public static final String CORE_INDEX_EMBEDDING_MODEL_NAME = "core.index.embedding.model_name"; + public static final String CORE_INDEX_EMBEDDING_SOLR_MODEL = "core.index.embedding.solr_model"; + public static final String CORE_INDEX_EMBEDDING_DIMENSIONS = "core.index.embedding.dimensions"; + public static final String CORE_INDEX_EMBEDDING_DEFAULT_TOP_K = "core.index.embedding.default_top_k"; + public static final String CORE_EVENTS_ENABLED = "core.events.enabled"; public static final String CORE_EVENTS_NOTIFIER_AND_HANDLER_ARE_THE_SAME = "core.events.notifier_and_handler_are_the_same"; public static final String CORE_EVENTS_NOTIFIER_CLASS = "core.events.notifier_class"; @@ -1061,6 +1069,9 @@ public enum OrchestratorType { public static final String INDEX_SEARCH = "search"; public static final String INDEX_WILDCARD = "*"; + public static final String INDEX_EMBEDDING_VECTOR = "embedding_vector"; + public static final String INDEX_VECTORIZED = "vectorized_b"; + public static final String INDEX_INSTANCE_ID = "instanceId"; public static final String INDEX_INSTANCE_NAME = "instanceName"; diff --git a/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/index/filter/FilterParameter.java b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/index/filter/FilterParameter.java index 327d4b1564..62e5ede0d9 100644 --- a/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/index/filter/FilterParameter.java +++ b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/index/filter/FilterParameter.java @@ -39,12 +39,14 @@ @JsonSubTypes.Type(value = AndFiltersParameters.class, name = "AndFiltersParameters"), @JsonSubTypes.Type(value = AllFilterParameter.class, name = "AllFilterParameter"), @JsonSubTypes.Type(value = ParentWhichFilterParameter.class, name = "ParentWhichFilterParameter"), - @JsonSubTypes.Type(value = ChildOfFilterParameter.class, name = "ChildOfFilterParameter"),}) + @JsonSubTypes.Type(value = ChildOfFilterParameter.class, name = "ChildOfFilterParameter"), + @JsonSubTypes.Type(value = TextToVectorFilterParameter.class, name = "TextToVectorFilterParameter"),}) @Schema(type = "object", subTypes = {BasicSearchFilterParameter.class, EmptyKeyFilterParameter.class, LikeFilterParameter.class, NotSimpleFilterParameter.class, OneOfManyFilterParameter.class, DateIntervalFilterParameter.class, DateRangeFilterParameter.class, LongRangeFilterParameter.class, StringRangeFilterParameter.class, SimpleFilterParameter.class, OrFiltersParameters.class, AndFiltersParameters.class, - AllFilterParameter.class, ParentWhichFilterParameter.class, ChildOfFilterParameter.class}, discriminatorMapping = { + AllFilterParameter.class, ParentWhichFilterParameter.class, ChildOfFilterParameter.class, + TextToVectorFilterParameter.class}, discriminatorMapping = { @DiscriminatorMapping(value = "BasicSearchFilterParameter", schema = BasicSearchFilterParameter.class), @DiscriminatorMapping(value = "LikeFilterParameter", schema = LikeFilterParameter.class), @DiscriminatorMapping(value = "NotSimpleFilterParameter", schema = NotSimpleFilterParameter.class), @@ -58,7 +60,9 @@ @DiscriminatorMapping(value = "AndFiltersParameters", schema = AndFiltersParameters.class), @DiscriminatorMapping(value = "AllFilterParameter", schema = AllFilterParameter.class), @DiscriminatorMapping(value = "ParentWhichFilterParameter", schema = ParentWhichFilterParameter.class), - @DiscriminatorMapping(value = "ChildOfFilterParameter", schema = ChildOfFilterParameter.class)}, discriminatorProperty = "type") + @DiscriminatorMapping(value = "ChildOfFilterParameter", schema = ChildOfFilterParameter.class), + @DiscriminatorMapping(value = "TextToVectorFilterParameter", + schema = TextToVectorFilterParameter.class)}, discriminatorProperty = "type") public abstract class FilterParameter implements Serializable { @Serial private static final long serialVersionUID = 3744111668897879761L; diff --git a/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/index/filter/TextToVectorFilterParameter.java b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/index/filter/TextToVectorFilterParameter.java new file mode 100644 index 0000000000..7ad0ae3e0c --- /dev/null +++ b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/index/filter/TextToVectorFilterParameter.java @@ -0,0 +1,133 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE file at the root of the source + * tree and available online at + * + * https://github.com/keeps/roda + */ +package org.roda.core.data.v2.index.filter; + +import java.io.Serial; + +import com.fasterxml.jackson.annotation.JsonTypeName; + +/** + * Semantic search parameter that vectorizes {@link #getQuery()} at query time + * (via Solr's text-to-vector module) and restricts results to the + * {@link #getTopK()} nearest neighbours of {@link #getField()}. Vectors + * themselves are populated out-of-band by an external enrichment service, not + * by RODA. + */ +@JsonTypeName("TextToVectorFilterParameter") +public class TextToVectorFilterParameter extends FilterParameter { + @Serial + private static final long serialVersionUID = 1L; + + private String field; + private String query; + private String model; + private int topK; + + /** + * Constructs an empty {@link TextToVectorFilterParameter}. + */ + public TextToVectorFilterParameter() { + // do nothing + } + + public TextToVectorFilterParameter(String field, String query, String model, int topK) { + setField(field); + setQuery(query); + setModel(model); + setTopK(topK); + } + + public TextToVectorFilterParameter(TextToVectorFilterParameter other) { + this(other.getField(), other.getQuery(), other.getModel(), other.getTopK()); + } + + /** + * @return the name of the {@code knn_vector} field to search against. + */ + public String getField() { + return field; + } + + public void setField(String field) { + this.field = field; + } + + /** + * @return the free-text query to vectorize. + */ + public String getQuery() { + return query; + } + + public void setQuery(String query) { + this.query = query; + } + + /** + * @return the name the embedding model is registered under in Solr's + * text-to-vector-model-store. + */ + public String getModel() { + return model; + } + + public void setModel(String model) { + this.model = model; + } + + /** + * @return the number of nearest neighbours to retrieve. + */ + public int getTopK() { + return topK; + } + + public void setTopK(int topK) { + this.topK = topK; + } + + @Override + public String toString() { + return "TextToVectorFilterParameter(field=" + field + ", query=" + query + ", model=" + model + ", topK=" + topK + + ")"; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + ((field == null) ? 0 : field.hashCode()); + result = prime * result + ((query == null) ? 0 : query.hashCode()); + result = prime * result + ((model == null) ? 0 : model.hashCode()); + result = prime * result + topK; + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!super.equals(obj)) { + return false; + } + if (!(obj instanceof TextToVectorFilterParameter other)) { + return false; + } + if (topK != other.topK) { + return false; + } + if (field == null ? other.field != null : !field.equals(other.field)) { + return false; + } + if (query == null ? other.query != null : !query.equals(other.query)) { + return false; + } + return model == null ? other.model == null : model.equals(other.model); + } +} diff --git a/roda-core/roda-core-tests/src/main/java/org/roda/core/index/SolrUtilsTest.java b/roda-core/roda-core-tests/src/main/java/org/roda/core/index/SolrUtilsTest.java index 6be98d7077..ce9d176bf6 100644 --- a/roda-core/roda-core-tests/src/main/java/org/roda/core/index/SolrUtilsTest.java +++ b/roda-core/roda-core-tests/src/main/java/org/roda/core/index/SolrUtilsTest.java @@ -46,6 +46,7 @@ import org.roda.core.data.v2.index.filter.LongRangeFilterParameter; import org.roda.core.data.v2.index.filter.OneOfManyFilterParameter; import org.roda.core.data.v2.index.filter.SimpleFilterParameter; +import org.roda.core.data.v2.index.filter.TextToVectorFilterParameter; import org.roda.core.data.v2.index.sort.SortParameter; import org.roda.core.data.v2.index.sort.Sorter; import org.roda.core.index.utils.SolrUtils; @@ -263,6 +264,40 @@ public void testParseWithOneDateIntervalFilterParameter() { } } + @Test + public void testParserWithOneTextToVectorFilterParameter() { + try { + Filter filter = new Filter(); + filter.add( + new TextToVectorFilterParameter(RodaConstants.INDEX_EMBEDDING_VECTOR, "iraqi ministry of defence meeting", + "roda-embedding-model", 10)); + String stringFilter = SolrUtils.parseFilter(filter); + assertNotNull(stringFilter); + assertEquals(String.format("({!knn_text_to_vector model=%s f=%s topK=%d}%s)", "roda-embedding-model", + RodaConstants.INDEX_EMBEDDING_VECTOR, 10, "iraqi ministry of defence meeting"), stringFilter); + } catch (RODAException e) { + Assert.fail("An exception was not expected!"); + } + } + + @Test + public void testParserWithTextToVectorAndSimpleFilterParameter() { + try { + Filter filter = new Filter(); + filter.add(new TextToVectorFilterParameter(RodaConstants.INDEX_EMBEDDING_VECTOR, "meeting minutes", + "roda-embedding-model", 5)); + filter.add(new SimpleFilterParameter(RodaConstants.INDEX_SEARCH, FONDS)); + String stringFilter = SolrUtils.parseFilter(filter); + assertNotNull(stringFilter); + assertEquals( + String.format("({!knn_text_to_vector model=%s f=%s topK=%d}%s) AND (%s:\"%s\")", "roda-embedding-model", + RodaConstants.INDEX_EMBEDDING_VECTOR, 5, "meeting minutes", RodaConstants.INDEX_SEARCH, FONDS), + stringFilter); + } catch (RODAException e) { + Assert.fail("An exception was not expected!"); + } + } + @Test public void testParseSorter() { Sorter sorter = null; diff --git a/roda-core/roda-core/src/main/java/org/roda/core/index/schema/Field.java b/roda-core/roda-core/src/main/java/org/roda/core/index/schema/Field.java index 0478b30f8a..968f87871c 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/index/schema/Field.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/index/schema/Field.java @@ -33,6 +33,7 @@ public class Field { public static final String TYPE_INT = "pint"; public static final String TYPE_DATE = "pdate"; public static final String TYPE_STRING = "string"; + public static final String TYPE_KNN_VECTOR = "knn_vector"; public static final String FIELD_SEARCH = RodaConstants.INDEX_SEARCH; diff --git a/roda-core/roda-core/src/main/java/org/roda/core/index/schema/SolrBootstrapUtils.java b/roda-core/roda-core/src/main/java/org/roda/core/index/schema/SolrBootstrapUtils.java index 1e5b9ee10a..c14818518c 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/index/schema/SolrBootstrapUtils.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/index/schema/SolrBootstrapUtils.java @@ -8,26 +8,37 @@ package org.roda.core.index.schema; import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.request.GenericSolrRequest; import org.apache.solr.client.solrj.request.schema.SchemaRequest; import org.apache.solr.client.solrj.response.schema.SchemaResponse.CopyFieldsResponse; import org.apache.solr.client.solrj.response.schema.SchemaResponse.DynamicFieldsResponse; import org.apache.solr.client.solrj.response.schema.SchemaResponse.FieldsResponse; +import org.roda.core.RodaCoreFactory; +import org.roda.core.data.common.RodaConstants; import org.roda.core.data.exceptions.GenericException; import org.roda.core.data.v2.IsModelObject; import org.roda.core.data.v2.index.IsIndexed; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.fasterxml.jackson.databind.ObjectMapper; + public class SolrBootstrapUtils { private static final Logger LOGGER = LoggerFactory.getLogger(SolrBootstrapUtils.class); + private static final String TEXT_TO_VECTOR_MODEL_STORE_PATH = "/schema/text-to-vector-model-store"; + private static final String LANGCHAIN4J_OPENAI_MODEL_CLASS = "dev.langchain4j.model.openai.OpenAiEmbeddingModel"; private static Map getFields(SolrClient client, String collectionName) throws GenericException { @@ -111,6 +122,68 @@ private static void bootstrapColl } else { LOGGER.info("Collection {} is up to date", collection.getIndexName()); } + + boolean hasVectorField = collection.getFields().stream() + .anyMatch(f -> Field.TYPE_KNN_VECTOR.equals(f.getType())); + if (hasVectorField) { + registerTextToVectorModel(client, collection.getIndexName()); + } + } + + /** + * Registers (or re-registers) the embedding model used for query-time + * text-to-vector search, so that {@code {!knn_text_to_vector model=...}} + * queries can resolve it. This only wires up query-time vectorization - + * index-time vectors are written by an external enrichment service, not by + * RODA. Best-effort: semantic search is an optional feature, so failures + * here are logged but must not prevent RODA from starting. + */ + private static void registerTextToVectorModel(SolrClient client, String collectionName) { + boolean enabled = Boolean + .parseBoolean(RodaCoreFactory.getRodaConfigurationAsString(RodaConstants.CORE_INDEX_EMBEDDING_ENABLED)); + if (!enabled) { + return; + } + + String solrModel = RodaCoreFactory.getRodaConfigurationAsString(RodaConstants.CORE_INDEX_EMBEDDING_SOLR_MODEL); + String baseUrl = RodaCoreFactory.getRodaConfigurationAsString(RodaConstants.CORE_INDEX_EMBEDDING_BASE_URL); + String modelName = RodaCoreFactory.getRodaConfigurationAsString(RodaConstants.CORE_INDEX_EMBEDDING_MODEL_NAME); + String apiKey = RodaCoreFactory.getRodaConfigurationAsString(RodaConstants.CORE_INDEX_EMBEDDING_API_KEY); + + if (StringUtils.isBlank(solrModel) || StringUtils.isBlank(baseUrl) || StringUtils.isBlank(modelName)) { + LOGGER.warn( + "Semantic search is enabled (core.index.embedding.enabled=true) but base_url/model_name/solr_model " + + "are not fully configured; skipping text-to-vector model registration for collection {}", + collectionName); + return; + } + + Map params = new LinkedHashMap<>(); + params.put("baseUrl", baseUrl); + params.put("modelName", modelName); + // LangChain4j's OpenAI client rejects a null/absent apiKey even against + // auth-free servers, so always send a value. + params.put("apiKey", StringUtils.isNotBlank(apiKey) ? apiKey : "not-needed"); + + Map payload = new LinkedHashMap<>(); + payload.put("class", LANGCHAIN4J_OPENAI_MODEL_CLASS); + payload.put("name", solrModel); + payload.put("params", params); + + try { + byte[] body = new ObjectMapper().writeValueAsBytes(payload); + GenericSolrRequest request = new GenericSolrRequest(SolrRequest.METHOD.PUT, TEXT_TO_VECTOR_MODEL_STORE_PATH) + .withContent(body, "application/json"); + request.setRequiresCollection(true); + request.process(client, collectionName); + LOGGER.info("Registered text-to-vector model '{}' for collection {}", solrModel, collectionName); + } catch (SolrServerException | IOException | RuntimeException e) { + LOGGER.warn( + "Could not register text-to-vector model '{}' for collection {} - semantic search queries against " + + "this collection will fail with an unknown model error until this is resolved (requires the " + + "language-models Solr module to be enabled, see docker-compose SOLR_MODULES)", + solrModel, collectionName, e); + } } public static void bootstrapSchemas(SolrClient client) throws GenericException { diff --git a/roda-core/roda-core/src/main/java/org/roda/core/index/schema/collections/AIPCollection.java b/roda-core/roda-core/src/main/java/org/roda/core/index/schema/collections/AIPCollection.java index f5dbf7f309..d22959e766 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/index/schema/collections/AIPCollection.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/index/schema/collections/AIPCollection.java @@ -129,6 +129,12 @@ public List getFields() { fields.add(SolrCollection.getSortFieldOf(RodaConstants.AIP_TITLE)); + // Semantic search: vector is populated by an external enrichment service on an + // asynchronous second pass, not by RODA. vectorized_b lets that service find + // documents still awaiting vectorization. + fields.add(new Field(RodaConstants.INDEX_EMBEDDING_VECTOR, Field.TYPE_KNN_VECTOR).setStored(false)); + fields.add(new Field(RodaConstants.INDEX_VECTORIZED, Field.TYPE_BOOLEAN).setDefaultValue("false")); + return fields; } diff --git a/roda-core/roda-core/src/main/java/org/roda/core/index/schema/collections/FileCollection.java b/roda-core/roda-core/src/main/java/org/roda/core/index/schema/collections/FileCollection.java index b9b6e6b60c..b835611ec4 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/index/schema/collections/FileCollection.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/index/schema/collections/FileCollection.java @@ -116,6 +116,12 @@ public List getFields() { fields.add(new Field(RodaConstants.INGEST_UPDATE_JOB_IDS, Field.TYPE_STRING).setMultiValued(true)); fields.add(new Field(RodaConstants.FILE_CREATED_ON, Field.TYPE_DATE)); + // Semantic search: vector is populated by an external enrichment service on an + // asynchronous second pass, not by RODA. vectorized_b lets that service find + // documents still awaiting vectorization. + fields.add(new Field(RodaConstants.INDEX_EMBEDDING_VECTOR, Field.TYPE_KNN_VECTOR).setStored(false)); + fields.add(new Field(RodaConstants.INDEX_VECTORIZED, Field.TYPE_BOOLEAN).setDefaultValue("false")); + return fields; } diff --git a/roda-core/roda-core/src/main/java/org/roda/core/index/schema/collections/RepresentationCollection.java b/roda-core/roda-core/src/main/java/org/roda/core/index/schema/collections/RepresentationCollection.java index 0cb27e880b..d0dfa8a78c 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/index/schema/collections/RepresentationCollection.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/index/schema/collections/RepresentationCollection.java @@ -101,6 +101,12 @@ public List getFields() { fields.add(SolrCollection.getSortFieldOf(RodaConstants.REPRESENTATION_TYPE)); // pataki@ END + // Semantic search: vector is populated by an external enrichment service on an + // asynchronous second pass, not by RODA. vectorized_b lets that service find + // documents still awaiting vectorization. + fields.add(new Field(RodaConstants.INDEX_EMBEDDING_VECTOR, Field.TYPE_KNN_VECTOR).setStored(false)); + fields.add(new Field(RodaConstants.INDEX_VECTORIZED, Field.TYPE_BOOLEAN).setDefaultValue("false")); + return fields; } diff --git a/roda-core/roda-core/src/main/java/org/roda/core/index/utils/SolrUtils.java b/roda-core/roda-core/src/main/java/org/roda/core/index/utils/SolrUtils.java index 8ea50319be..2985575f8d 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/index/utils/SolrUtils.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/index/utils/SolrUtils.java @@ -108,6 +108,7 @@ import org.roda.core.data.v2.index.filter.OrFiltersParameters; import org.roda.core.data.v2.index.filter.ParentWhichFilterParameter; import org.roda.core.data.v2.index.filter.SimpleFilterParameter; +import org.roda.core.data.v2.index.filter.TextToVectorFilterParameter; import org.roda.core.data.v2.index.sort.SortParameter; import org.roda.core.data.v2.index.sort.Sorter; import org.roda.core.data.v2.index.sublist.Sublist; @@ -1022,6 +1023,8 @@ private static void parseFilterParameter(StringBuilder ret, FilterParameter para appendBlockJoinFilterParameter(ret, nestParentFilterParameter, prefixWithANDOperatorIfBuilderNotEmpty); } else if (parameter instanceof ChildOfFilterParameter nestChildOfFilterParameter) { appendBlockJoinChildrenFilterParameter(ret, nestChildOfFilterParameter, prefixWithANDOperatorIfBuilderNotEmpty); + } else if (parameter instanceof TextToVectorFilterParameter param) { + appendTextToVector(ret, param, prefixWithANDOperatorIfBuilderNotEmpty); } else { LOGGER.error("Unsupported filter parameter class: {}", parameter.getClass().getName()); throw new RequestNotValidException("Unsupported filter parameter class: " + parameter.getClass().getName()); @@ -1078,6 +1081,31 @@ private static void appendOROperator(StringBuilder ret, boolean prefixWithOROper } } + /** + * Renders a semantic-search clause vectorizing {@code parameter.getQuery()} at + * query time via Solr's {@code knn_text_to_vector} query parser (see + * https://solr.apache.org/guide/solr/latest/query-guide/text-to-vector.html), + * restricting results to the top-K nearest neighbours of the target + * {@code knn_vector} field. + */ + private static void appendTextToVector(StringBuilder ret, TextToVectorFilterParameter parameter, + boolean prefixWithANDOperatorIfBuilderNotEmpty) { + appendANDOperator(ret, prefixWithANDOperatorIfBuilderNotEmpty); + // The query text is passed via the "v" local param (quoted) rather than as trailing + // text after the local params block. Trailing text is only unambiguous for a single + // word - as soon as the free-text query contains a space, the outer query parser + // (this clause is combined with others via AND/OR into one larger query string) can + // split on it and try to resolve the extra words against the default search field, + // which RODA doesn't define, causing "undefined field _text_". + ret.append("({!knn_text_to_vector model=").append(parameter.getModel()).append(" f=") + .append(parameter.getField()).append(" topK=").append(parameter.getTopK()).append(" v='") + .append(escapeSolrLocalParamValue(parameter.getQuery())).append("'})"); + } + + private static String escapeSolrLocalParamValue(String value) { + return value.replace("\\", "\\\\").replace("'", "\\'"); + } + private static void appendExactMatch(StringBuilder ret, String key, String value, boolean appendDoubleQuotes, boolean prefixWithANDOperatorIfBuilderNotEmpty) { appendANDOperator(ret, prefixWithANDOperatorIfBuilderNotEmpty); diff --git a/roda-core/roda-core/src/main/resources/config/index/common/conf/managed-schema.xml b/roda-core/roda-core/src/main/resources/config/index/common/conf/managed-schema.xml index a6410e8d07..a8101b6f4b 100644 --- a/roda-core/roda-core/src/main/resources/config/index/common/conf/managed-schema.xml +++ b/roda-core/roda-core/src/main/resources/config/index/common/conf/managed-schema.xml @@ -1047,4 +1047,14 @@ + + + + \ No newline at end of file diff --git a/roda-core/roda-core/src/main/resources/config/index/common/conf/solrconfig.xml b/roda-core/roda-core/src/main/resources/config/index/common/conf/solrconfig.xml index 0be4ef0ddd..fb19c6bc2f 100644 --- a/roda-core/roda-core/src/main/resources/config/index/common/conf/solrconfig.xml +++ b/roda-core/roda-core/src/main/resources/config/index/common/conf/solrconfig.xml @@ -74,6 +74,12 @@ --> + + + + + +