From 1d2cb23e74af5f08df1a08cc4ab2b87447ac8a8d Mon Sep 17 00:00:00 2001 From: Abhinav Dangeti Date: Fri, 24 Feb 2023 13:29:08 -0700 Subject: [PATCH 1/2] Ability to differentiate between nested fields and those with `.` + Reference: https://issues.couchbase.com/browse/MB-55699 + bleve uses "." as the path separator for nested field names. This can conflict with those fields whose names contains "." within them - which is an allowed parameter. + So if one were to index a document such as this .. ``` { "x": { "y": "1" }, "x.y": "2" } ``` The field "x.y" will contain tokens "1" and "2". The real problem seeps in here when different analyzers are used for these 2 fields - and during search time, an analytic query will not be able to accurately pick an analyzer to apply over the search criteria. + The proposal here is decorate field names under the hood within backticks to preserve their true meaning. So for example .. - ``` `a.b` ``` is a single unnested field name - ``` `a`.`b` ``` is a nested field name with ``` `b` ``` being a child field of ``` `a` ``` + Here're the ramifications with this approach: - While indexing, users can still specify fields names as they appear in their JSON documents. Underneath the hood however, these field names will now be registered with their decorated versions to avoid ambiguity. - While querying, users can still specify fields as they expect to see them within their json documents. Note that, it will be the user's responsibility to differentiate between nested field names and others. Let's consider an index mapping that indexes the document used earlier^. The searches that'd work here are .. 1. ```{"field": "`x.y`", "match": 2}``` 2. ```{"field": "x.y", "match": 1}``` 3. ```{"field": "`x`.`y`", "match": 1}``` - Users will also be responsible for specifying sort keys, facet fields, highlight fields accordingly in their search requests. For example .. ``` x : interpreted as `x` `x` : interpreted as `x` x.y : interpreted as `x`.`y` `x.y` : interpreted as `x.y` `x`.`y`. : interpreted as `x`.`y` ``` - In the search response, users will now see decorated names for fragments, locations and facets to avoid any ambiguous interpretation of the field names. --- examples_test.go | 6 +- http/handlers_test.go | 14 +-- index_impl.go | 8 +- index_test.go | 117 +++++++++++++------------ mapping/document.go | 21 ++--- mapping/field.go | 12 +-- mapping/index.go | 7 +- mapping/mapping_test.go | 74 ++++++++-------- search/collector/topn.go | 2 + search/facet/facet_builder_datetime.go | 3 +- search/facet/facet_builder_numeric.go | 3 +- search/facet/facet_builder_terms.go | 3 +- search/query/bool_field.go | 4 + search/query/date_range.go | 11 ++- search/query/fuzzy.go | 4 + search/query/geo_boundingbox.go | 3 + search/query/geo_boundingpolygon.go | 3 + search/query/geo_distance.go | 3 + search/query/geo_shape.go | 3 + search/query/ip_range.go | 4 + search/query/match.go | 3 + search/query/match_phrase.go | 3 + search/query/multi_phrase.go | 10 ++- search/query/numeric_range.go | 4 + search/query/phrase.go | 10 ++- search/query/prefix.go | 4 + search/query/regexp.go | 3 + search/query/term.go | 4 + search/query/term_range.go | 5 ++ search/query/wildcard.go | 3 + search/sort.go | 5 +- search_test.go | 38 ++++---- test/tests/basic/searches.json | 16 ++-- test/tests/employee/searches.json | 4 +- test/tests/facet/searches.json | 8 +- {mapping => util}/reflect.go | 89 ++++++++++++++++--- {mapping => util}/reflect_test.go | 2 +- 37 files changed, 337 insertions(+), 179 deletions(-) rename {mapping => util}/reflect.go (50%) rename {mapping => util}/reflect_test.go (97%) diff --git a/examples_test.go b/examples_test.go index 66d12bd0e..6202464d4 100644 --- a/examples_test.go +++ b/examples_test.go @@ -321,7 +321,7 @@ func ExampleNewHighlight() { panic(err) } - fmt.Println(searchResults.Hits[0].Fragments["Name"][0]) + fmt.Println(searchResults.Hits[0].Fragments["`Name`"][0]) // Output: // great nameless one } @@ -335,7 +335,7 @@ func ExampleNewHighlightWithStyle() { panic(err) } - fmt.Println(searchResults.Hits[0].Fragments["Name"][0]) + fmt.Println(searchResults.Hits[0].Fragments["`Name`"][0]) // Output: // great nameless one } @@ -446,7 +446,7 @@ func ExampleSearchRequest_SortByCustom() { searchRequest := NewSearchRequest(query) searchRequest.SortByCustom(search.SortOrder{ &search.SortField{ - Field: "Age", + Field: "`Age`", Missing: search.SortFieldMissingFirst, }, &search.SortDocID{}, diff --git a/http/handlers_test.go b/http/handlers_test.go index aff659ccb..f21e994e8 100644 --- a/http/handlers_test.go +++ b/http/handlers_test.go @@ -292,9 +292,9 @@ func TestHandlers(t *testing.T) { }, Status: http.StatusOK, ResponseMatch: map[string]bool{ - `"id":"a"`: true, - `"body":"test"`: true, - `"name":"a"`: true, + "\"id\":\"a\"": true, + "\"`body`\":\"test\"": true, + "\"`name`\":\"a\"": true, }, }, { @@ -483,10 +483,10 @@ func TestHandlers(t *testing.T) { }, Status: http.StatusOK, ResponseMatch: map[string]bool{ - `"fields":`: true, - `"name"`: true, - `"body"`: true, - `"_all"`: true, + "\"fields\"": true, + "\"`name`\"": true, + "\"`body`\"": true, + "\"_all\"": true, }, }, { diff --git a/index_impl.go b/index_impl.go index c5a0c46f4..8e30b8ebf 100644 --- a/index_impl.go +++ b/index_impl.go @@ -34,6 +34,7 @@ import ( "github.com/blevesearch/bleve/v2/search/collector" "github.com/blevesearch/bleve/v2/search/facet" "github.com/blevesearch/bleve/v2/search/highlight" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -631,7 +632,7 @@ func LoadAndHighlightFields(hit *search.DocumentMatch, req *SearchRequest, fieldsToLoad := deDuplicate(req.Fields) for _, f := range fieldsToLoad { doc.VisitFields(func(docF index.Field) { - if f == "*" || docF.Name() == f { + if f == "*" || docF.Name() == util.CleansePath(f) { var value interface{} switch docF := docF.(type) { case index.TextField: @@ -683,7 +684,7 @@ func LoadAndHighlightFields(hit *search.DocumentMatch, req *SearchRequest, } } for _, hf := range highlightFields { - highlighter.BestFragmentsInField(hit, doc, hf, 1) + highlighter.BestFragmentsInField(hit, doc, util.CleansePath(hf), 1) } } } else if doc == nil { @@ -737,6 +738,7 @@ func (i *indexImpl) FieldDict(field string) (index.FieldDict, error) { return nil, err } + field = util.CleansePath(field) fieldDict, err := indexReader.FieldDict(field) if err != nil { i.mutex.RUnlock() @@ -764,6 +766,7 @@ func (i *indexImpl) FieldDictRange(field string, startTerm []byte, endTerm []byt return nil, err } + field = util.CleansePath(field) fieldDict, err := indexReader.FieldDictRange(field, startTerm, endTerm) if err != nil { i.mutex.RUnlock() @@ -791,6 +794,7 @@ func (i *indexImpl) FieldDictPrefix(field string, termPrefix []byte) (index.Fiel return nil, err } + field = util.CleansePath(field) fieldDict, err := indexReader.FieldDictPrefix(field, termPrefix) if err != nil { i.mutex.RUnlock() diff --git a/index_test.go b/index_test.go index 0c89b4e6d..c63a45e28 100644 --- a/index_test.go +++ b/index_test.go @@ -24,7 +24,6 @@ import ( "os" "path/filepath" "reflect" - "sort" "strconv" "strings" "sync" @@ -199,7 +198,7 @@ func TestCrud(t *testing.T) { } foundNameField := false doc.VisitFields(func(field index.Field) { - if field.Name() == "name" && string(field.Value()) == "marty" { + if field.Name() == "`name`" && string(field.Value()) == "marty" { foundNameField = true } }) @@ -212,9 +211,9 @@ func TestCrud(t *testing.T) { t.Fatal(err) } expectedFields := map[string]bool{ - "_all": false, - "name": false, - "desc": false, + "_all": false, + "`name`": false, + "`desc`": false, } if len(fields) < len(expectedFields) { t.Fatalf("expected %d fields got %d", len(expectedFields), len(fields)) @@ -399,10 +398,11 @@ func TestBytesRead(t *testing.T) { if err != nil { t.Error(err) } + stats, _ := idx.StatsMap()["index"].(map[string]interface{}) prevBytesRead, _ := stats["num_bytes_read_at_query_time"].(uint64) - if prevBytesRead != 32349 && res.BytesRead == prevBytesRead { - t.Fatalf("expected bytes read for query string 32349, got %v", + if prevBytesRead != 32475 && res.BytesRead == prevBytesRead { + t.Fatalf("expected bytes read for query string 32475, got %v", prevBytesRead) } @@ -580,8 +580,8 @@ func TestBytesReadStored(t *testing.T) { stats, _ := idx.StatsMap()["index"].(map[string]interface{}) bytesRead, _ := stats["num_bytes_read_at_query_time"].(uint64) - if bytesRead != 25928 && bytesRead == res.BytesRead { - t.Fatalf("expected the bytes read stat to be around 25928, got %v", bytesRead) + if bytesRead != 26054 && bytesRead == res.BytesRead { + t.Fatalf("expected the bytes read stat to be around 26054, got %v", bytesRead) } prevBytesRead := bytesRead @@ -651,8 +651,8 @@ func TestBytesReadStored(t *testing.T) { stats, _ = idx1.StatsMap()["index"].(map[string]interface{}) bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64) - if bytesRead != 18114 && bytesRead == res.BytesRead { - t.Fatalf("expected the bytes read stat to be around 18114, got %v", bytesRead) + if bytesRead != 18240 && bytesRead == res.BytesRead { + t.Fatalf("expected the bytes read stat to be around 18240, got %v", bytesRead) } prevBytesRead = bytesRead @@ -920,17 +920,17 @@ func TestStoredFieldPreserved(t *testing.T) { if len(res.Hits) != 1 { t.Fatalf("expected 1 hit, got %d", len(res.Hits)) } - if res.Hits[0].Fields["name"] != "Marty" { - t.Errorf("expected 'Marty' got '%s'", res.Hits[0].Fields["name"]) + if res.Hits[0].Fields["`name`"] != "Marty" { + t.Errorf("expected 'Marty' got '%s'", res.Hits[0].Fields["`name`"]) } - if res.Hits[0].Fields["desc"] != "GopherCON India" { - t.Errorf("expected 'GopherCON India' got '%s'", res.Hits[0].Fields["desc"]) + if res.Hits[0].Fields["`desc`"] != "GopherCON India" { + t.Errorf("expected 'GopherCON India' got '%s'", res.Hits[0].Fields["`desc`"]) } - if res.Hits[0].Fields["num"] != float64(1) { - t.Errorf("expected '1' got '%v'", res.Hits[0].Fields["num"]) + if res.Hits[0].Fields["`num`"] != float64(1) { + t.Errorf("expected '1' got '%v'", res.Hits[0].Fields["`num`"]) } - if res.Hits[0].Fields["bool"] != true { - t.Errorf("expected 'true' got '%v'", res.Hits[0].Fields["bool"]) + if res.Hits[0].Fields["`bool`"] != true { + t.Errorf("expected 'true' got '%v'", res.Hits[0].Fields["`bool`"]) } } @@ -1185,7 +1185,7 @@ func TestSortMatchSearch(t *testing.T) { } prev := "" for _, hit := range sr.Hits { - val := hit.Fields["Day"].(string) + val := hit.Fields["`Day`"].(string) if prev > val { t.Errorf("Hits must be sorted by 'Day'. Found '%s' before '%s'", prev, val) } @@ -1533,14 +1533,14 @@ func TestTermVectorArrayPositions(t *testing.T) { if results.Total != 1 { t.Fatalf("expected 1 result, got %d", results.Total) } - if len(results.Hits[0].Locations["Messages"]["second"]) < 1 { + if len(results.Hits[0].Locations["`Messages`"]["second"]) < 1 { t.Fatalf("expected at least one location") } - if len(results.Hits[0].Locations["Messages"]["second"][0].ArrayPositions) < 1 { + if len(results.Hits[0].Locations["`Messages`"]["second"][0].ArrayPositions) < 1 { t.Fatalf("expected at least one location array position") } - if results.Hits[0].Locations["Messages"]["second"][0].ArrayPositions[0] != 1 { - t.Fatalf("expected array position 1, got %d", results.Hits[0].Locations["Messages"]["second"][0].ArrayPositions[0]) + if results.Hits[0].Locations["`Messages`"]["second"][0].ArrayPositions[0] != 1 { + t.Fatalf("expected array position 1, got %d", results.Hits[0].Locations["`Messages`"]["second"][0].ArrayPositions[0]) } // repeat search for this document in Messages field @@ -1555,14 +1555,14 @@ func TestTermVectorArrayPositions(t *testing.T) { if results.Total != 1 { t.Fatalf("expected 1 result, got %d", results.Total) } - if len(results.Hits[0].Locations["Messages"]["third"]) < 1 { + if len(results.Hits[0].Locations["`Messages`"]["third"]) < 1 { t.Fatalf("expected at least one location") } - if len(results.Hits[0].Locations["Messages"]["third"][0].ArrayPositions) < 1 { + if len(results.Hits[0].Locations["`Messages`"]["third"][0].ArrayPositions) < 1 { t.Fatalf("expected at least one location array position") } - if results.Hits[0].Locations["Messages"]["third"][0].ArrayPositions[0] != 2 { - t.Fatalf("expected array position 2, got %d", results.Hits[0].Locations["Messages"]["third"][0].ArrayPositions[0]) + if results.Hits[0].Locations["`Messages`"]["third"][0].ArrayPositions[0] != 2 { + t.Fatalf("expected array position 2, got %d", results.Hits[0].Locations["`Messages`"]["third"][0].ArrayPositions[0]) } err = index.Close() @@ -1611,14 +1611,21 @@ func TestDocumentStaticMapping(t *testing.T) { if err != nil { t.Fatal(err) } - sort.Strings(fields) - expectedFields := []string{"Date", "Numeric", "Text", "_all"} + expectedFields := map[string]bool{ + "`Date`": false, + "`Numeric`": false, + "`Text`": false, + "_all": false, + } if len(fields) < len(expectedFields) { - t.Fatalf("invalid field count: %d", len(fields)) + t.Fatalf("expected %d fields got %d", len(expectedFields), len(fields)) + } + for _, f := range fields { + expectedFields[f] = true } - for i, expected := range expectedFields { - if expected != fields[i] { - t.Fatalf("unexpected field[%d]: %s", i, fields[i]) + for ef, efp := range expectedFields { + if !efp { + t.Errorf("field %s is missing", ef) } } @@ -1791,13 +1798,13 @@ func TestDocumentFieldArrayPositionsBug295(t *testing.T) { if results.Total != 1 { t.Fatalf("expected 1 result, got %d", results.Total) } - if len(results.Hits[0].Locations["Messages"]["bleve"]) != 2 { - t.Fatalf("expected 2 locations of 'bleve', got %d", len(results.Hits[0].Locations["Messages"]["bleve"])) + if len(results.Hits[0].Locations["`Messages`"]["bleve"]) != 2 { + t.Fatalf("expected 2 locations of 'bleve', got %d", len(results.Hits[0].Locations["`Messages`"]["bleve"])) } - if results.Hits[0].Locations["Messages"]["bleve"][0].ArrayPositions[0] != 0 { + if results.Hits[0].Locations["`Messages`"]["bleve"][0].ArrayPositions[0] != 0 { t.Errorf("expected array position to be 0") } - if results.Hits[0].Locations["Messages"]["bleve"][1].ArrayPositions[0] != 1 { + if results.Hits[0].Locations["`Messages`"]["bleve"][1].ArrayPositions[0] != 1 { t.Errorf("expected array position to be 1") } @@ -1812,13 +1819,13 @@ func TestDocumentFieldArrayPositionsBug295(t *testing.T) { if results.Total != 1 { t.Fatalf("expected 1 result, got %d", results.Total) } - if len(results.Hits[0].Locations["Messages"]["bleve"]) != 2 { - t.Fatalf("expected 2 locations of 'bleve', got %d", len(results.Hits[0].Locations["Messages"]["bleve"])) + if len(results.Hits[0].Locations["`Messages`"]["bleve"]) != 2 { + t.Fatalf("expected 2 locations of 'bleve', got %d", len(results.Hits[0].Locations["`Messages`"]["bleve"])) } - if results.Hits[0].Locations["Messages"]["bleve"][0].ArrayPositions[0] != 0 { + if results.Hits[0].Locations["`Messages`"]["bleve"][0].ArrayPositions[0] != 0 { t.Errorf("expected array position to be 0") } - if results.Hits[0].Locations["Messages"]["bleve"][1].ArrayPositions[0] != 1 { + if results.Hits[0].Locations["`Messages`"]["bleve"][1].ArrayPositions[0] != 1 { t.Errorf("expected array position to be 1") } @@ -2389,7 +2396,7 @@ func TestBatchMerge(t *testing.T) { foundNameField := false doc.VisitFields(func(field index.Field) { - if field.Name() == "name" && string(field.Value()) == "blahblah" { + if field.Name() == "`name`" && string(field.Value()) == "blahblah" { foundNameField = true } }) @@ -2403,10 +2410,10 @@ func TestBatchMerge(t *testing.T) { } expectedFields := map[string]bool{ - "_all": false, - "name": false, - "desc": false, - "country": false, + "_all": false, + "`name`": false, + "`desc`": false, + "`country`": false, } if len(fields) < len(expectedFields) { t.Fatalf("expected %d fields got %d", len(expectedFields), len(fields)) @@ -2837,7 +2844,7 @@ func TestCopyIndex(t *testing.T) { } foundNameField := false doc.VisitFields(func(field index.Field) { - if field.Name() == "name" && string(field.Value()) == "tester" { + if field.Name() == "`name`" && string(field.Value()) == "tester" { foundNameField = true } }) @@ -2850,9 +2857,9 @@ func TestCopyIndex(t *testing.T) { t.Fatal(err) } expectedFields := map[string]bool{ - "_all": false, - "name": false, - "desc": false, + "_all": false, + "`name`": false, + "`desc`": false, } if len(fields) < len(expectedFields) { t.Fatalf("expected %d fields got %d", len(expectedFields), len(fields)) @@ -2906,7 +2913,7 @@ func TestCopyIndex(t *testing.T) { } copyFoundNameField := false copyDoc.VisitFields(func(field index.Field) { - if field.Name() == "name" && string(field.Value()) == "tester" { + if field.Name() == "`name`" && string(field.Value()) == "tester" { copyFoundNameField = true } }) @@ -2919,9 +2926,9 @@ func TestCopyIndex(t *testing.T) { t.Fatal(err) } copyExpectedFields := map[string]bool{ - "_all": false, - "name": false, - "desc": false, + "_all": false, + "`name`": false, + "`desc`": false, } if len(copyFields) < len(copyExpectedFields) { t.Fatalf("expected %d fields got %d", len(copyExpectedFields), len(copyFields)) diff --git a/mapping/document.go b/mapping/document.go index 3f3bbfd38..52aec3019 100644 --- a/mapping/document.go +++ b/mapping/document.go @@ -23,6 +23,7 @@ import ( "time" "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/util" ) // A DocumentMapping describes how a type of document @@ -97,14 +98,14 @@ func (dm *DocumentMapping) analyzerNameForPath(path string) string { } func (dm *DocumentMapping) fieldDescribedByPath(path string) *FieldMapping { - pathElements := decodePath(path) + pathElements := util.DecodePath(path) if len(pathElements) > 1 { // easy case, there is more than 1 path element remaining // the next path element must match a property name // at this level for propName, subDocMapping := range dm.Properties { if propName == pathElements[0] { - return subDocMapping.fieldDescribedByPath(encodePath(pathElements[1:])) + return subDocMapping.fieldDescribedByPath(util.EncodePath(pathElements[1:])) } } } @@ -115,10 +116,10 @@ func (dm *DocumentMapping) fieldDescribedByPath(path string) *FieldMapping { // first look for property name with empty field for propName, subDocMapping := range dm.Properties { - if propName == path { + if propName == pathElements[0] { // found property name match, now look at its fields for _, field := range subDocMapping.Fields { - if field.Name == "" || field.Name == path { + if field.Name == "" || field.Name == pathElements[0] { // match return field } @@ -127,10 +128,10 @@ func (dm *DocumentMapping) fieldDescribedByPath(path string) *FieldMapping { } // next, walk the properties again, looking for field overriding the name for propName, subDocMapping := range dm.Properties { - if propName != path { + if propName != pathElements[0] { // property name isn't a match, but field name could override it for _, field := range subDocMapping.Fields { - if field.Name == path { + if field.Name == pathElements[0] { return field } } @@ -145,7 +146,7 @@ func (dm *DocumentMapping) fieldDescribedByPath(path string) *FieldMapping { // closest document mapping to a field not explicitly mapped // use closestDocMapping func (dm *DocumentMapping) documentMappingForPath(path string) *DocumentMapping { - pathElements := decodePath(path) + pathElements := util.DecodePath(path) current := dm OUTER: for i, pathElement := range pathElements { @@ -173,7 +174,7 @@ OUTER: // closestDocMapping findest the most specific document mapping that matches // part of the provided path func (dm *DocumentMapping) closestDocMapping(path string) *DocumentMapping { - pathElements := decodePath(path) + pathElements := util.DecodePath(path) current := dm OUTER: for _, pathElement := range pathElements { @@ -361,7 +362,7 @@ func (dm *DocumentMapping) walkDocument(data interface{}, path []string, indexes // if the field has a name under the specified tag, prefer that tag := field.Tag.Get(structTagKey) - tagFieldName := parseTagName(tag) + tagFieldName := util.ParseTagName(tag) if tagFieldName == "-" { continue } @@ -406,7 +407,7 @@ func (dm *DocumentMapping) walkDocument(data interface{}, path []string, indexes } func (dm *DocumentMapping) processProperty(property interface{}, path []string, indexes []uint64, context *walkContext) { - pathString := encodePath(path) + pathString := util.EncodePath(path) // look to see if there is a mapping for this field subDocMapping := dm.documentMappingForPath(pathString) closestDocMapping := dm.closestDocMapping(pathString) diff --git a/mapping/field.go b/mapping/field.go index 511782acc..955928481 100644 --- a/mapping/field.go +++ b/mapping/field.go @@ -20,12 +20,12 @@ import ( "net" "time" - "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" - index "github.com/blevesearch/bleve_index_api" - "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" "github.com/blevesearch/bleve/v2/document" "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" ) // control the default behavior for dynamic fields (those not explicitly mapped) @@ -377,11 +377,11 @@ func (fm *FieldMapping) analyzerForField(path []string, context *walkContext) an func getFieldName(pathString string, path []string, fieldMapping *FieldMapping) string { fieldName := pathString if fieldMapping.Name != "" { - parentName := "" + parentName := []string{} if len(path) > 1 { - parentName = encodePath(path[:len(path)-1]) + pathSeparator + parentName = path[:len(path)-1] } - fieldName = parentName + fieldMapping.Name + fieldName = util.EncodePath(append(parentName, fieldMapping.Name)) } return fieldName } diff --git a/mapping/index.go b/mapping/index.go index 1d982dd41..f0be5446d 100644 --- a/mapping/index.go +++ b/mapping/index.go @@ -17,13 +17,14 @@ package mapping import ( "encoding/json" "fmt" - index "github.com/blevesearch/bleve_index_api" "github.com/blevesearch/bleve/v2/analysis" "github.com/blevesearch/bleve/v2/analysis/analyzer/standard" "github.com/blevesearch/bleve/v2/analysis/datetime/optional" "github.com/blevesearch/bleve/v2/document" "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" ) var MappingJSONStrict = false @@ -310,7 +311,7 @@ func (im *IndexMappingImpl) determineType(data interface{}) string { } // now see if we can find a type using the mapping - typ, ok := mustString(lookupPropertyPath(data, im.TypeField)) + typ, ok := util.LookupPropertyPathStr(data, im.TypeField) if ok { return typ } @@ -375,7 +376,7 @@ func (im *IndexMappingImpl) AnalyzerNameForPath(path string) string { } // next we will try default analyzers for the path - pathDecoded := decodePath(path) + pathDecoded := util.DecodePath(path) for _, docMapping := range im.TypeMapping { rv := docMapping.defaultAnalyzerName(pathDecoded) if rv != "" { diff --git a/mapping/mapping_test.go b/mapping/mapping_test.go index e0151af7a..aeedfca71 100644 --- a/mapping/mapping_test.go +++ b/mapping/mapping_test.go @@ -105,10 +105,10 @@ func TestMappingStructWithJSONTags(t *testing.T) { foundNoJSONName := false count := 0 for _, f := range doc.Fields { - if f.Name() == "name" { + if f.Name() == "`name`" { foundJSONName = true } - if f.Name() == "NoJSONTag" { + if f.Name() == "`NoJSONTag`" { foundNoJSONName = true } count++ @@ -145,10 +145,10 @@ func TestMappingStructWithJSONTagsOneDisabled(t *testing.T) { foundNoJSONName := false count := 0 for _, f := range doc.Fields { - if f.Name() == "name" { + if f.Name() == "`name`" { foundJSONName = true } - if f.Name() == "NoJSONTag" { + if f.Name() == "`NoJSONTag`" { foundNoJSONName = true } count++ @@ -185,10 +185,10 @@ func TestMappingStructWithAlternateTags(t *testing.T) { foundNoBLEVEName := false count := 0 for _, f := range doc.Fields { - if f.Name() == "name" { + if f.Name() == "`name`" { foundBLEVEName = true } - if f.Name() == "NoBLEVETag" { + if f.Name() == "`NoBLEVETag`" { foundNoBLEVEName = true } count++ @@ -227,10 +227,10 @@ func TestMappingStructWithAlternateTagsTwoDisabled(t *testing.T) { foundNoBLEVEName := false count := 0 for _, f := range doc.Fields { - if f.Name() == "name" { + if f.Name() == "`name`" { foundBLEVEName = true } - if f.Name() == "NoBLEVETag" { + if f.Name() == "`NoBLEVETag`" { foundNoBLEVEName = true } count++ @@ -266,7 +266,7 @@ func TestMappingStructWithPointerToString(t *testing.T) { found := false count := 0 for _, f := range doc.Fields { - if f.Name() == "Name" { + if f.Name() == "`Name`" { found = true } count++ @@ -298,7 +298,7 @@ func TestMappingJSONWithNull(t *testing.T) { found := false count := 0 for _, f := range doc.Fields { - if f.Name() == "name" { + if f.Name() == "`name`" { found = true } count++ @@ -504,10 +504,10 @@ func TestMappingBool(t *testing.T) { foundPProp := false count := 0 for _, f := range doc.Fields { - if f.Name() == "prop" { + if f.Name() == "`prop`" { foundProp = true } - if f.Name() == "pprop" { + if f.Name() == "`pprop`" { foundPProp = true } count++ @@ -735,17 +735,17 @@ func TestAnonymousStructFields(t *testing.T) { if len(doc.Fields) != 4 { t.Fatalf("expected 4 fields, got %d", len(doc.Fields)) } - if doc.Fields[0].Name() != "Contact0" { - t.Errorf("expected field named 'Contact0', got '%s'", doc.Fields[0].Name()) + if doc.Fields[0].Name() != "`Contact0`" { + t.Errorf("expected field named '`Contact0`', got '%s'", doc.Fields[0].Name()) } - if doc.Fields[1].Name() != "Name" { - t.Errorf("expected field named 'Name', got '%s'", doc.Fields[1].Name()) + if doc.Fields[1].Name() != "`Name`" { + t.Errorf("expected field named '`Name`', got '%s'", doc.Fields[1].Name()) } - if doc.Fields[2].Name() != "Contact2.Name" { - t.Errorf("expected field named 'Contact2.Name', got '%s'", doc.Fields[2].Name()) + if doc.Fields[2].Name() != "`Contact2`.`Name`" { + t.Errorf("expected field named '`Contact2`.`Name`', got '%s'", doc.Fields[2].Name()) } - if doc.Fields[3].Name() != "Contact3" { - t.Errorf("expected field named 'Contact3', got '%s'", doc.Fields[3].Name()) + if doc.Fields[3].Name() != "`Contact3`" { + t.Errorf("expected field named '`Contact3`', got '%s'", doc.Fields[3].Name()) } type AnotherThing struct { @@ -775,17 +775,17 @@ func TestAnonymousStructFields(t *testing.T) { if len(doc2.Fields) != 4 { t.Fatalf("expected 4 fields, got %d", len(doc2.Fields)) } - if doc2.Fields[0].Name() != "Alternate0" { - t.Errorf("expected field named 'Alternate0', got '%s'", doc2.Fields[0].Name()) + if doc2.Fields[0].Name() != "`Alternate0`" { + t.Errorf("expected field named '`Alternate0`', got '%s'", doc2.Fields[0].Name()) } - if doc2.Fields[1].Name() != "Alternate1.Name" { - t.Errorf("expected field named 'Name', got '%s'", doc2.Fields[1].Name()) + if doc2.Fields[1].Name() != "`Alternate1`.`Name`" { + t.Errorf("expected field named '`Alternte1`.`Name`', got '%s'", doc2.Fields[1].Name()) } - if doc2.Fields[2].Name() != "Alternate2.Name" { - t.Errorf("expected field named 'Alternate2.Name', got '%s'", doc2.Fields[2].Name()) + if doc2.Fields[2].Name() != "`Alternate2`.`Name`" { + t.Errorf("expected field named '`Alternate2`.`Name`', got '%s'", doc2.Fields[2].Name()) } - if doc2.Fields[3].Name() != "Alternate3" { - t.Errorf("expected field named 'Alternate3', got '%s'", doc2.Fields[3].Name()) + if doc2.Fields[3].Name() != "`Alternate3`" { + t.Errorf("expected field named '`Alternate3`', got '%s'", doc2.Fields[3].Name()) } } @@ -811,8 +811,8 @@ func TestAnonymousStructFieldWithJSONStructTagEmptString(t *testing.T) { if len(doc.Fields) != 1 { t.Fatalf("expected 1 field, got %d", len(doc.Fields)) } - if doc.Fields[0].Name() != "key" { - t.Errorf("expected field named 'key', got '%s'", doc.Fields[0].Name()) + if doc.Fields[0].Name() != "`key`" { + t.Errorf("expected field named '`key`', got '%s'", doc.Fields[0].Name()) } } @@ -968,7 +968,7 @@ func TestMappingForGeo(t *testing.T) { var foundGeo bool for _, f := range doc.Fields { - if f.Name() == "location" { + if f.Name() == "`location`" { foundGeo = true geoF, ok := f.(index.GeoPointField) if !ok { @@ -1030,8 +1030,8 @@ func TestMappingForTextMarshaler(t *testing.T) { if len(doc.Fields) != 1 { t.Fatalf("expected 1 field, got: %d", len(doc.Fields)) } - if doc.Fields[0].Name() != "Marshalable.Extra" { - t.Errorf("expected field to be named 'Marshalable.Extra', got: '%s'", doc.Fields[0].Name()) + if doc.Fields[0].Name() != "`Marshalable`.`Extra`" { + t.Errorf("expected field to be named '`Marshalable`.`Extra`', got: '%s'", doc.Fields[0].Name()) } if string(doc.Fields[0].Value()) != tm.Marshalable.Extra { t.Errorf("expected field value to be '%s', got: '%s'", tm.Marshalable.Extra, string(doc.Fields[0].Value())) @@ -1051,8 +1051,8 @@ func TestMappingForTextMarshaler(t *testing.T) { t.Fatalf("expected 1 field, got: %d", len(doc.Fields)) } - if doc.Fields[0].Name() != "Marshalable" { - t.Errorf("expected field to be named 'Marshalable', got: '%s'", doc.Fields[0].Name()) + if doc.Fields[0].Name() != "`Marshalable`" { + t.Errorf("expected field to be named '`Marshalable`', got: '%s'", doc.Fields[0].Name()) } want, err := tm.Marshalable.MarshalText() if err != nil { @@ -1167,7 +1167,7 @@ func TestWrongAnalyzerSearchableAs(t *testing.T) { indexMapping := NewIndexMapping() indexMapping.AddDocumentMapping("brewery", docMapping) - analyzerName := indexMapping.AnalyzerNameForPath("geo.geo.accuracy") + analyzerName := indexMapping.AnalyzerNameForPath("`geo`.`geo.accuracy`") if analyzerName != "xyz" { t.Errorf("expected analyzer name `xyz`, got `%s`", analyzerName) } @@ -1228,7 +1228,7 @@ func TestMappingArrayOfStringGeoPoints(t *testing.T) { } for _, f := range doc.Fields { - if f.Name() == "points" { + if f.Name() == "`points`" { geoF, ok := f.(*document.GeoPointField) if !ok { t.Errorf("expected a geopoint field!") diff --git a/search/collector/topn.go b/search/collector/topn.go index 4d19cd455..edb290549 100644 --- a/search/collector/topn.go +++ b/search/collector/topn.go @@ -22,6 +22,7 @@ import ( "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/size" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -180,6 +181,7 @@ func (hc *TopNCollector) Collect(ctx context.Context, searcher search.Searcher, } hc.updateFieldVisitor = func(field string, term []byte) { + field = util.CleansePath(field) if hc.facetsBuilder != nil { hc.facetsBuilder.UpdateVisitor(field, term) } diff --git a/search/facet/facet_builder_datetime.go b/search/facet/facet_builder_datetime.go index ff5167f21..a9990c05a 100644 --- a/search/facet/facet_builder_datetime.go +++ b/search/facet/facet_builder_datetime.go @@ -22,6 +22,7 @@ import ( "github.com/blevesearch/bleve/v2/numeric" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/size" + "github.com/blevesearch/bleve/v2/util" ) var reflectStaticSizeDateTimeFacetBuilder int @@ -52,7 +53,7 @@ type DateTimeFacetBuilder struct { func NewDateTimeFacetBuilder(field string, size int) *DateTimeFacetBuilder { return &DateTimeFacetBuilder{ size: size, - field: field, + field: util.CleansePath(field), termsCount: make(map[string]int), ranges: make(map[string]*dateTimeRange, 0), } diff --git a/search/facet/facet_builder_numeric.go b/search/facet/facet_builder_numeric.go index f19634d7b..c35b149e2 100644 --- a/search/facet/facet_builder_numeric.go +++ b/search/facet/facet_builder_numeric.go @@ -21,6 +21,7 @@ import ( "github.com/blevesearch/bleve/v2/numeric" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/size" + "github.com/blevesearch/bleve/v2/util" ) var reflectStaticSizeNumericFacetBuilder int @@ -51,7 +52,7 @@ type NumericFacetBuilder struct { func NewNumericFacetBuilder(field string, size int) *NumericFacetBuilder { return &NumericFacetBuilder{ size: size, - field: field, + field: util.CleansePath(field), termsCount: make(map[string]int), ranges: make(map[string]*numericRange, 0), } diff --git a/search/facet/facet_builder_terms.go b/search/facet/facet_builder_terms.go index c5a1c8318..04a8ac1ed 100644 --- a/search/facet/facet_builder_terms.go +++ b/search/facet/facet_builder_terms.go @@ -20,6 +20,7 @@ import ( "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/size" + "github.com/blevesearch/bleve/v2/util" ) var reflectStaticSizeTermsFacetBuilder int @@ -41,7 +42,7 @@ type TermsFacetBuilder struct { func NewTermsFacetBuilder(field string, size int) *TermsFacetBuilder { return &TermsFacetBuilder{ size: size, - field: field, + field: util.CleansePath(field), termsCount: make(map[string]int), } } diff --git a/search/query/bool_field.go b/search/query/bool_field.go index 5aa7bb8af..92b315b84 100644 --- a/search/query/bool_field.go +++ b/search/query/bool_field.go @@ -20,6 +20,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -57,7 +58,10 @@ func (q *BoolFieldQuery) Searcher(ctx context.Context, i index.IndexReader, m ma field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } + term := "F" if q.Bool { term = "T" diff --git a/search/query/date_range.go b/search/query/date_range.go index ef18f2fb8..a7f567538 100644 --- a/search/query/date_range.go +++ b/search/query/date_range.go @@ -27,6 +27,7 @@ import ( "github.com/blevesearch/bleve/v2/registry" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -96,7 +97,9 @@ type DateRangeQuery struct { // NewDateRangeQuery creates a new Query for ranges // of date values. // Date strings are parsed using the DateTimeParser configured in the -// top-level config.QueryDateTimeParser +// +// top-level config.QueryDateTimeParser +// // Either, but not both endpoints can be nil. func NewDateRangeQuery(start, end time.Time) *DateRangeQuery { return NewDateRangeInclusiveQuery(start, end, nil, nil) @@ -105,7 +108,9 @@ func NewDateRangeQuery(start, end time.Time) *DateRangeQuery { // NewDateRangeInclusiveQuery creates a new Query for ranges // of date values. // Date strings are parsed using the DateTimeParser configured in the -// top-level config.QueryDateTimeParser +// +// top-level config.QueryDateTimeParser +// // Either, but not both endpoints can be nil. // startInclusive and endInclusive control inclusion of the endpoints. func NewDateRangeInclusiveQuery(start, end time.Time, startInclusive, endInclusive *bool) *DateRangeQuery { @@ -143,6 +148,8 @@ func (q *DateRangeQuery) Searcher(ctx context.Context, i index.IndexReader, m ma field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } return searcher.NewNumericRangeSearcher(ctx, i, min, max, q.InclusiveStart, q.InclusiveEnd, field, q.BoostVal.Value(), options) diff --git a/search/query/fuzzy.go b/search/query/fuzzy.go index f24eb0c20..2491e0227 100644 --- a/search/query/fuzzy.go +++ b/search/query/fuzzy.go @@ -20,6 +20,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -74,6 +75,9 @@ func (q *FuzzyQuery) Searcher(ctx context.Context, i index.IndexReader, m mappin field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } + return searcher.NewFuzzySearcher(ctx, i, q.Term, q.Prefix, q.Fuzziness, field, q.BoostVal.Value(), options) } diff --git a/search/query/geo_boundingbox.go b/search/query/geo_boundingbox.go index ac9125393..954809d66 100644 --- a/search/query/geo_boundingbox.go +++ b/search/query/geo_boundingbox.go @@ -23,6 +23,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -61,6 +62,8 @@ func (q *GeoBoundingBoxQuery) Searcher(ctx context.Context, i index.IndexReader, field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } if q.BottomRight[0] < q.TopLeft[0] { diff --git a/search/query/geo_boundingpolygon.go b/search/query/geo_boundingpolygon.go index 467f39b28..cb98a425b 100644 --- a/search/query/geo_boundingpolygon.go +++ b/search/query/geo_boundingpolygon.go @@ -23,6 +23,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -59,6 +60,8 @@ func (q *GeoBoundingPolygonQuery) Searcher(ctx context.Context, i index.IndexRea field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } return searcher.NewGeoBoundedPolygonSearcher(ctx, i, q.Points, field, q.BoostVal.Value(), options) diff --git a/search/query/geo_distance.go b/search/query/geo_distance.go index f05bf6723..4b4ccf2b3 100644 --- a/search/query/geo_distance.go +++ b/search/query/geo_distance.go @@ -23,6 +23,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -62,6 +63,8 @@ func (q *GeoDistanceQuery) Searcher(ctx context.Context, i index.IndexReader, m field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } dist, err := geo.ParseDistance(q.Distance) diff --git a/search/query/geo_shape.go b/search/query/geo_shape.go index a63ec80f7..65e323155 100644 --- a/search/query/geo_shape.go +++ b/search/query/geo_shape.go @@ -22,6 +22,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -105,6 +106,8 @@ func (q *GeoShapeQuery) Searcher(ctx context.Context, i index.IndexReader, field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } return searcher.NewGeoShapeSearcher(ctx, i, q.Geometry.Shape, q.Geometry.Relation, field, diff --git a/search/query/ip_range.go b/search/query/ip_range.go index ba46f0b25..3f93eeab5 100644 --- a/search/query/ip_range.go +++ b/search/query/ip_range.go @@ -22,6 +22,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -58,7 +59,10 @@ func (q *IPRangeQuery) Searcher(ctx context.Context, i index.IndexReader, m mapp field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } + _, ipNet, err := net.ParseCIDR(q.CIDR) if err != nil { ip := net.ParseIP(q.CIDR) diff --git a/search/query/match.go b/search/query/match.go index 61c00a003..34b5351cb 100644 --- a/search/query/match.go +++ b/search/query/match.go @@ -21,6 +21,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -120,6 +121,8 @@ func (q *MatchQuery) Searcher(ctx context.Context, i index.IndexReader, m mappin field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } analyzerName := "" diff --git a/search/query/match_phrase.go b/search/query/match_phrase.go index fa8ac720b..8989efaf8 100644 --- a/search/query/match_phrase.go +++ b/search/query/match_phrase.go @@ -21,6 +21,7 @@ import ( "github.com/blevesearch/bleve/v2/analysis" "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -66,6 +67,8 @@ func (q *MatchPhraseQuery) Searcher(ctx context.Context, i index.IndexReader, m field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } analyzerName := "" diff --git a/search/query/multi_phrase.go b/search/query/multi_phrase.go index 2887be16a..b44ea6e54 100644 --- a/search/query/multi_phrase.go +++ b/search/query/multi_phrase.go @@ -22,6 +22,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -57,7 +58,14 @@ func (q *MultiPhraseQuery) Boost() float64 { } func (q *MultiPhraseQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { - return searcher.NewMultiPhraseSearcher(ctx, i, q.Terms, q.Field, options) + field := q.Field + if q.Field == "" { + field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) + } + + return searcher.NewMultiPhraseSearcher(ctx, i, q.Terms, field, options) } func (q *MultiPhraseQuery) Validate() error { diff --git a/search/query/numeric_range.go b/search/query/numeric_range.go index ad2474167..c78f38977 100644 --- a/search/query/numeric_range.go +++ b/search/query/numeric_range.go @@ -21,6 +21,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -76,7 +77,10 @@ func (q *NumericRangeQuery) Searcher(ctx context.Context, i index.IndexReader, m field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } + return searcher.NewNumericRangeSearcher(ctx, i, q.Min, q.Max, q.InclusiveMin, q.InclusiveMax, field, q.BoostVal.Value(), options) } diff --git a/search/query/phrase.go b/search/query/phrase.go index 207e66b17..ac44f6288 100644 --- a/search/query/phrase.go +++ b/search/query/phrase.go @@ -22,6 +22,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -54,7 +55,14 @@ func (q *PhraseQuery) Boost() float64 { } func (q *PhraseQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { - return searcher.NewPhraseSearcher(ctx, i, q.Terms, q.Field, options) + field := q.Field + if q.Field == "" { + field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) + } + + return searcher.NewPhraseSearcher(ctx, i, q.Terms, field, options) } func (q *PhraseQuery) Validate() error { diff --git a/search/query/prefix.go b/search/query/prefix.go index debbbc1e3..656522833 100644 --- a/search/query/prefix.go +++ b/search/query/prefix.go @@ -20,6 +20,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -59,6 +60,9 @@ func (q *PrefixQuery) Searcher(ctx context.Context, i index.IndexReader, m mappi field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } + return searcher.NewTermPrefixSearcher(ctx, i, q.Prefix, field, q.BoostVal.Value(), options) } diff --git a/search/query/regexp.go b/search/query/regexp.go index 6b3da9554..d4ee30163 100644 --- a/search/query/regexp.go +++ b/search/query/regexp.go @@ -21,6 +21,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -62,6 +63,8 @@ func (q *RegexpQuery) Searcher(ctx context.Context, i index.IndexReader, m mappi field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } // require that pattern NOT be anchored to start and end of term. diff --git a/search/query/term.go b/search/query/term.go index 5c6af3962..2a85c041c 100644 --- a/search/query/term.go +++ b/search/query/term.go @@ -20,6 +20,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -58,6 +59,9 @@ func (q *TermQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } + return searcher.NewTermSearcher(ctx, i, q.Term, field, q.BoostVal.Value(), options) } diff --git a/search/query/term_range.go b/search/query/term_range.go index 4dc3a34b7..54b0ab38a 100644 --- a/search/query/term_range.go +++ b/search/query/term_range.go @@ -21,6 +21,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -76,7 +77,10 @@ func (q *TermRangeQuery) Searcher(ctx context.Context, i index.IndexReader, m ma field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } + var minTerm []byte if q.Min != "" { minTerm = []byte(q.Min) @@ -85,6 +89,7 @@ func (q *TermRangeQuery) Searcher(ctx context.Context, i index.IndexReader, m ma if q.Max != "" { maxTerm = []byte(q.Max) } + return searcher.NewTermRangeSearcher(ctx, i, minTerm, maxTerm, q.InclusiveMin, q.InclusiveMax, field, q.BoostVal.Value(), options) } diff --git a/search/query/wildcard.go b/search/query/wildcard.go index f04f3f2ed..2d747c5e2 100644 --- a/search/query/wildcard.go +++ b/search/query/wildcard.go @@ -21,6 +21,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -81,6 +82,8 @@ func (q *WildcardQuery) Searcher(ctx context.Context, i index.IndexReader, m map field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } regexpString := wildcardRegexpReplacer.Replace(q.Wildcard) diff --git a/search/sort.go b/search/sort.go index 3a744af99..995ee6e00 100644 --- a/search/sort.go +++ b/search/sort.go @@ -25,6 +25,7 @@ import ( "github.com/blevesearch/bleve/v2/geo" "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/util" ) var HighTerm = strings.Repeat(string(utf8.MaxRune), 3) @@ -64,6 +65,7 @@ func ParseSearchSortObj(input map[string]interface{}) (SearchSort, error) { if !ok { return nil, fmt.Errorf("search sort mode geo_distance must specify field") } + field = util.CleansePath(field) lon, lat, foundLocation := geo.ExtractGeoPoint(input["location"]) if !foundLocation { return nil, fmt.Errorf("unable to parse geo_distance location") @@ -89,6 +91,7 @@ func ParseSearchSortObj(input map[string]interface{}) (SearchSort, error) { if !ok { return nil, fmt.Errorf("search sort mode field must specify field") } + field = util.CleansePath(field) rv := &SortField{ Field: field, Desc: descending, @@ -156,7 +159,7 @@ func ParseSearchSortString(input string) SearchSort { } } return &SortField{ - Field: input, + Field: util.CleansePath(input), Desc: descending, } } diff --git a/search_test.go b/search_test.go index e4a880a9a..72fc1a157 100644 --- a/search_test.go +++ b/search_test.go @@ -480,7 +480,7 @@ func TestNestedBooleanSearchers(t *testing.T) { doc := document.NewDocument(strconv.Itoa(i)) doc.Fields = []document.Field{ - document.NewTextFieldCustom("hostname", []uint64{}, []byte(hostname), + document.NewTextFieldCustom("`hostname`", []uint64{}, []byte(hostname), index.IndexField, &analysis.DefaultAnalyzer{ Tokenizer: single.NewSingleTokenTokenizer(), @@ -492,7 +492,7 @@ func TestNestedBooleanSearchers(t *testing.T) { } for k, v := range metadata { doc.AddField(document.NewTextFieldWithIndexingOptions( - fmt.Sprintf("metadata.%s", k), []uint64{}, []byte(v), index.IndexField)) + fmt.Sprintf("`metadata`.`%s`", k), []uint64{}, []byte(v), index.IndexField)) } doc.CompositeFields = []*document.CompositeField{ document.NewCompositeFieldWithIndexingOptions( @@ -640,9 +640,9 @@ func TestNestedBooleanMustNotSearcherUpsidedown(t *testing.T) { for i := 0; i < len(docs); i++ { doc := document.NewDocument(docs[i].id) doc.Fields = []document.Field{ - document.NewTextField("id", []uint64{}, []byte(docs[i].id)), - document.NewBooleanField("hasRole", []uint64{}, docs[i].hasRole), - document.NewTextField("investigationId", []uint64{}, []byte(docs[i].investigationId)), + document.NewTextField("`id`", []uint64{}, []byte(docs[i].id)), + document.NewBooleanField("`hasRole`", []uint64{}, docs[i].hasRole), + document.NewTextField("`investigationId`", []uint64{}, []byte(docs[i].investigationId)), } doc.CompositeFields = []*document.CompositeField{ @@ -776,10 +776,10 @@ func TestMultipleNestedBooleanMustNotSearchersOnScorch(t *testing.T) { doc := document.NewDocument("1-child-0") doc.Fields = []document.Field{ - document.NewTextField("id", []uint64{}, []byte("1-child-0")), - document.NewBooleanField("hasRole", []uint64{}, false), - document.NewTextField("roles", []uint64{}, []byte("R1")), - document.NewNumericField("type", []uint64{}, 0), + document.NewTextField("`id`", []uint64{}, []byte("1-child-0")), + document.NewBooleanField("`hasRole`", []uint64{}, false), + document.NewTextField("`roles`", []uint64{}, []byte("R1")), + document.NewNumericField("`type`", []uint64{}, 0), } doc.CompositeFields = []*document.CompositeField{ document.NewCompositeFieldWithIndexingOptions( @@ -821,9 +821,9 @@ func TestMultipleNestedBooleanMustNotSearchersOnScorch(t *testing.T) { for i := 0; i < len(docs); i++ { doc := document.NewDocument(docs[i].id) doc.Fields = []document.Field{ - document.NewTextField("id", []uint64{}, []byte(docs[i].id)), - document.NewBooleanField("hasRole", []uint64{}, docs[i].hasRole), - document.NewNumericField("type", []uint64{}, float64(docs[i].typ)), + document.NewTextField("`id`", []uint64{}, []byte(docs[i].id)), + document.NewBooleanField("`hasRole`", []uint64{}, docs[i].hasRole), + document.NewNumericField("`type`", []uint64{}, float64(docs[i].typ)), } doc.CompositeFields = []*document.CompositeField{ @@ -846,9 +846,9 @@ func TestMultipleNestedBooleanMustNotSearchersOnScorch(t *testing.T) { // Update 1st doc doc = document.NewDocument("1-child-0") doc.Fields = []document.Field{ - document.NewTextField("id", []uint64{}, []byte("1-child-0")), - document.NewBooleanField("hasRole", []uint64{}, false), - document.NewNumericField("type", []uint64{}, 0), + document.NewTextField("`id`", []uint64{}, []byte("1-child-0")), + document.NewBooleanField("`hasRole`", []uint64{}, false), + document.NewNumericField("`type`", []uint64{}, 0), } doc.CompositeFields = []*document.CompositeField{ document.NewCompositeFieldWithIndexingOptions( @@ -1254,7 +1254,7 @@ func TestDuplicateLocationsIssue1168(t *testing.T) { if err != nil { t.Fatalf("bleve search err: %v", err) } - if len(sres.Hits[0].Locations["name1"]["marty"]) != 1 { + if len(sres.Hits[0].Locations["`name1`"]["marty"]) != 1 { t.Fatalf("duplicate marty") } } @@ -1866,7 +1866,7 @@ func TestHightlightingWithHTMLCharacterFilter(t *testing.T) { } if len(searchResults.Hits) != 1 || - len(searchResults.Hits[0].Locations["content"][searchStr]) != 1 { + len(searchResults.Hits[0].Locations["`content`"][searchStr]) != 1 { t.Fatalf("Expected 1 hit with 1 location") } @@ -1877,8 +1877,8 @@ func TestHightlightingWithHTMLCharacterFilter(t *testing.T) { } expectedFragment := "<div> Welcome to blevesearch. </div>" - gotLocation := searchResults.Hits[0].Locations["content"]["blevesearch"][0] - gotFragment := searchResults.Hits[0].Fragments["content"][0] + gotLocation := searchResults.Hits[0].Locations["`content`"]["blevesearch"][0] + gotFragment := searchResults.Hits[0].Fragments["`content`"][0] if !reflect.DeepEqual(expectedLocation, gotLocation) { t.Fatalf("Mismatch in locations, got: %v, expected: %v", diff --git a/test/tests/basic/searches.json b/test/tests/basic/searches.json index 7ddfce375..7678e4ae5 100644 --- a/test/tests/basic/searches.json +++ b/test/tests/basic/searches.json @@ -338,7 +338,7 @@ { "id": "a", "fields": { - "tags": ["gopher", "belieber"] + "`tags`": ["gopher", "belieber"] } } ] @@ -385,7 +385,7 @@ { "id": "b", "fragments": { - "name": ["steve has <a> long & complicated name"] + "`name`": ["steve has <a> long & complicated name"] } } ] @@ -409,7 +409,7 @@ { "id": "b", "fragments": { - "name": ["steve has <a> long & complicated name"] + "`name`": ["steve has <a> long & complicated name"] } } ] @@ -433,8 +433,8 @@ { "id": "b", "fields": { - "age": 27, - "birthday": "2001-09-09T01:46:40Z" + "`age`": 27, + "`birthday`": "2001-09-09T01:46:40Z" } } ] @@ -485,8 +485,8 @@ { "id": "b", "fragments": { - "name": ["steve has <a> long & complicated name"], - "title": ["missess"] + "`name`": ["steve has <a> long & complicated name"], + "`title`": ["missess"] } } ] @@ -512,7 +512,7 @@ { "id": "a", "fragments": { - "tags": ["gopher"] + "`tags`": ["gopher"] } } ] diff --git a/test/tests/employee/searches.json b/test/tests/employee/searches.json index d4db280b5..5e64cca60 100644 --- a/test/tests/employee/searches.json +++ b/test/tests/employee/searches.json @@ -17,7 +17,7 @@ { "id": "emp10508560", "locations": { - "manages.reports": { + "`manages`.`reports`": { "julián": [ { "pos": 2, @@ -38,4 +38,4 @@ ] } } -] \ No newline at end of file +] diff --git a/test/tests/facet/searches.json b/test/tests/facet/searches.json index 6752282a4..0d3bbe352 100644 --- a/test/tests/facet/searches.json +++ b/test/tests/facet/searches.json @@ -19,7 +19,7 @@ "hits": [], "facets": { "types": { - "field": "type", + "field": "`type`", "total": 10, "missing": 0, "other": 0, @@ -71,7 +71,7 @@ "hits": [], "facets": { "types": { - "field": "rating", + "field": "`rating`", "total": 10, "missing": 0, "other": 0, @@ -121,7 +121,7 @@ "hits": [], "facets": { "types": { - "field": "updated", + "field": "`updated`", "total": 10, "missing": 0, "other": 0, @@ -141,4 +141,4 @@ } } } -] \ No newline at end of file +] diff --git a/mapping/reflect.go b/util/reflect.go similarity index 50% rename from mapping/reflect.go rename to util/reflect.go index 6500a7059..f251d6f06 100644 --- a/mapping/reflect.go +++ b/util/reflect.go @@ -12,15 +12,81 @@ // See the License for the specific language governing permissions and // limitations under the License. -package mapping +package util import ( "reflect" "strings" ) +func LookupPropertyPathStr(data interface{}, path string) (string, bool) { + return mustString(lookupPropertyPath(data, path)) +} + +// ParseTagName extracts the field name from a struct tag +func ParseTagName(tag string) string { + if idx := strings.Index(tag, ","); idx != -1 { + return tag[:idx] + } + return tag +} + +// DecodePath splits a path into its parts +// For example: +// (1) a.b.c will be split into a, b and c +// (2) a.`b.c` will be split into a and b.c +// (3) `a.b`.c will be split into a.b and c +// (4) `a`.`b`.c will be split into a, b and c +func DecodePath(path string) []string { + var parts []string + var start int + var inQuote bool + for i, c := range path { + if c == '`' { + inQuote = !inQuote + } else if c == '.' && !inQuote { + parts = append(parts, stripEnclosingBackticks(path[start:i])) + start = i + 1 + } + } + parts = append(parts, stripEnclosingBackticks(path[start:])) + return parts +} + +// EncodePath concats a list of strings into a path +// by individually enclosing each string in backticks +// and separating them with a dot. +func EncodePath(pathElements []string) string { + var rv string + for i := 0; i < len(pathElements); i++ { + rv += encloseInBackticks(pathElements[i]) + if i < len(pathElements)-1 { + rv += pathSeparator + } + } + + return rv +} + +var internalFields = map[string]bool{ + "_all": true, + "_id": true, + "_score": true, +} + +// CleansePath cleanses a path by decoding and re-encoding it +// to make sure it is in the right format. +func CleansePath(path string) string { + if len(path) == 0 || internalFields[path] { + return path + } + return EncodePath(DecodePath(path)) +} + +// ----------------------------------------------------------------------------- + func lookupPropertyPath(data interface{}, path string) interface{} { - pathParts := decodePath(path) + pathParts := DecodePath(path) current := data for _, part := range pathParts { @@ -65,12 +131,15 @@ func lookupPropertyPathPart(data interface{}, part string) interface{} { const pathSeparator = "." -func decodePath(path string) []string { - return strings.Split(path, pathSeparator) +func encloseInBackticks(s string) string { + return "`" + s + "`" } -func encodePath(pathElements []string) string { - return strings.Join(pathElements, pathSeparator) +func stripEnclosingBackticks(s string) string { + if len(s) > 1 && s[0] == '`' && s[len(s)-1] == '`' { + return s[1 : len(s)-1] + } + return s } func mustString(data interface{}) (string, bool) { @@ -82,11 +151,3 @@ func mustString(data interface{}) (string, bool) { } return "", false } - -// parseTagName extracts the field name from a struct tag -func parseTagName(tag string) string { - if idx := strings.Index(tag, ","); idx != -1 { - return tag[:idx] - } - return tag -} diff --git a/mapping/reflect_test.go b/util/reflect_test.go similarity index 97% rename from mapping/reflect_test.go rename to util/reflect_test.go index fdba06f88..d77dcc02a 100644 --- a/mapping/reflect_test.go +++ b/util/reflect_test.go @@ -1,4 +1,4 @@ -package mapping +package util import ( "reflect" From c4245c8eca39ea5685f0cf9f007a49b019e38a4d Mon Sep 17 00:00:00 2001 From: Abhinav Dangeti Date: Fri, 24 Feb 2023 15:24:23 -0700 Subject: [PATCH 2/2] MB-55699: Unit test demo-ing new behavior --- search_test.go | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/search_test.go b/search_test.go index 72fc1a157..61d48551b 100644 --- a/search_test.go +++ b/search_test.go @@ -2100,3 +2100,56 @@ func TestGeoShapePolygonContainsPoint(t *testing.T) { } } } + +func TestMB55699(t *testing.T) { + // Unit test to demonstrate capability to differentiate between + // a nested field name and one that has a "." within it. + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + idx, err := New(tmpIndexPath, NewIndexMapping()) + if err != nil { + t.Fatal(err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + docBytes := []byte(` + { + "x": { + "y": "1" + }, + "x.y": "2" + } + `) + var doc map[string]interface{} + if err = json.Unmarshal(docBytes, &doc); err != nil { + t.Fatal(err) + } + + if err = idx.Index("doc", doc); err != nil { + t.Fatal(err) + } + + q1 := query.NewMatchQuery("1") + q1.SetField("x.y") + if res, err := idx.Search(NewSearchRequest(q1)); err != nil || len(res.Hits) != 1 { + t.Fatalf("Expected x.y to contain 1") + } + q1.SetField("`x`.`y`") + if res, err := idx.Search(NewSearchRequest(q1)); err != nil || len(res.Hits) != 1 { + t.Fatalf("Expected `x`.`y` to contain 1") + } + + q2 := query.NewMatchQuery("2") + q2.SetField("x.y") + if res, err := idx.Search(NewSearchRequest(q2)); err != nil || len(res.Hits) != 0 { + t.Fatalf("Expected `x`.`y` to not contain 2") + } + q2.SetField("`x.y`") + if res, err := idx.Search(NewSearchRequest(q2)); err != nil || len(res.Hits) != 1 { + t.Fatalf("Expected `x.y` to contain 2") + } +}