From c0969601e93449645ed3fff48330664e477c35d5 Mon Sep 17 00:00:00 2001 From: Minoru Osuka Date: Thu, 9 Apr 2020 11:17:28 +0900 Subject: [PATCH 1/2] Replace cld2 to whatlanggo --- detectlang/README.md | 4 ++-- detectlang/detect_lang_filter.go | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/detectlang/README.md b/detectlang/README.md index 5037660..211ffed 100644 --- a/detectlang/README.md +++ b/detectlang/README.md @@ -2,7 +2,7 @@ ## detect_lang token filter -A bleve token filter which passes the text of each token and passes it to the cld2 library. The library determines what it thinks the language most likely is. The ISO-639 language code replaces the token term. +A bleve token filter which passes the text of each token and passes it to the [whatlanggo](https://github.com/abadojack/whatlanggo) library. The library determines what it thinks the language most likely is. The ISO-639 language code replaces the token term. ## detect_lang analyzer @@ -12,4 +12,4 @@ When you configure a field to use this analyzer, the effect will be to index a s ### Dependency -This language uses a [Go wrapper](https://github.com/blevesearch/cld2) around [CLD2](https://github.com/CLD2Owners/cld2). The wrapper includes the C++ source directly so previous complications around static vs dynamic linking have gone away. +This language uses a [whatlanggo](https://github.com/abadojack/whatlanggo). diff --git a/detectlang/detect_lang_filter.go b/detectlang/detect_lang_filter.go index bacd8ec..f6735a2 100644 --- a/detectlang/detect_lang_filter.go +++ b/detectlang/detect_lang_filter.go @@ -10,10 +10,9 @@ package detectlang import ( + "github.com/abadojack/whatlanggo" "github.com/blevesearch/bleve/analysis" "github.com/blevesearch/bleve/registry" - - "github.com/blevesearch/cld2" ) const FilterName = "detect_lang" @@ -30,7 +29,7 @@ func (f *DetectLangFilter) Filter(input analysis.TokenStream) analysis.TokenStre offset := 0 for _, token := range input { - token.Term = []byte(cld2.Detect(string(token.Term))) + token.Term = []byte(whatlanggo.Detect(string(token.Term)).Lang.Iso6391()) token.Start = offset token.End = token.Start + len(token.Term) token.Type = analysis.AlphaNumeric From 8d1869e159a95d2e63d3bb83ddb9b055e9d87ddf Mon Sep 17 00:00:00 2001 From: Minoru Osuka Date: Thu, 9 Apr 2020 11:27:10 +0900 Subject: [PATCH 2/2] Format --- detectlang/detect_lang_filter.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/detectlang/detect_lang_filter.go b/detectlang/detect_lang_filter.go index f6735a2..37dd73b 100644 --- a/detectlang/detect_lang_filter.go +++ b/detectlang/detect_lang_filter.go @@ -10,9 +10,10 @@ package detectlang import ( - "github.com/abadojack/whatlanggo" "github.com/blevesearch/bleve/analysis" "github.com/blevesearch/bleve/registry" + + "github.com/abadojack/whatlanggo" ) const FilterName = "detect_lang"