diff --git a/detectlang/README.md b/detectlang/README.md index 5037660..211ffed 100644 --- a/detectlang/README.md +++ b/detectlang/README.md @@ -2,7 +2,7 @@ ## detect_lang token filter -A bleve token filter which passes the text of each token and passes it to the cld2 library. The library determines what it thinks the language most likely is. The ISO-639 language code replaces the token term. +A bleve token filter which passes the text of each token and passes it to the [whatlanggo](https://github.com/abadojack/whatlanggo) library. The library determines what it thinks the language most likely is. The ISO-639 language code replaces the token term. ## detect_lang analyzer @@ -12,4 +12,4 @@ When you configure a field to use this analyzer, the effect will be to index a s ### Dependency -This language uses a [Go wrapper](https://github.com/blevesearch/cld2) around [CLD2](https://github.com/CLD2Owners/cld2). The wrapper includes the C++ source directly so previous complications around static vs dynamic linking have gone away. +This language uses a [whatlanggo](https://github.com/abadojack/whatlanggo). diff --git a/detectlang/detect_lang_filter.go b/detectlang/detect_lang_filter.go index bacd8ec..37dd73b 100644 --- a/detectlang/detect_lang_filter.go +++ b/detectlang/detect_lang_filter.go @@ -13,7 +13,7 @@ import ( "github.com/blevesearch/bleve/analysis" "github.com/blevesearch/bleve/registry" - "github.com/blevesearch/cld2" + "github.com/abadojack/whatlanggo" ) const FilterName = "detect_lang" @@ -30,7 +30,7 @@ func (f *DetectLangFilter) Filter(input analysis.TokenStream) analysis.TokenStre offset := 0 for _, token := range input { - token.Term = []byte(cld2.Detect(string(token.Term))) + token.Term = []byte(whatlanggo.Detect(string(token.Term)).Lang.Iso6391()) token.Start = offset token.End = token.Start + len(token.Term) token.Type = analysis.AlphaNumeric