From 4d5a16b5369b802b1cb6c922e71d9d7b3244eacf Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Thu, 15 Aug 2019 08:41:14 -0300 Subject: [PATCH 1/4] Remove unique filter from repo indexer analyzer. --- modules/indexer/repo.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/indexer/repo.go b/modules/indexer/repo.go index 787f9a9467ab..402f3aa06477 100644 --- a/modules/indexer/repo.go +++ b/modules/indexer/repo.go @@ -13,7 +13,6 @@ import ( "github.com/blevesearch/bleve" "github.com/blevesearch/bleve/analysis/analyzer/custom" "github.com/blevesearch/bleve/analysis/token/lowercase" - "github.com/blevesearch/bleve/analysis/token/unique" "github.com/blevesearch/bleve/analysis/tokenizer/unicode" "github.com/blevesearch/bleve/search/query" "github.com/ethantkoenig/rupture" @@ -110,7 +109,7 @@ func createRepoIndexer(path string, latestVersion int) error { "type": custom.Name, "char_filters": []string{}, "tokenizer": unicode.Name, - "token_filters": []string{unicodeNormalizeName, lowercase.Name, unique.Name}, + "token_filters": []string{unicodeNormalizeName, lowercase.Name}, }); err != nil { return err } From 74ee89999e41381784a78c5f0c7c3a946abda84e Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Thu, 15 Aug 2019 09:31:54 -0300 Subject: [PATCH 2/4] Bump repoIndexerLatestVersion to 4 --- modules/indexer/repo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexer/repo.go b/modules/indexer/repo.go index 402f3aa06477..91ed173aa769 100644 --- a/modules/indexer/repo.go +++ b/modules/indexer/repo.go @@ -22,7 +22,7 @@ const ( repoIndexerAnalyzer = "repoIndexerAnalyzer" repoIndexerDocType = "repoIndexerDocType" - repoIndexerLatestVersion = 2 + repoIndexerLatestVersion = 4 ) // repoIndexer (thread-safe) index for repository contents From 038de9e1bf37e7491e78c99f8d216d013d6c4728 Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Thu, 15 Aug 2019 09:38:49 -0300 Subject: [PATCH 3/4] Corrrect fmt --- modules/indexer/repo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexer/repo.go b/modules/indexer/repo.go index af7a6d8b253f..91ed173aa769 100644 --- a/modules/indexer/repo.go +++ b/modules/indexer/repo.go @@ -21,7 +21,7 @@ import ( const ( repoIndexerAnalyzer = "repoIndexerAnalyzer" repoIndexerDocType = "repoIndexerDocType" - + repoIndexerLatestVersion = 4 ) From bb74fa3faee72a15107b7cf1cb624eebab841576 Mon Sep 17 00:00:00 2001 From: Guillermo Prandi Date: Thu, 15 Aug 2019 09:42:24 -0300 Subject: [PATCH 4/4] make vendor to remove unique dependency --- .../bleve/analysis/token/unique/unique.go | 53 ------------------- vendor/modules.txt | 1 - 2 files changed, 54 deletions(-) delete mode 100644 vendor/github.com/blevesearch/bleve/analysis/token/unique/unique.go diff --git a/vendor/github.com/blevesearch/bleve/analysis/token/unique/unique.go b/vendor/github.com/blevesearch/bleve/analysis/token/unique/unique.go deleted file mode 100644 index c60e8c979387..000000000000 --- a/vendor/github.com/blevesearch/bleve/analysis/token/unique/unique.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) 2018 Couchbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package unique - -import ( - "github.com/blevesearch/bleve/analysis" - "github.com/blevesearch/bleve/registry" -) - -const Name = "unique" - -// UniqueTermFilter retains only the tokens which mark the first occurrence of -// a term. Tokens whose term appears in a preceding token are dropped. -type UniqueTermFilter struct{} - -func NewUniqueTermFilter() *UniqueTermFilter { - return &UniqueTermFilter{} -} - -func (f *UniqueTermFilter) Filter(input analysis.TokenStream) analysis.TokenStream { - encounteredTerms := make(map[string]struct{}, len(input)/4) - j := 0 - for _, token := range input { - term := string(token.Term) - if _, ok := encounteredTerms[term]; ok { - continue - } - encounteredTerms[term] = struct{}{} - input[j] = token - j++ - } - return input[:j] -} - -func UniqueTermFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) { - return NewUniqueTermFilter(), nil -} - -func init() { - registry.RegisterTokenFilter(Name, UniqueTermFilterConstructor) -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 703f161a8aab..0f1de7d856cc 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -26,7 +26,6 @@ github.com/blevesearch/bleve github.com/blevesearch/bleve/analysis/analyzer/custom github.com/blevesearch/bleve/analysis/token/lowercase github.com/blevesearch/bleve/analysis/token/unicodenorm -github.com/blevesearch/bleve/analysis/token/unique github.com/blevesearch/bleve/analysis/tokenizer/unicode github.com/blevesearch/bleve/index/upsidedown github.com/blevesearch/bleve/mapping