-
Notifications
You must be signed in to change notification settings - Fork 24.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Added lenient flag for synonym token filter (#31484)
* Added lenient flag for synonym-tokenfilter. Relates to #30968 * added docs for synonym-graph-tokenfilter -- Also made lenient final -- changed from !lenient to lenient == false * Changes after review (1) -- Renamed to ElasticsearchSynonymParser -- Added explanation for ElasticsearchSynonymParser::add method -- Changed ElasticsearchSynonymParser::logger instance to static * Added lenient option for WordnetSynonymParser -- also added more documentation * Added additional documentation * Improved documentation
- Loading branch information
1 parent
3189ef4
commit 88c270d
Showing
8 changed files
with
400 additions
and
15 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
68 changes: 68 additions & 0 deletions
68
server/src/main/java/org/elasticsearch/index/analysis/ESSolrSynonymParser.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| /* | ||
| * Licensed to Elasticsearch under one or more contributor | ||
| * license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright | ||
| * ownership. Elasticsearch licenses this file to you under | ||
| * the Apache License, Version 2.0 (the "License"); you may | ||
| * not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
|
|
||
| package org.elasticsearch.index.analysis; | ||
|
|
||
| import org.apache.logging.log4j.Logger; | ||
| import org.apache.lucene.analysis.Analyzer; | ||
| import org.apache.lucene.analysis.synonym.SolrSynonymParser; | ||
| import org.apache.lucene.util.CharsRef; | ||
| import org.apache.lucene.util.CharsRefBuilder; | ||
| import org.elasticsearch.common.logging.Loggers; | ||
|
|
||
| import java.io.IOException; | ||
|
|
||
| public class ESSolrSynonymParser extends SolrSynonymParser { | ||
|
|
||
| private final boolean lenient; | ||
| private static final Logger logger = | ||
| Loggers.getLogger(ESSolrSynonymParser.class, "ESSolrSynonymParser"); | ||
|
|
||
| public ESSolrSynonymParser(boolean dedup, boolean expand, boolean lenient, Analyzer analyzer) { | ||
| super(dedup, expand, analyzer); | ||
| this.lenient = lenient; | ||
| } | ||
|
|
||
| @Override | ||
| public void add(CharsRef input, CharsRef output, boolean includeOrig) { | ||
| // This condition follows up on the overridden analyze method. In case lenient was set to true and there was an | ||
| // exception during super.analyze we return a zero-length CharsRef for that word which caused an exception. When | ||
| // the synonym mappings for the words are added using the add method we skip the ones that were left empty by | ||
| // analyze i.e., in the case when lenient is set we only add those combinations which are non-zero-length. The | ||
| // else would happen only in the case when the input or output is empty and lenient is set, in which case we | ||
| // quietly ignore it. For more details on the control-flow see SolrSynonymParser::addInternal. | ||
| if (lenient == false || (input.length > 0 && output.length > 0)) { | ||
| super.add(input, output, includeOrig); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public CharsRef analyze(String text, CharsRefBuilder reuse) throws IOException { | ||
| try { | ||
| return super.analyze(text, reuse); | ||
| } catch (IllegalArgumentException ex) { | ||
| if (lenient) { | ||
| logger.info("Synonym rule for [" + text + "] was ignored"); | ||
| return new CharsRef(""); | ||
| } else { | ||
| throw ex; | ||
| } | ||
| } | ||
| } | ||
| } |
68 changes: 68 additions & 0 deletions
68
server/src/main/java/org/elasticsearch/index/analysis/ESWordnetSynonymParser.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| /* | ||
| * Licensed to Elasticsearch under one or more contributor | ||
| * license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright | ||
| * ownership. Elasticsearch licenses this file to you under | ||
| * the Apache License, Version 2.0 (the "License"); you may | ||
| * not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
|
|
||
| package org.elasticsearch.index.analysis; | ||
|
|
||
| import org.apache.logging.log4j.Logger; | ||
| import org.apache.lucene.analysis.Analyzer; | ||
| import org.apache.lucene.analysis.synonym.WordnetSynonymParser; | ||
| import org.apache.lucene.util.CharsRef; | ||
| import org.apache.lucene.util.CharsRefBuilder; | ||
| import org.elasticsearch.common.logging.Loggers; | ||
|
|
||
| import java.io.IOException; | ||
|
|
||
| public class ESWordnetSynonymParser extends WordnetSynonymParser { | ||
|
|
||
| private final boolean lenient; | ||
| private static final Logger logger = | ||
| Loggers.getLogger(ESSolrSynonymParser.class, "ESWordnetSynonymParser"); | ||
|
|
||
| public ESWordnetSynonymParser(boolean dedup, boolean expand, boolean lenient, Analyzer analyzer) { | ||
| super(dedup, expand, analyzer); | ||
| this.lenient = lenient; | ||
| } | ||
|
|
||
| @Override | ||
| public void add(CharsRef input, CharsRef output, boolean includeOrig) { | ||
| // This condition follows up on the overridden analyze method. In case lenient was set to true and there was an | ||
| // exception during super.analyze we return a zero-length CharsRef for that word which caused an exception. When | ||
| // the synonym mappings for the words are added using the add method we skip the ones that were left empty by | ||
| // analyze i.e., in the case when lenient is set we only add those combinations which are non-zero-length. The | ||
| // else would happen only in the case when the input or output is empty and lenient is set, in which case we | ||
| // quietly ignore it. For more details on the control-flow see SolrSynonymParser::addInternal. | ||
| if (lenient == false || (input.length > 0 && output.length > 0)) { | ||
| super.add(input, output, includeOrig); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public CharsRef analyze(String text, CharsRefBuilder reuse) throws IOException { | ||
| try { | ||
| return super.analyze(text, reuse); | ||
| } catch (IllegalArgumentException ex) { | ||
| if (lenient) { | ||
| logger.info("Synonym rule for [" + text + "] was ignored"); | ||
| return new CharsRef(""); | ||
| } else { | ||
| throw ex; | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
78 changes: 78 additions & 0 deletions
78
server/src/test/java/org/elasticsearch/index/analysis/ESSolrSynonymParserTests.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| /* | ||
| * Licensed to Elasticsearch under one or more contributor | ||
| * license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright | ||
| * ownership. Elasticsearch licenses this file to you under | ||
| * the Apache License, Version 2.0 (the "License"); you may | ||
| * not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
|
|
||
| package org.elasticsearch.index.analysis; | ||
|
|
||
| import org.apache.lucene.analysis.CharArraySet; | ||
| import org.apache.lucene.analysis.StopFilter; | ||
| import org.apache.lucene.analysis.TokenStream; | ||
| import org.apache.lucene.analysis.Tokenizer; | ||
| import org.apache.lucene.analysis.standard.StandardAnalyzer; | ||
| import org.apache.lucene.analysis.standard.StandardTokenizer; | ||
| import org.apache.lucene.analysis.synonym.SynonymFilter; | ||
| import org.apache.lucene.analysis.synonym.SynonymMap; | ||
| import org.elasticsearch.test.ESTokenStreamTestCase; | ||
|
|
||
| import java.io.IOException; | ||
| import java.io.StringReader; | ||
| import java.text.ParseException; | ||
|
|
||
| import static org.hamcrest.Matchers.containsString; | ||
|
|
||
| public class ESSolrSynonymParserTests extends ESTokenStreamTestCase { | ||
|
|
||
| public void testLenientParser() throws IOException, ParseException { | ||
| ESSolrSynonymParser parser = new ESSolrSynonymParser(true, false, true, new StandardAnalyzer()); | ||
| String rules = | ||
| "&,and\n" + | ||
| "come,advance,approach\n"; | ||
| StringReader rulesReader = new StringReader(rules); | ||
| parser.parse(rulesReader); | ||
| SynonymMap synonymMap = parser.build(); | ||
| Tokenizer tokenizer = new StandardTokenizer(); | ||
| tokenizer.setReader(new StringReader("approach quietly then advance & destroy")); | ||
| TokenStream ts = new SynonymFilter(tokenizer, synonymMap, false); | ||
| assertTokenStreamContents(ts, new String[]{"come", "quietly", "then", "come", "destroy"}); | ||
| } | ||
|
|
||
| public void testLenientParserWithSomeIncorrectLines() throws IOException, ParseException { | ||
| CharArraySet stopSet = new CharArraySet(1, true); | ||
| stopSet.add("bar"); | ||
| ESSolrSynonymParser parser = | ||
| new ESSolrSynonymParser(true, false, true, new StandardAnalyzer(stopSet)); | ||
| String rules = "foo,bar,baz"; | ||
| StringReader rulesReader = new StringReader(rules); | ||
| parser.parse(rulesReader); | ||
| SynonymMap synonymMap = parser.build(); | ||
| Tokenizer tokenizer = new StandardTokenizer(); | ||
| tokenizer.setReader(new StringReader("first word is foo, then bar and lastly baz")); | ||
| TokenStream ts = new SynonymFilter(new StopFilter(tokenizer, stopSet), synonymMap, false); | ||
| assertTokenStreamContents(ts, new String[]{"first", "word", "is", "foo", "then", "and", "lastly", "foo"}); | ||
| } | ||
|
|
||
| public void testNonLenientParser() { | ||
| ESSolrSynonymParser parser = new ESSolrSynonymParser(true, false, false, new StandardAnalyzer()); | ||
| String rules = | ||
| "&,and=>and\n" + | ||
| "come,advance,approach\n"; | ||
| StringReader rulesReader = new StringReader(rules); | ||
| ParseException ex = expectThrows(ParseException.class, () -> parser.parse(rulesReader)); | ||
| assertThat(ex.getMessage(), containsString("Invalid synonym rule at line 1")); | ||
| } | ||
| } |
Oops, something went wrong.