Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SOLR-17575: Fixed broken backwards compatibility with the legacy "langid.whitelist" config in Solr Langid #2886

Merged
Original file line number Diff line number Diff line change
Expand Up @@ -111,19 +111,20 @@ private void initParams(SolrParams params) {
overwrite = params.getBool(OVERWRITE, false);
langAllowlist = new HashSet<>();
threshold = params.getDouble(THRESHOLD, DOCID_THRESHOLD_DEFAULT);
String legacyAllowList = params.get(LANG_WHITELIST, "");
if (legacyAllowList.length() > 0) {
final String legacyAllowList = params.get(LANG_WHITELIST, "").trim();
if (!legacyAllowList.isEmpty()) {
// nowarn compile time string concatenation
log.warn(
LANG_WHITELIST
+ " parameter is deprecated; use "
+ LANG_ALLOWLIST
+ " instead."); // nowarn
}
if (params.get(LANG_ALLOWLIST, legacyAllowList).length() > 0) {
for (String lang : params.get(LANG_ALLOWLIST, "").split(",")) {
langAllowlist.add(lang);
}
if (!params.get(LANG_ALLOWLIST, legacyAllowList).isEmpty()) {
janhoy marked this conversation as resolved.
Show resolved Hide resolved
Arrays.stream(params.get(LANG_ALLOWLIST, legacyAllowList).split(","))
.map(String::trim)
.filter(lang -> !lang.isEmpty())
.forEach(langAllowlist::add);
}

// Mapping params (field centric)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
Expand Down Expand Up @@ -464,6 +465,31 @@ public void testMapIndividual() throws Exception {
assertTrue(mappedIndividual.containsKey("text2_ru"));
}

@Test
public void testAllowlist() throws Exception {
ModifiableSolrParams parameters = new ModifiableSolrParams();
parameters.add("langid.fl", "name,subject");
parameters.add("langid.langField", "language_s");
parameters.add("langid.allowlist", "no,en ,,,sv");
liProcessor = createLangIdProcessor(parameters);

// Make sure that empty language codes have been filtered out and others trimmed.
janhoy marked this conversation as resolved.
Show resolved Hide resolved
assertEquals(Set.of("no", "en", "sv"), liProcessor.langAllowlist);
}

@Test
public void testAllowlistBackwardsCompatabilityWithLegacyAllowlist() throws Exception {
// The "legacy allowlist" is "langid.whitelist"
ModifiableSolrParams parameters = new ModifiableSolrParams();
parameters.add("langid.fl", "name,subject");
parameters.add("langid.langField", "language_s");
parameters.add("langid.whitelist", "no,en ,,,sv");
liProcessor = createLangIdProcessor(parameters);

// Make sure that empty language codes have been filtered out and others trimmed.
assertEquals(Set.of("no", "en", "sv"), liProcessor.langAllowlist);
}

// Various utility methods

private SolrInputDocument englishDoc() {
Expand Down
Loading