Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Upgrade to latest Lucene snapshot #33505

Merged
merged 10 commits into from
Sep 10, 2018
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion buildSrc/version.properties
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
elasticsearch = 7.0.0-alpha1
lucene = 8.0.0-snapshot-4d78db26be
lucene = 8.0.0-snapshot-66c671ea80

# optional dependencies
spatial4j = 0.7
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
58b9db095c569b4c4da491810f14e1429878b594
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.DoubleConstValueSource;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Scorable;
import org.apache.lucene.search.SortField;
import org.elasticsearch.SpecialPermission;
import org.elasticsearch.common.Nullable;
Expand Down Expand Up @@ -336,7 +336,7 @@ public void setDocument(int docid) {
}

@Override
public void setScorer(Scorer scorer) {
public void setScorer(Scorable scorer) {
script.setScorer(scorer);
}

Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
f009ee188453aabae77fad55aea08bc60323bb3e

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
af3d2ae975e3560c1ea69222d6c46072857952ba

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
f17bc5e532d9dc2786a13bd577df64023d1baae1

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
7ad89d33c1cd960c91afa05b22024137fe108567

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3f11fb254256d74e911b953994b47e7a95915954

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
b2348d140ef0c3e674cb81173f61c5e5f430facb

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
485a0c3be58a5942b4a28639f1019181ef4cd0e3

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
a22f1c6749ca4a3fbc9b330161a8ea3301cac8de

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
41ce415b93d75662cc2e790d09120bc0234d6b1b

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
06c1e4fa838807059d27aaf5405cfdfe7303369c

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
5b0a019a938deb58160647e7640b348bb99c10a8

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
4d813f3ba0ddd56bac728edb88ed8875e6acfd18

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
00c7e20b6a35ebecc875dd52bfb324967c5555d6

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
e4dbff54a0befdc7d67c0f39890586c220df718e

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
74d17f6bdf1fa4d499f02904432aa3b1024bde88

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
bec78be38f777765146c35f65e247909563d6814

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
74b76f8fed44400bc2a5d938ca2611a97b4d7a7c

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
2f65fa728b3bc924db6538f4c3caf2fcd25451cf

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
916a91f0cab2d3684707c59e9adca7b3030b2c66

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
eb3e630d6013e41838fb277943ce921f256f1c61

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
fa10ff14eab2f579cff2f0fa33c9c7f3b24daf12

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3dd65ca6612b4f98530847b99ab348fd83055fdf
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@
package org.apache.lucene.search.grouping;

import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.Scorable;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TotalHits;
Expand All @@ -44,7 +44,7 @@ public final class CollapsingTopDocsCollector<T> extends FirstPassGroupingCollec
protected final String collapseField;

protected final Sort sort;
protected Scorer scorer;
protected Scorable scorer;

private int totalHitCount;

Expand Down Expand Up @@ -102,7 +102,7 @@ public ScoreMode scoreMode() {
}

@Override
public void setScorer(Scorer scorer) throws IOException {
public void setScorer(Scorable scorer) throws IOException {
super.setScorer(scorer);
this.scorer = scorer;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@

package org.elasticsearch.action.search;

import org.apache.lucene.search.Scorable;
import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.SimpleCollector;

import java.io.IOException;
Expand All @@ -30,12 +30,12 @@
*/
public class MaxScoreCollector extends SimpleCollector {

private Scorer scorer;
private Scorable scorer;
private float maxScore = Float.NEGATIVE_INFINITY;
private boolean hasHits = false;

@Override
public void setScorer(Scorer scorer) {
public void setScorer(Scorable scorer) {
this.scorer = scorer;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,9 @@
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.LeafCollector;
import org.apache.lucene.search.Scorable;
import org.apache.lucene.search.ScoreCachingWrappingScorer;
import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.SimpleCollector;

import java.io.IOException;
Expand All @@ -34,7 +34,7 @@ public class MinimumScoreCollector extends SimpleCollector {
private final Collector collector;
private final float minimumScore;

private Scorer scorer;
private Scorable scorer;
private LeafCollector leafCollector;

public MinimumScoreCollector(Collector collector, float minimumScore) {
Expand All @@ -43,7 +43,7 @@ public MinimumScoreCollector(Collector collector, float minimumScore) {
}

@Override
public void setScorer(Scorer scorer) throws IOException {
public void setScorer(Scorable scorer) throws IOException {
if (!(scorer instanceof ScoreCachingWrappingScorer)) {
scorer = new ScoreCachingWrappingScorer(scorer);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@
*/
package org.elasticsearch.common.lucene;

import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Scorable;

public interface ScorerAware {

void setScorer(Scorer scorer);
void setScorer(Scorable scorer);

}
Original file line number Diff line number Diff line change
Expand Up @@ -19,28 +19,24 @@

package org.elasticsearch.common.lucene.search.function;

import java.io.IOException;

import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.ScoreCachingWrappingScorer;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.TwoPhaseIterator;
import org.apache.lucene.search.Weight;

import java.io.IOException;

/** A {@link Scorer} that filters out documents that have a score that is
* lower than a configured constant. */
final class MinScoreScorer extends Scorer {

private final Scorer in;
private final float minScore;

private float curScore;

MinScoreScorer(Weight weight, Scorer scorer, float minScore) {
super(weight);
if (scorer instanceof ScoreCachingWrappingScorer == false) {
// when minScore is set, scores might be requested twice: once
// to verify the match, and once by the collector
scorer = new ScoreCachingWrappingScorer(scorer);
}
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ScoreCachingWrapperScorer is now a Scorable, so this no longer works. Instead I've just duplicated the caching logic here. Given that SCWS is only ever applied by Collectors, and this is a real Scorer (so won't be generated by a Collector), we don't need to worry about double wrapping.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you probably need to fix the two-phase iterator to call this.score() rather than in.score() as well then, or otherwise it by-passes caching?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

actually maybe the two-phase iterator could just do curScore = in.score(); return curScore >= minScore; and then score() wouldn't even have to check the current doc ID

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

++

I also changed MinScoreScorerTests to check that score() only gets called once on the wrapped Scorable.

this.in = scorer;
this.minScore = minScore;
}
Expand All @@ -55,8 +51,8 @@ public int docID() {
}

@Override
public float score() throws IOException {
return in.score();
public float score() {
return curScore;
}

@Override
Expand Down Expand Up @@ -87,7 +83,8 @@ public boolean matches() throws IOException {
if (inTwoPhase != null && inTwoPhase.matches() == false) {
return false;
}
return in.score() >= minScore;
curScore = in.score();
return curScore >= minScore;
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Scorable;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.BitSet;
import org.apache.lucene.util.BytesRef;
Expand Down Expand Up @@ -71,7 +71,7 @@ protected SortedBinaryDocValues getValues(LeafReaderContext context) throws IOEx
return indexFieldData.load(context).getBytesValues();
}

protected void setScorer(Scorer scorer) {}
protected void setScorer(Scorable scorer) {}

@Override
public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) {
Expand Down Expand Up @@ -101,7 +101,7 @@ protected SortedDocValues getSortedDocValues(LeafReaderContext context, String f
}

@Override
public void setScorer(Scorer scorer) {
public void setScorer(Scorable scorer) {
BytesRefFieldComparatorSource.this.setScorer(scorer);
}

Expand All @@ -125,7 +125,7 @@ protected BinaryDocValues getBinaryDocValues(LeafReaderContext context, String f
}

@Override
public void setScorer(Scorer scorer) {
public void setScorer(Scorable scorer) {
BytesRefFieldComparatorSource.this.setScorer(scorer);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Scorable;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.BitSet;
import org.elasticsearch.common.Nullable;
Expand Down Expand Up @@ -57,7 +57,7 @@ protected SortedNumericDoubleValues getValues(LeafReaderContext context) throws
return indexFieldData.load(context).getDoubleValues();
}

protected void setScorer(Scorer scorer) {}
protected void setScorer(Scorable scorer) {}

@Override
public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) {
Expand All @@ -81,7 +81,7 @@ protected NumericDocValues getNumericDocValues(LeafReaderContext context, String
return selectedValues.getRawDoubleValues();
}
@Override
public void setScorer(Scorer scorer) {
public void setScorer(Scorable scorer) {
DoubleValuesComparatorSource.this.setScorer(scorer);
}
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

package org.elasticsearch.script;

import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Scorable;
import org.elasticsearch.search.lookup.DocLookup;

import java.io.IOException;
Expand All @@ -32,9 +32,9 @@
*/
public final class ScoreAccessor extends Number implements Comparable<Number> {

Scorer scorer;
Scorable scorer;

public ScoreAccessor(Scorer scorer) {
public ScoreAccessor(Scorable scorer) {
this.scorer = scorer;
}

Expand Down
Loading