Skip to content

Commit

Permalink
Handle missing and multiple values in script
Browse files Browse the repository at this point in the history 
Previously in script for numeric fields, there was no way to check if
a document is missing a value. Also certain operations on multiple-
values fields were missing.

This PR adds the following:

- return null for doc['field'] if a document is missing a 'field1':
    Now we can do this:
    if (doc['field'] == null) {return -1;} return doc['field'].value;  or
    doc['field']?.value ?: -1

- add the following functions for multiple-valued numeric fields:
    doc['field'].min returns the minumum amoung values
    doc['field'].max returns the maximum amoung values
    doc['field'].sum returns the sum of amoung values
    doc['field'].avg returns the average of values

Closes #29286
  • Loading branch information
@mayya-sharipova
mayya-sharipova committed Apr 20, 2018
1 parent e2d770d commit 5aeab57
Show file tree
Hide file tree
Showing 11 changed files with 277 additions and 24 deletions.
16 changes: 16 additions & 0 deletions docs/painless/painless-getting-started.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ GET hockey/_search
----------------------------------------------------------------
// CONSOLE


Alternatively, you could do the same thing using a script field instead of a function score:

[source,js]
Expand Down Expand Up @@ -119,6 +120,21 @@ GET hockey/_search
----------------------------------------------------------------
// CONSOLE

[float]
===== Missing and multiple values

If a document is missing a field `field`, `doc['field']` for this document
will return null.

There is also a number of operations designed for numeric fields,
if a document has multiple values in such a field:

- `doc['field'].min` - gets the minimum value among values
- `doc['field'].max` - gets the maximum value among values
- `doc['field'].sum` - gets the sum of all values
- `doc['field'].avg` - gets the average of all values


[float]
==== Updating Fields with Painless

Expand Down
8 changes: 8 additions & 0 deletions ...les/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,10 @@ class org.elasticsearch.index.fielddata.ScriptDocValues$Strings {
class org.elasticsearch.index.fielddata.ScriptDocValues$Longs {
Long get(int)
long getValue()
long getMin()
long getMax()
long getSum()
double getAvg()
List getValues()
org.joda.time.ReadableDateTime getDate()
List getDates()
Expand All @@ -89,6 +93,10 @@ class org.elasticsearch.index.fielddata.ScriptDocValues$Dates {
class org.elasticsearch.index.fielddata.ScriptDocValues$Doubles {
Double get(int)
double getValue()
double getMin()
double getMax()
double getSum()
double getAvg()
List getValues()
}

Expand Down
2 changes: 1 addition & 1 deletion modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ setup:
script_fields:
bar:
script:
source: "(doc['missing'].value?.length() ?: 0) + params.x;"
source: "(doc['missing']?.value?.length() ?: 0) + params.x;"
params:
x: 5

Expand Down
117 changes: 117 additions & 0 deletions ...ainless/src/test/resources/rest-api-spec/test/painless/70_missing_and_multiple_values.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
setup:
- do:
indices.create:
index: test
body:
settings:
number_of_shards: 1
mappings:
_doc:
properties:
dval:
type: double
lval:
type: long

- do:
index:
index: test
type: _doc
id: 1
body: { "dval": 5.5, "lval": 5 }

- do:
index:
index: test
type: _doc
id: 2
body: { "dval": [5.5, 3.5, 4.5] }


- do:
index:
index: test
type: _doc
id: 3
body: { "lval": [5, 3, 4] }

- do:
indices.refresh: {}

---
"check double and long values: missing values and operations on multiple values":
- skip:
version: " - 6.99.99"
reason: Handling missing values and operations on multiple values were added after these versions

- do:
search:
body:
script_fields:
val_dval:
script:
source: "doc['dval']?.value ?: -1.0"
min_dval:
script:
source: "doc['dval']?.min ?: -1.0"
max_dval:
script:
source: "doc['dval']?.max ?: -1.0"
sum_dval:
script:
source: "doc['dval']?.sum ?: -1.0"
avg_dval:
script:
source: "doc['dval']?.avg ?: -1.0"
val_lval:
script:
source: "doc['lval']?.value ?: -1"
min_lval:
script:
source: "doc['lval']?.min ?: -1"
max_lval:
script:
source: "doc['lval']?.max ?: -1"
sum_lval:
script:
source: "doc['lval']?.sum ?: -1"
avg_lval:
script:
source: "doc['lval']?.avg ?: -1"

- match: { hits.hits.0.fields.val_dval.0: 5.5}
- match: { hits.hits.0.fields.min_dval.0: 5.5}
- match: { hits.hits.0.fields.max_dval.0: 5.5}
- match: { hits.hits.0.fields.sum_dval.0: 5.5}
- match: { hits.hits.0.fields.avg_dval.0: 5.5}

- match: { hits.hits.0.fields.val_lval.0: 5}
- match: { hits.hits.0.fields.min_lval.0: 5}
- match: { hits.hits.0.fields.max_lval.0: 5}
- match: { hits.hits.0.fields.sum_lval.0: 5}
- match: { hits.hits.0.fields.avg_lval.0: 5}

- match: { hits.hits.1.fields.val_dval.0: 3.5}
- match: { hits.hits.1.fields.min_dval.0: 3.5}
- match: { hits.hits.1.fields.max_dval.0: 5.5}
- match: { hits.hits.1.fields.sum_dval.0: 13.5}
- match: { hits.hits.1.fields.avg_dval.0: 4.5}

- match: { hits.hits.1.fields.val_lval.0: -1}
- match: { hits.hits.1.fields.min_lval.0: -1}
- match: { hits.hits.1.fields.max_lval.0: -1}
- match: { hits.hits.1.fields.sum_lval.0: -1}
- match: { hits.hits.1.fields.avg_lval.0: -1}

- match: { hits.hits.2.fields.val_dval.0: -1.0}
- match: { hits.hits.2.fields.min_dval.0: -1.0}
- match: { hits.hits.2.fields.max_dval.0: -1.0}
- match: { hits.hits.2.fields.sum_dval.0: -1.0}
- match: { hits.hits.2.fields.avg_dval.0: -1.0}

- match: { hits.hits.2.fields.val_lval.0: 3}
- match: { hits.hits.2.fields.min_lval.0: 3}
- match: { hits.hits.2.fields.max_lval.0: 5}
- match: { hits.hits.2.fields.sum_lval.0: 12}
- match: { hits.hits.2.fields.avg_lval.0: 4}

101 changes: 88 additions & 13 deletions server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ public abstract class ScriptDocValues<T> extends AbstractList<T> {
/**
* Set the current doc ID.
*/
public abstract void setNextDocId(int docId) throws IOException;
public abstract boolean setNextDocId(int docId) throws IOException;

/**
* Return a copy of the list of the values for the current document.
Expand Down Expand Up @@ -124,9 +124,10 @@ public Longs(SortedNumericDocValues in) {
}

@Override
public void setNextDocId(int docId) throws IOException {
public boolean setNextDocId(int docId) throws IOException {
this.docId = docId;
if (in.advanceExact(docId)) {
boolean docHasValues = in.advanceExact(docId);
if (docHasValues) {
resize(in.docValueCount());
for (int i = 0; i < count; i++) {
values[i] = in.nextValue();
Expand All @@ -137,6 +138,7 @@ public void setNextDocId(int docId) throws IOException {
if (dates != null) {
dates.setNextDocId(docId);
}
return docHasValues;
}

/**
Expand All @@ -159,6 +161,37 @@ public long getValue() {
return values[0];
}

public long getMin() {
if (count == 0) {
return 0L;
};
return values[0];
}

public long getMax() {
if (count == 0) {
return 0L;
};
return values[count - 1];
}

public long getSum() {
if (count == 0) {
return 0L;
};
long sum = 0L;
for (int i = 0; i < count; i++)
sum += values[i];
return sum;
}

public double getAvg() {
if (count == 0) {
return 0d;
};
return getSum() * 1.0/count;
}

@Deprecated
public ReadableDateTime getDate() throws IOException {
deprecated("getDate on numeric fields is deprecated. Use a date field to get dates.");
Expand Down Expand Up @@ -285,13 +318,15 @@ public int size() {
}

@Override
public void setNextDocId(int docId) throws IOException {
if (in.advanceExact(docId)) {
public boolean setNextDocId(int docId) throws IOException {
boolean docHasValues = in.advanceExact(docId);
if (docHasValues) {
count = in.docValueCount();
} else {
count = 0;
}
refreshArray();
return docHasValues;
}

/**
Expand Down Expand Up @@ -355,15 +390,17 @@ public Doubles(SortedNumericDoubleValues in) {
}

@Override
public void setNextDocId(int docId) throws IOException {
if (in.advanceExact(docId)) {
public boolean setNextDocId(int docId) throws IOException {
boolean docHasValues = in.advanceExact(docId);
if (docHasValues) {
resize(in.docValueCount());
for (int i = 0; i < count; i++) {
values[i] = in.nextValue();
}
} else {
resize(0);
}
return docHasValues;
}

/**
Expand All @@ -386,6 +423,38 @@ public double getValue() {
return values[0];
}

public double getMin() {
if (count == 0) {
return 0d;
};
return values[0];
}

public double getMax() {
if (count == 0) {
return 0d;
};
return values[count - 1];
}

public double getSum() {
if (count == 0) {
return 0d;
};
double sum = 0d;
for (int i = 0; i < count; i++)
sum += values[i];
return sum;
}

public double getAvg() {
if (count == 0) {
return 0d;
};
return getSum() / count;
}


@Override
public Double get(int index) {
return values[index];
Expand All @@ -408,8 +477,9 @@ public GeoPoints(MultiGeoPointValues in) {
}

@Override
public void setNextDocId(int docId) throws IOException {
if (in.advanceExact(docId)) {
public boolean setNextDocId(int docId) throws IOException {
boolean docHasValues = in.advanceExact(docId);
if (docHasValues) {
resize(in.docValueCount());
for (int i = 0; i < count; i++) {
GeoPoint point = in.nextValue();
Expand All @@ -418,6 +488,7 @@ public void setNextDocId(int docId) throws IOException {
} else {
resize(0);
}
return docHasValues;
}

/**
Expand Down Expand Up @@ -528,15 +599,17 @@ public Booleans(SortedNumericDocValues in) {
}

@Override
public void setNextDocId(int docId) throws IOException {
if (in.advanceExact(docId)) {
public boolean setNextDocId(int docId) throws IOException {
boolean docHasValues = in.advanceExact(docId);
if (docHasValues) {
resize(in.docValueCount());
for (int i = 0; i < count; i++) {
values[i] = in.nextValue() == 1;
}
} else {
resize(0);
}
return docHasValues;
}

/**
Expand Down Expand Up @@ -584,8 +657,9 @@ abstract static class BinaryScriptDocValues<T> extends ScriptDocValues<T> {
}

@Override
public void setNextDocId(int docId) throws IOException {
if (in.advanceExact(docId)) {
public boolean setNextDocId(int docId) throws IOException {
boolean docHasValues = in.advanceExact(docId);
if (docHasValues) {
resize(in.docValueCount());
for (int i = 0; i < count; i++) {
// We need to make a copy here, because BytesBinaryDVAtomicFieldData's SortedBinaryDocValues
Expand All @@ -596,6 +670,7 @@ public void setNextDocId(int docId) throws IOException {
} else {
resize(0);
}
return docHasValues;
}

/**
Expand Down
Loading

0 comments on commit 5aeab57

Please sign in to comment.