Handle missing and multiple values in script

docs/painless/painless-getting-started.asciidoc

@@ -68,6 +68,7 @@ GET hockey/_search
 ----------------------------------------------------------------
 // CONSOLE
 
+
 Alternatively, you could do the same thing using a script field instead of a function score:
 
 [source,js]
@@ -119,6 +120,27 @@ GET hockey/_search
 ----------------------------------------------------------------
 // CONSOLE
 
+
+[float]
+===== Missing values
+
+If you request the value from a field `field` that isn’t in
+the document, `doc['field'].value` for this document returns:
+
+- `0` if a `field` has a numeric datatype (long, double etc.)
+- `false` is a `field` has a boolean datatype
+- epoch date if a `field` has a date datatype
+- `null` if a `field` has a string datatype
+- `null` if a `field` has a geo datatype
+- `""` if a `field` has a binary datatype
+
+IMPORTANT: Starting in 7.0, `doc['field']` returns a value of null if
+the field is missing in a document. To enable this behavior now,
+set a <<jvm-options,`jvm.option`>>
+`-Des.script.null_for_missing_value=true` on a node. If you do not enable
+this behavior, a deprecation warning is logged on start up.
+
+
 [float]
 ==== Updating Fields with Painless
 
@@ -357,6 +379,16 @@ Note: all of the `_update_by_query` examples above could really do with a
 as using any other query because script queries aren't able to use the inverted
 index to limit the documents that they have to check.
 
+[[multi-value-field-operations]]
+=== Multi-value field operations
+
+Painless supports the following operations for multi-value numeric fields:
+
+- `doc['field'].min` - gets the minimum value
+- `doc['field'].max` - gets the maximum value
+- `doc['field'].sum` - gets the sum of the values
+- `doc['field'].avg` - gets the average of the values
+
 [[modules-scripting-painless-dispatch]]
 === How painless dispatches functions
 

modules/lang-painless/build.gradle

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-import org.apache.tools.ant.types.Path
+import org.elasticsearch.gradle.test.RestIntegTestTask
 
 esplugin {
   description 'An easy, safe and fast scripting language for Elasticsearch'
@@ -28,6 +28,15 @@ integTestCluster {
   module project.project(':modules:mapper-extras')
 }
 
+Task additionalClusterTest = tasks.create(
+        name: "additionalClusterTest", type: RestIntegTestTask)
+additionalClusterTestCluster {
+  systemProperty 'es.script.null_for_missing_value', 'true'
+  distribution = 'integ-test-zip'
+  module project // add the lang-painless module itself
+  module project.project(':modules:mapper-extras')
+}
+
 dependencies {
   compile 'org.antlr:antlr4-runtime:4.5.3'
   compile 'org.ow2.asm:asm-debug-all:5.1'

modules/lang-painless/src/main/resources/org/elasticsearch/painless/spi/org.elasticsearch.txt

@@ -73,6 +73,10 @@ class org.elasticsearch.index.fielddata.ScriptDocValues$Strings {
 class org.elasticsearch.index.fielddata.ScriptDocValues$Longs {
   Long get(int)
   long getValue()
+  long getMin()
+  long getMax()
+  long getSum()
+  double getAvg()
   List getValues()
 }
 
@@ -85,6 +89,10 @@ class org.elasticsearch.index.fielddata.ScriptDocValues$Dates {
 class org.elasticsearch.index.fielddata.ScriptDocValues$Doubles {
   Double get(int)
   double getValue()
+  double getMin()
+  double getMax()
+  double getSum()
+  double getAvg()
   List getValues()
 }
 

modules/lang-painless/src/test/resources/rest-api-spec/test/painless/20_scriptfield.yml

@@ -95,7 +95,7 @@ setup:
                 script_fields:
                     bar:
                         script:
-                            source: "(doc['missing'].value?.length() ?: 0) + params.x;"
+                            source: "(doc['missing']?.value?.length() ?: 0) + params.x;"
                             params:
                                 x: 5
 

modules/lang-painless/src/test/resources/rest-api-spec/test/painless/70_missing_and_multiple_values.yml

@@ -0,0 +1,117 @@
+setup:
+    - do:
+        indices.create:
+            index: test
+            body:
+                settings:
+                    number_of_shards: 1
+                mappings:
+                    _doc:
+                        properties:
+                            dval:
+                                type: double
+                            lval:
+                                type: long
+
+    - do:
+        index:
+            index:  test
+            type:   _doc
+            id:     1
+            body:   { "dval": 5.5, "lval": 5 }
+
+    - do:
+        index:
+            index:  test
+            type:   _doc
+            id:     2
+            body:   { "dval": [5.5, 3.5, 4.5] }
+
+
+    - do:
+        index:
+            index:  test
+            type:   _doc
+            id:     3
+            body:   { "lval": [5, 3, 4] }
+
+    - do:
+        indices.refresh: {}
+
+---
+"check double and long values: missing values and operations on multiple values":
+    - skip:
+        version: " - 6.99.99"
+        reason: Handling missing values and operations on multiple values were added after these versions
+
+    - do:
+        search:
+            body:
+                script_fields:
+                    val_dval:
+                        script:
+                            source: "doc['dval']?.value ?: 0"
+                    min_dval:
+                        script:
+                            source: "doc['dval']?.min ?: 0"
+                    max_dval:
+                        script:
+                            source: "doc['dval']?.max ?: 0"
+                    sum_dval:
+                        script:
+                            source: "doc['dval']?.sum ?: 0"
+                    avg_dval:
+                        script:
+                            source: "doc['dval']?.avg ?: 0"
+                    val_lval:
+                        script:
+                            source: "doc['lval']?.value ?: 0"
+                    min_lval:
+                        script:
+                            source: "doc['lval']?.min ?: 0"
+                    max_lval:
+                        script:
+                            source: "doc['lval']?.max ?: 0"
+                    sum_lval:
+                        script:
+                            source: "doc['lval']?.sum ?: 0"
+                    avg_lval:
+                        script:
+                            source: "doc['lval']?.avg ?: 0"
+
+    - match: { hits.hits.0.fields.val_dval.0: 5.5}
+    - match: { hits.hits.0.fields.min_dval.0: 5.5}
+    - match: { hits.hits.0.fields.max_dval.0: 5.5}
+    - match: { hits.hits.0.fields.sum_dval.0: 5.5}
+    - match: { hits.hits.0.fields.avg_dval.0: 5.5}
+
+    - match: { hits.hits.0.fields.val_lval.0: 5}
+    - match: { hits.hits.0.fields.min_lval.0: 5}
+    - match: { hits.hits.0.fields.max_lval.0: 5}
+    - match: { hits.hits.0.fields.sum_lval.0: 5}
+    - match: { hits.hits.0.fields.avg_lval.0: 5}
+
+    - match: { hits.hits.1.fields.val_dval.0: 3.5}
+    - match: { hits.hits.1.fields.min_dval.0: 3.5}
+    - match: { hits.hits.1.fields.max_dval.0: 5.5}
+    - match: { hits.hits.1.fields.sum_dval.0: 13.5}
+    - match: { hits.hits.1.fields.avg_dval.0: 4.5}
+
+    - match: { hits.hits.1.fields.val_lval.0: 0}
+    - match: { hits.hits.1.fields.min_lval.0: 0}
+    - match: { hits.hits.1.fields.max_lval.0: 0}
+    - match: { hits.hits.1.fields.sum_lval.0: 0}
+    - match: { hits.hits.1.fields.avg_lval.0: 0}
+
+    - match: { hits.hits.2.fields.val_dval.0: 0}
+    - match: { hits.hits.2.fields.min_dval.0: 0}
+    - match: { hits.hits.2.fields.max_dval.0: 0}
+    - match: { hits.hits.2.fields.sum_dval.0: 0}
+    - match: { hits.hits.2.fields.avg_dval.0: 0}
+
+    - match: { hits.hits.2.fields.val_lval.0: 3}
+    - match: { hits.hits.2.fields.min_lval.0: 3}
+    - match: { hits.hits.2.fields.max_lval.0: 5}
+    - match: { hits.hits.2.fields.sum_lval.0: 12}
+    - match: { hits.hits.2.fields.avg_lval.0: 4}
+

modules/parent-join/build.gradle

@@ -22,3 +22,6 @@ esplugin {
   classname 'org.elasticsearch.join.ParentJoinPlugin'
   hasClientJar = true
 }
+test.configure {
+  systemProperty 'es.script.null_for_missing_value', 'true'
+}

modules/percolator/build.gradle

@@ -36,3 +36,7 @@ dependencyLicenses {
 
 compileJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes"
 compileTestJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes"
+
+test.configure {
+    systemProperty 'es.script.null_for_missing_value', 'true'
+}

server/build.gradle

@@ -338,3 +338,7 @@ if (isEclipse == false || project.path == ":server-tests") {
   check.dependsOn integTest
   integTest.mustRunAfter test
 }
+
+test.configure {
+    systemProperty 'es.script.null_for_missing_value', 'true'
+}