Merge branch '6.x' into ccr-6.x

* 6.x: 6d711fa (origin/6.x, 6.x) Uncouple persistent task state and status (#31031) f0f16b7 [TEST] Make SSL restrictions update atomic (#31050) 652193f Describe how to add a plugin in Dockerfile (#31340) bb568ab TEST: getCapturedRequestsAndClear should be atomic (#31312) 6f94914 Do not set vm.max_map_count when unnecessary (#31285) c12f3c7 Painless: Fix bug for static method calls on interfaces (#31348) 21128e2 QA: Fix resolution of default distribution (#31351) df17a83 Build: Fix the license in the pom zip and tar (#31336) 3e76b15 Treat ack timeout more like a publish timeout (#31303)
elastic · Jun 15, 2018 · 22d1df9 · 22d1df9
2 parents a4f5390 + 6d711fa
commit 22d1df9
Show file tree

Hide file tree

Showing 68 changed files with 759 additions and 469 deletions.
diff --git a/build.gradle b/build.gradle
@@ -43,9 +43,23 @@ subprojects {
   description = "Elasticsearch subproject ${project.path}"
 }
 
+apply plugin: 'nebula.info-scm'
+String licenseCommit
+if (VersionProperties.elasticsearch.toString().endsWith('-SNAPSHOT')) {
+  licenseCommit = scminfo.change ?: "master" // leniency for non git builds
+} else {
+  licenseCommit = "v${version}"
+}
+String elasticLicenseUrl = "https://raw.githubusercontent.com/elastic/elasticsearch/${licenseCommit}/licenses/ELASTIC-LICENSE.txt"
+
 subprojects {
+  // Default to the apache license
   project.ext.licenseName = 'The Apache Software License, Version 2.0'
   project.ext.licenseUrl = 'http://www.apache.org/licenses/LICENSE-2.0.txt'
+
+  // But stick the Elastic license url in project.ext so we can get it if we need to switch to it
+  project.ext.elasticLicenseUrl = elasticLicenseUrl
+
   // we only use maven publish to add tasks for pom generation
   plugins.withType(MavenPublishPlugin).whenPluginAdded {
     publishing {

diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle
@@ -228,6 +228,8 @@ subprojects {
   check.dependsOn checkNotice
 
   if (project.name == 'zip' || project.name == 'tar') {
+    project.ext.licenseName = 'Elastic License'
+    project.ext.licenseUrl = ext.elasticLicenseUrl
     task checkMlCppNotice {
       dependsOn buildDist, checkExtraction
       onlyIf toolExists

diff --git a/distribution/packages/src/deb/init.d/elasticsearch b/distribution/packages/src/deb/init.d/elasticsearch
@@ -122,7 +122,7 @@ case "$1" in
 		ulimit -l $MAX_LOCKED_MEMORY
 	fi
 
-	if [ -n "$MAX_MAP_COUNT" -a -f /proc/sys/vm/max_map_count ]; then
+        if [ -n "$MAX_MAP_COUNT" -a -f /proc/sys/vm/max_map_count -a "$MAX_MAP_COUNT" -ge $(cat /proc/sys/vm/max_map_count) ]; then
 		sysctl -q -w vm.max_map_count=$MAX_MAP_COUNT
 	fi
 

diff --git a/distribution/packages/src/rpm/init.d/elasticsearch b/distribution/packages/src/rpm/init.d/elasticsearch
@@ -90,7 +90,7 @@ start() {
     if [ -n "$MAX_LOCKED_MEMORY" ]; then
         ulimit -l $MAX_LOCKED_MEMORY
     fi
-    if [ -n "$MAX_MAP_COUNT" -a -f /proc/sys/vm/max_map_count ]; then
+    if [ -n "$MAX_MAP_COUNT" -a -f /proc/sys/vm/max_map_count -a "$MAX_MAP_COUNT" -ge $(cat /proc/sys/vm/max_map_count) ]; then
         sysctl -q -w vm.max_map_count=$MAX_MAP_COUNT
     fi
 

diff --git a/docs/reference/setup/install/docker.asciidoc b/docs/reference/setup/install/docker.asciidoc
@@ -279,6 +279,13 @@ docker build --tag=elasticsearch-custom .
 docker run -ti -v /usr/share/elasticsearch/data elasticsearch-custom
 --------------------------------------------
 
+Some plugins require additional security permissions. You have to explicitly accept
+them either by attaching a `tty` when you run the Docker image and accepting yes at 
+the prompts, or inspecting the security permissions separately and if you are 
+comfortable with them adding the `--batch` flag to the plugin install command.
+See {plugins}/_other_command_line_parameters.html[Plugin Management documentation]
+for more details.
+
 ===== D. Override the image's default https://docs.docker.com/engine/reference/run/#cmd-default-command-or-options[CMD]
 
 Options can be passed as command-line options to the {es} process by

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java
@@ -376,7 +376,8 @@ private static MethodHandle lookupReferenceInternal(Definition definition, Looku
              ref.delegateClassName,
              ref.delegateInvokeType,
              ref.delegateMethodName,
-             ref.delegateMethodType
+             ref.delegateMethodType,
+             ref.isDelegateInterface ? 1 : 0
          );
          return callSite.dynamicInvoker().asType(MethodType.methodType(clazz.clazz, captures));
      }

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.painless;
 
 import org.elasticsearch.painless.spi.Whitelist;
+import org.objectweb.asm.Opcodes;
 
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
@@ -202,16 +203,28 @@ public MethodType getMethodType() {
 
         public void write(MethodWriter writer) {
             final org.objectweb.asm.Type type;
+            final Class<?> clazz;
             if (augmentation != null) {
                 assert java.lang.reflect.Modifier.isStatic(modifiers);
+                clazz = augmentation;
                 type = org.objectweb.asm.Type.getType(augmentation);
             } else {
+                clazz = owner.clazz;
                 type = owner.type;
             }
 
             if (java.lang.reflect.Modifier.isStatic(modifiers)) {
-                writer.invokeStatic(type, method);
-            } else if (java.lang.reflect.Modifier.isInterface(owner.clazz.getModifiers())) {
+                // invokeStatic assumes that the owner class is not an interface, so this is a
+                // special case for interfaces where the interface method boolean needs to be set to
+                // true to reference the appropriate class constant when calling a static interface
+                // method since java 8 did not check, but java 9 and 10 do
+                if (java.lang.reflect.Modifier.isInterface(clazz.getModifiers())) {
+                    writer.visitMethodInsn(Opcodes.INVOKESTATIC,
+                            type.getInternalName(), name, getMethodType().toMethodDescriptorString(), true);
+                } else {
+                    writer.invokeStatic(type, method);
+                }
+            } else if (java.lang.reflect.Modifier.isInterface(clazz.getModifiers())) {
                 writer.invokeInterface(type, method);
             } else {
                 writer.invokeVirtual(type, method);

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java
@@ -66,6 +66,9 @@ public class FunctionRef {
     /** delegate method type method as type */
     public final Type delegateType;
 
+    /** whether a call is made on a delegate interface */
+    public final boolean isDelegateInterface;
+
     /**
      * Creates a new FunctionRef, which will resolve {@code type::call} from the whitelist.
      * @param definition the whitelist against which this script is being compiled
@@ -97,10 +100,13 @@ public FunctionRef(Class<?> expected, Method interfaceMethod, Method delegateMet
         // the Painless$Script class can be inferred if owner is null
         if (delegateMethod.owner == null) {
             delegateClassName = CLASS_NAME;
+            isDelegateInterface = false;
         } else if (delegateMethod.augmentation != null) {
             delegateClassName = delegateMethod.augmentation.getName();
+            isDelegateInterface = delegateMethod.augmentation.isInterface();
         } else {
             delegateClassName = delegateMethod.owner.clazz.getName();
+            isDelegateInterface = delegateMethod.owner.clazz.isInterface();
         }
 
         if ("<init>".equals(delegateMethod.name)) {
@@ -139,6 +145,7 @@ public FunctionRef(Class<?> expected,
         delegateInvokeType = H_INVOKESTATIC;
         this.delegateMethodName = delegateMethodName;
         this.delegateMethodType = delegateMethodType.dropParameterTypes(0, numCaptures);
+        isDelegateInterface = false;
 
         this.interfaceMethod = null;
         delegateMethod = null;

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaBootstrap.java
@@ -188,6 +188,10 @@ private Capture(int count, Class<?> type) {
      * @param delegateMethodName The name of the method to be called in the Painless script class
      * @param delegateMethodType The type of method call in the Painless script class without
      *                           the captured types
+     * @param isDelegateInterface If the method to be called is owned by an interface where
+     *                            if the value is '1' if the delegate is an interface and '0'
+     *                            otherwise; note this is an int because the bootstrap method
+     *                            cannot convert constants to boolean
      * @return A {@link CallSite} linked to a factory method for creating a lambda class
      * that implements the expected functional interface
      * @throws LambdaConversionException Thrown when an illegal type conversion occurs at link time
@@ -200,7 +204,8 @@ public static CallSite lambdaBootstrap(
             String delegateClassName,
             int delegateInvokeType,
             String delegateMethodName,
-            MethodType delegateMethodType)
+            MethodType delegateMethodType,
+            int isDelegateInterface)
             throws LambdaConversionException {
         Loader loader = (Loader)lookup.lookupClass().getClassLoader();
         String lambdaClassName = Type.getInternalName(lookup.lookupClass()) + "$$Lambda" + loader.newLambdaIdentifier();
@@ -225,7 +230,7 @@ public static CallSite lambdaBootstrap(
 
         generateInterfaceMethod(cw, factoryMethodType, lambdaClassType, interfaceMethodName,
             interfaceMethodType, delegateClassType, delegateInvokeType,
-            delegateMethodName, delegateMethodType, captures);
+            delegateMethodName, delegateMethodType, isDelegateInterface == 1, captures);
 
         endLambdaClass(cw);
 
@@ -369,6 +374,7 @@ private static void generateInterfaceMethod(
             int delegateInvokeType,
             String delegateMethodName,
             MethodType delegateMethodType,
+            boolean isDelegateInterface,
             Capture[] captures)
             throws LambdaConversionException {
 
@@ -434,7 +440,7 @@ private static void generateInterfaceMethod(
         Handle delegateHandle =
             new Handle(delegateInvokeType, delegateClassType.getInternalName(),
                 delegateMethodName, delegateMethodType.toMethodDescriptorString(),
-                delegateInvokeType == H_INVOKEINTERFACE);
+                isDelegateInterface);
         iface.invokeDynamic(delegateMethodName, Type.getMethodType(interfaceMethodType
                 .toMethodDescriptorString()).getDescriptor(), DELEGATE_BOOTSTRAP_HANDLE,
                 delegateHandle);

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java
@@ -141,8 +141,8 @@ public final class WriterConstants {
 
     /** invokedynamic bootstrap for lambda expression/method references */
     public static final MethodType LAMBDA_BOOTSTRAP_TYPE =
-            MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class,
-                                  MethodType.class, MethodType.class, String.class, int.class, String.class, MethodType.class);
+            MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class,
+                    MethodType.class, String.class, int.class, String.class, MethodType.class, int.class);
     public static final Handle LAMBDA_BOOTSTRAP_HANDLE =
             new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(LambdaBootstrap.class),
                 "lambdaBootstrap", LAMBDA_BOOTSTRAP_TYPE.toMethodDescriptorString(), false);

diff --git a/...es/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java b/...es/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java
@@ -121,7 +121,8 @@ void write(MethodWriter writer, Globals globals) {
                 ref.delegateClassName,
                 ref.delegateInvokeType,
                 ref.delegateMethodName,
-                ref.delegateType
+                ref.delegateType,
+                ref.isDelegateInterface ? 1 : 0
             );
         }
     }

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java
@@ -112,7 +112,8 @@ void write(MethodWriter writer, Globals globals) {
                 ref.delegateClassName,
                 ref.delegateInvokeType,
                 ref.delegateMethodName,
-                ref.delegateType
+                ref.delegateType,
+                ref.isDelegateInterface ? 1 : 0
             );
         } else {
             // TODO: don't do this: its just to cutover :)

diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java
@@ -222,7 +222,8 @@ void write(MethodWriter writer, Globals globals) {
                 ref.delegateClassName,
                 ref.delegateInvokeType,
                 ref.delegateMethodName,
-                ref.delegateType
+                ref.delegateType,
+                ref.isDelegateInterface ? 1 : 0
             );
         } else {
             // placeholder

diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java
@@ -264,6 +264,11 @@ public void testNullSafeDeref() {
 //        assertEquals(null, exec("def a = ['thing': 'bar']; a.other?.cat?.dog = 'wombat'; return a.other?.cat?.dog"));
     }
 
+    // test to ensure static interface methods are called correctly
+    public void testStaticInterfaceMethod() {
+        assertEquals(4, exec("def values = [1, 4, 3, 2]; values.sort(Comparator.comparing(p -> p)); return values[3]"));
+    }
+
     private void assertMustBeNullable(String script) {
         Exception e = expectScriptThrows(IllegalArgumentException.class, false, () -> exec(script));
         assertEquals("Result of null safe operator must be nullable", e.getMessage());

diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java
@@ -184,6 +184,11 @@ public void testInterfaceDefaultMethodDef() {
                                  "def map = new HashMap(); f(map::getOrDefault)"));
     }
 
+    public void testInterfaceStaticMethod() {
+        assertEquals(-1, exec("Supplier get(Supplier supplier) { return supplier }" +
+                              "Supplier s = get(Comparator::naturalOrder); s.get().compare(1, 2)"));
+    }
+
     public void testMethodMissing() {
         Exception e = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("List l = [2, 1]; l.sort(Integer::bogus); return l.get(0);");

diff --git a/qa/build.gradle b/qa/build.gradle
@@ -5,6 +5,20 @@ subprojects { Project subproj ->
   subproj.tasks.withType(RestIntegTestTask) {
     subproj.extensions.configure("${it.name}Cluster") { cluster ->
       cluster.distribution = System.getProperty('tests.distribution', 'oss-zip')
+      if (cluster.distribution == 'zip') {
+        /*
+         * Add Elastic's repositories so we can resolve older versions of the
+         * default distribution. Those aren't in maven central.
+         */
+        repositories {
+          maven {
+            url "https://artifacts.elastic.co/maven"
+          }
+          maven {
+            url "https://snapshots.elastic.co/maven"
+          }
+        }
+      }
     }
   }
 }
diff --git a/qa/vagrant/src/test/resources/packaging/tests/70_sysv_initd.bats b/qa/vagrant/src/test/resources/packaging/tests/70_sysv_initd.bats
@@ -163,3 +163,31 @@ setup() {
     assert_file_exist /var/log/elasticsearch/gc.log.0.current
     stop_elasticsearch_service
 }
+
+# Ensures that if $MAX_MAP_COUNT is less than the set value on the OS
+# it will be updated
+@test "[INIT.D] sysctl is run when the value set is too small" {
+  # intentionally a ridiculously low number
+  sysctl -q -w vm.max_map_count=100
+  start_elasticsearch_service
+  max_map_count=$(sysctl -n vm.max_map_count)
+  stop_elasticsearch_service
+
+  [ $max_map_count = 262144 ]
+
+}
+
+# Ensures that if $MAX_MAP_COUNT is greater than the set vaule on the OS
+# we do not attempt to update it this should cover equality as well as I think
+# we can trust that equality operators work as intended.
+@test "[INIT.D] sysctl is not run when it already has a larger or equal value set" {
+  # intentionally set to the default +1
+  sysctl -q -w vm.max_map_count=262145
+  start_elasticsearch_service
+  max_map_count=$(sysctl -n vm.max_map_count)
+  stop_elasticsearch_service
+
+  # default value +1
+  [ $max_map_count = 262145 ]
+
+}