Merge remote-tracking branch 'upstream/master' into move-index-patter…

…ns-server-pt2

.ci/Jenkinsfile_flaky

@@ -8,26 +8,28 @@ def JOB_PARTS = params.CI_GROUP.split(':')
 def IS_XPACK = JOB_PARTS[0] == 'xpack'
 def JOB = JOB_PARTS[1]
 def CI_GROUP = JOB_PARTS.size() > 2 ? JOB_PARTS[2] : ''
+def EXECUTIONS = params.NUMBER_EXECUTIONS.toInteger()
+def AGENT_COUNT = getAgentCount(EXECUTIONS)
 
 def worker = getWorkerFromParams(IS_XPACK, JOB, CI_GROUP)
 
 def workerFailures = []
 
 currentBuild.displayName += trunc(" ${params.GITHUB_OWNER}:${params.branch_specifier}", 24)
-currentBuild.description = "${params.CI_GROUP}<br />Executions: ${params.NUMBER_EXECUTIONS}"
-
-// Note: If you increase agent count, it will execute NUMBER_EXECUTIONS per agent. It will not divide them up amongst the agents
-// e.g. NUMBER_EXECUTIONS = 25, agentCount = 4 results in 100 total executions
-def agentCount = 1
+currentBuild.description = "${params.CI_GROUP}<br />Agents: ${AGENT_COUNT}<br />Executions: ${params.NUMBER_EXECUTIONS}"
 
 stage("Kibana Pipeline") {
   timeout(time: 180, unit: 'MINUTES') {
     timestamps {
       ansiColor('xterm') {
         def agents = [:]
-        for(def agentNumber = 1; agentNumber <= agentCount; agentNumber++) {
+        for(def agentNumber = 1; agentNumber <= AGENT_COUNT; agentNumber++) {
+          def agentNumberInside = agentNumber
+          def agentExecutions = floor(EXECUTIONS/AGENT_COUNT) + (agentNumber <= EXECUTIONS%AGENT_COUNT ? 1 : 0)
           agents["agent-${agentNumber}"] = {
             catchError {
+              print "Agent ${agentNumberInside} - ${agentExecutions} executions"
+
               kibanaPipeline.withWorkers('flaky-test-runner', {
                 if (!IS_XPACK) {
                   kibanaPipeline.buildOss()
@@ -37,7 +39,7 @@ stage("Kibana Pipeline") {
                 } else {
                   kibanaPipeline.buildXpack()
                 }
-              }, getWorkerMap(agentNumber, params.NUMBER_EXECUTIONS.toInteger(), worker, workerFailures))()
+              }, getWorkerMap(agentNumberInside, agentExecutions, worker, workerFailures))()
             }
           }
         }
@@ -77,17 +79,20 @@ def getWorkerFromParams(isXpack, job, ciGroup) {
   }
 }
 
-def getWorkerMap(agentNumber, numberOfExecutions, worker, workerFailures, maxWorkers = 14) {
+def getWorkerMap(agentNumber, numberOfExecutions, worker, workerFailures, maxWorkerProcesses = 12) {
   def workerMap = [:]
-  def numberOfWorkers = Math.min(numberOfExecutions, maxWorkers)
+  def numberOfWorkers = Math.min(numberOfExecutions, maxWorkerProcesses)
 
   for(def i = 1; i <= numberOfWorkers; i++) {
     def workerExecutions = numberOfExecutions/numberOfWorkers + (i <= numberOfExecutions%numberOfWorkers ? 1 : 0)
 
     workerMap["agent-${agentNumber}-worker-${i}"] = { workerNumber ->
       for(def j = 0; j < workerExecutions; j++) {
         print "Execute agent-${agentNumber} worker-${workerNumber}: ${j}"
-        withEnv(["JOB=agent-${agentNumber}-worker-${workerNumber}-${j}"]) {
+        withEnv([
+          "JOB=agent-${agentNumber}-worker-${workerNumber}-${j}",
+          "REMOVE_KIBANA_INSTALL_DIR=1",
+        ]) {
           catchError {
             try {
               worker(workerNumber)
@@ -104,10 +109,23 @@ def getWorkerMap(agentNumber, numberOfExecutions, worker, workerFailures, maxWor
   return workerMap
 }
 
+def getAgentCount(executions) {
+  // Increase agent count every 24 worker processess, up to 3 agents maximum
+  return Math.min(3, 1 + floor(executions/24))
+}
+
 def trunc(str, length) {
   if (str.size() >= length) {
     return str.take(length) + "..."
   }
 
   return str;
 }
+
+// All of the real rounding/truncating methods are sandboxed
+def floor(num) {
+  return num
+    .toString()
+    .split('\\.')[0]
+    .toInteger()
+}

.ci/jobs.yml

@@ -14,7 +14,7 @@ JOB:
   - kibana-ciGroup10
   - kibana-ciGroup11
   - kibana-ciGroup12
-  # - kibana-visualRegression
+  - kibana-visualRegression
 
   # make sure all x-pack-ciGroups are listed in test/scripts/jenkins_xpack_ci_group.sh
   - x-pack-firefoxSmoke
@@ -28,7 +28,7 @@ JOB:
   - x-pack-ciGroup8
   - x-pack-ciGroup9
   - x-pack-ciGroup10
-  # - x-pack-visualRegression
+  - x-pack-visualRegression
 
 # `~` is yaml for `null`
 exclude: ~

.eslintrc.js

@@ -206,8 +206,6 @@ module.exports = {
       files: ['x-pack/legacy/plugins/ml/**/*.{js,ts,tsx}'],
       rules: {
         'react-hooks/exhaustive-deps': 'off',
-        'react-hooks/rules-of-hooks': 'off',
-        'jsx-a11y/click-events-have-key-events': 'off',
       },
     },
     {
@@ -229,12 +227,6 @@ module.exports = {
         'react-hooks/exhaustive-deps': 'off',
       },
     },
-    {
-      files: ['x-pack/legacy/plugins/transform/**/*.{js,ts,tsx}'],
-      rules: {
-        'react-hooks/exhaustive-deps': 'off',
-      },
-    },
     {
       files: ['x-pack/legacy/plugins/uptime/**/*.{js,ts,tsx}'],
       rules: {
@@ -370,10 +362,10 @@ module.exports = {
                   '!src/core/server/*.test.mocks.ts',
 
                   'src/plugins/**/public/**/*',
-                  '!src/plugins/**/public/index*',
+                  '!src/plugins/**/public/index.{js,ts,tsx}',
 
                   'src/plugins/**/server/**/*',
-                  '!src/plugins/**/server/index*',
+                  '!src/plugins/**/server/index.{js,ts,tsx}',
                 ],
                 allowSameFolder: true,
               },

.github/CODEOWNERS

@@ -8,9 +8,14 @@
 
 # App Architecture
 /src/plugins/data/ @elastic/kibana-app-arch
-/src/plugins/kibana_utils/ @elastic/kibana-app-arch
+/src/plugins/embeddable/ @elastic/kibana-app-arch
+/src/plugins/expressions/ @elastic/kibana-app-arch
 /src/plugins/kibana_react/ @elastic/kibana-app-arch
+/src/plugins/kibana_utils/ @elastic/kibana-app-arch
 /src/plugins/navigation/ @elastic/kibana-app-arch
+/src/plugins/ui_actions/ @elastic/kibana-app-arch
+/src/plugins/visualizations/ @elastic/kibana-app-arch
+/x-pack/plugins/advanced_ui_actions/ @elastic/kibana-app-arch
 
 # APM
 /x-pack/legacy/plugins/apm/  @elastic/apm-ui
@@ -37,7 +42,9 @@
 /x-pack/test/functional/apps/machine_learning/  @elastic/ml-ui
 /x-pack/test/functional/services/machine_learning/  @elastic/ml-ui
 /x-pack/test/functional/services/ml.ts  @elastic/ml-ui
-/x-pack/legacy/plugins/transform/  @elastic/ml-ui
+# ML team owns the transform plugin, ES team added here for visibility
+# because the plugin lives in Kibana's Elasticsearch management section.
+/x-pack/legacy/plugins/transform/  @elastic/ml-ui @elastic/es-ui 
 
 # Operations
 /renovate.json5 @elastic/kibana-operations
@@ -56,6 +63,7 @@
 /src/legacy/server/saved_objects/  @elastic/kibana-platform
 /config/kibana.yml @elastic/kibana-platform
 /x-pack/plugins/features/  @elastic/kibana-platform
+/x-pack/plugins/licensing/  @elastic/kibana-platform
 
 # Security
 /x-pack/legacy/plugins/security/  @elastic/kibana-security
@@ -88,9 +96,6 @@
 /x-pack/legacy/plugins/rollup/  @elastic/es-ui
 /x-pack/legacy/plugins/searchprofiler/  @elastic/es-ui
 /x-pack/legacy/plugins/snapshot_restore/  @elastic/es-ui
-# ML team owns the transform plugin, ES team added here for visibility
-# because the plugin lives in Kibana's Elasticsearch management section.
-/x-pack/legacy/plugins/transform/  @elastic/es-ui
 /x-pack/legacy/plugins/watcher/  @elastic/es-ui
 
 # Kibana TSVB external contractors

.i18nrc.json

@@ -3,6 +3,7 @@
     "common.ui": "src/legacy/ui",
     "data": ["src/legacy/core_plugins/data", "src/plugins/data"],
     "expressions": "src/legacy/core_plugins/expressions",
+    "expressions_np": "src/plugins/expressions",
     "kibana_react": "src/legacy/core_plugins/kibana_react",
     "navigation": "src/legacy/core_plugins/navigation",
     "server": "src/legacy/server",
@@ -29,6 +30,7 @@
     "kbnESQuery": "packages/kbn-es-query",
     "inspector": "src/plugins/inspector",
     "kibana-react": "src/plugins/kibana_react",
+    "visualizations": "src/plugins/visualizations",
     "telemetry": "src/legacy/core_plugins/telemetry",
     "esUi": "src/plugins/es_ui_shared",
     "uiActions": "src/plugins/ui_actions"

Jenkinsfile

@@ -25,7 +25,7 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a
               'oss-ciGroup11': kibanaPipeline.getOssCiGroupWorker(11),
               'oss-ciGroup12': kibanaPipeline.getOssCiGroupWorker(12),
               'oss-firefoxSmoke': kibanaPipeline.getPostBuildWorker('firefoxSmoke', { runbld './test/scripts/jenkins_firefox_smoke.sh' }),
-              // 'oss-visualRegression': kibanaPipeline.getPostBuildWorker('visualRegression', { runbld './test/scripts/jenkins_visual_regression.sh' }),
+              'oss-visualRegression': kibanaPipeline.getPostBuildWorker('visualRegression', { runbld './test/scripts/jenkins_visual_regression.sh' }),
             ]),
             'kibana-xpack-agent': kibanaPipeline.withWorkers('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
               'xpack-ciGroup1': kibanaPipeline.getXpackCiGroupWorker(1),
@@ -39,12 +39,12 @@ stage("Kibana Pipeline") { // This stage is just here to help the BlueOcean UI a
               'xpack-ciGroup9': kibanaPipeline.getXpackCiGroupWorker(9),
               'xpack-ciGroup10': kibanaPipeline.getXpackCiGroupWorker(10),
               'xpack-firefoxSmoke': kibanaPipeline.getPostBuildWorker('xpack-firefoxSmoke', { runbld './test/scripts/jenkins_xpack_firefox_smoke.sh' }),
-              // 'xpack-visualRegression': kibanaPipeline.getPostBuildWorker('xpack-visualRegression', { runbld './test/scripts/jenkins_xpack_visual_regression.sh' }),
+              'xpack-visualRegression': kibanaPipeline.getPostBuildWorker('xpack-visualRegression', { runbld './test/scripts/jenkins_xpack_visual_regression.sh' }),
             ]),
           ])
         }
         kibanaPipeline.sendMail()
       }
     }
   }
-}
+}

docs/discover/kuery.asciidoc

@@ -73,3 +73,87 @@ set these terms will be matched against all fields. For example, a query for `re
 in the response field, but a query for just `200` will search for 200 across all fields in your index.
 ============
 
+===== Nested Field Support
+
+KQL supports querying on {ref}/nested.html[nested fields] through a special syntax. You can query nested fields in subtly different
+ways, depending on the results you want, so crafting nested queries requires extra thought.
+
+One main consideration is how to match parts of the nested query to the individual nested documents.
+There are two main approaches to take:
+
+* *Parts of the query may only match a single nested document.* This is what most users want when querying on a nested field.
+* *Parts of the query can match different nested documents.* This is how a regular object field works. 
+ Although generally less useful, there might be occasions where you want to query a nested field in this way.
+
+Let's take a look at the first approach. In the following document, `items` is a nested field:
+
+[source,json]
+----------------------------------
+{
+  "grocery_name": "Elastic Eats",
+  "items": [
+    {
+      "name": "banana",
+      "stock": "12",
+      "category": "fruit"
+    },
+    {
+      "name": "peach",
+      "stock": "10",
+      "category": "fruit"
+    },
+    {
+      "name": "carrot",
+      "stock": "9",
+      "category": "vegetable"
+    },
+    {
+      "name": "broccoli",
+      "stock": "5",
+      "category": "vegetable"
+    }
+  ]
+}
+----------------------------------
+
+To find stores that have more than 10 bananas in stock, you would write a query like this:
+
+`items:{ name:banana and stock > 10 }`
+
+`items` is the "nested path". Everything inside the curly braces (the "nested group") must match a single document. 
+For example, `items:{ name:banana and stock:9 }` does not match because there isn't a single nested document that 
+matches the entire query in the nested group. 
+
+What if you want to find a store with more than 10 bananas that *also* stocks vegetables? This is the second way of querying a nested field, and you can do it like this:
+
+`items:{ name:banana and stock > 10 } and items:{ category:vegetable }`
+
+The first nested group (`name:banana and stock > 10`) must still match a single document, but the `category:vegetables`
+subquery can match a different nested document because it is in a separate group.
+
+KQL's syntax also supports nested fields inside of other nested fields—you simply have to specify the full path. Suppose you
+have a document where `level1` and `level2` are both nested fields:
+
+[source,json]
+----------------------------------
+{
+  "level1": [
+    {
+      "level2": [
+        {
+          "prop1": "foo",
+          "prop2": "bar"
+        },
+        {
+          "prop1": "baz",
+          "prop2": "qux"
+        }
+      ]
+    }
+  ]
+}
+----------------------------------
+
+You can match on a single nested document by specifying the full path:
+
+`level1.level2:{ prop1:foo and prop2:bar }`

docs/uptime-guide/security.asciidoc

@@ -31,17 +31,6 @@ PUT /_security/role/uptime
       "allow_restricted_indices" : false
     }
   ],
-  "applications" : [
-    {
-      "application" : "kibana-.kibana",
-      "privileges" : [
-        "all"
-      ],
-      "resources" : [
-        "*"
-      ]
-    }
-  ],
   "transient_metadata" : {
     "enabled" : true
   }
@@ -52,7 +41,8 @@ PUT /_security/role/uptime
 [float]
 === Assign the role to a user
 
-Next, you'll need to create a user with both the `kibana_user`, and `uptime` roles. 
+Next, you'll need to create a user with both the `uptime` role, and another role with sufficient {kibana-ref}/kibana-privileges.html[Kibana privileges],
+such as the `kibana_user` role.
 You can do this with the following request:
 
 ["source","sh",subs="attributes,callouts"]

docs/user/reporting/index.asciidoc

@@ -32,8 +32,8 @@ for different operating systems.
 
 . Open {kib} in your web browser and log in. If you are running {kib}
 locally, go to `http://localhost:5601`. To access {kib} and generate
-reports, you need the `kibana_user` and `reporting_user` roles. For more
-information, see <<secure-reporting>>.
+reports, you need the `reporting_user` role, and an additional role with succifient <<kibana-privileges, Kibana privileges>>, such as the `kibana_user` role.
+For more information, see <<secure-reporting>>.
 
 . Open the dashboard, visualization, or saved search you want to include
 in the report.

docs/user/security/authentication/index.asciidoc

@@ -138,7 +138,7 @@ xpack.security.authc.saml.maxRedirectURLSize: 1kb
 ==== OpenID Connect Single Sign-On
 
 Similar to SAML, authentication with OpenID Connect allows users to log in to {kib} using an OpenID Connect Provider such as Google, or Okta. OpenID Connect
-should also be configured in {es}, see {xpack-ref}/saml-guide.html[Configuring OpenID Connect Single-Sign-On on the Elastic Stack] for more details.
+should also be configured in {es}. For more details, see {ref}/oidc-guide.html[Configuring single sign-on to the {stack} using OpenID Connect].
 
 Set the configuration values in `kibana.yml` as follows:
 

docs/user/security/reporting.asciidoc

@@ -19,7 +19,7 @@ and `kibana_user` roles:
 * If you're using the `native` realm, you can assign roles through
 **Management / Users** UI in Kibana or with the `user` API. For example,
 the following request creates a `reporter` user that has the
-`reporting_user` and `kibana_user` roles:
+`reporting_user` role, and another role with sufficient <<kibana-privileges, Kibana privileges>>, such as the `kibana_user` role:
 +
 [source, sh]
 ---------------------------------------------------------------

docs/user/security/securing-kibana.asciidoc

@@ -117,7 +117,7 @@ user you've assigned a {kib} user role. For example, you could log in as the
 +
 --
 
-NOTE: This must be a user who has been assigned the `kibana_user` role.
+NOTE: This must be a user who has been assigned <<kibana-privileges, Kibana privileges>>.
 {kib} server credentials should only be used internally by the {kib} server.
 
 --