Merge branch 'main' into improve-task-avoidance

.buildkite/hooks/pre-command

@@ -92,7 +92,7 @@ fi
 
 # Initialize the build scan and gobld annotations with empty/open <details> tags
 # This ensures that they are collapsible when they get appended to
-if [[ "${BUILDKITE_LABEL:-}" == *"Pipeline upload"* ]]; then
+if [[ "${BUILDKITE_LABEL:-}" == *"Pipeline upload"* || "${BUILDKITE_LABEL:-}" == *"Upload Pipeline"* ]]; then
   cat << EOF | buildkite-agent annotate --context "gradle-build-scans" --style "info"
 <details>
 

.buildkite/pipelines/periodic.template.yml

@@ -89,7 +89,7 @@ steps:
           GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
   - label: release-tests
     command: .buildkite/scripts/release-tests.sh
-    timeout_in_minutes: 300
+    timeout_in_minutes: 360
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic.yml

@@ -1150,7 +1150,7 @@ steps:
           GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
   - label: release-tests
     command: .buildkite/scripts/release-tests.sh
-    timeout_in_minutes: 300
+    timeout_in_minutes: 360
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2004

.ci/jobs.t/elastic+elasticsearch+periodic+bwc.yml

@@ -1,9 +1,8 @@
 ---
-jjbb-template: matrix-gradle-unix.yml
+jjbb-template: matrix-gradle-unix-disabled.yml
 vars:
   - job-name: elastic+elasticsearch+%BRANCH%+periodic+bwc
   - job-display-name: "elastic / elasticsearch # %BRANCH% - backwards compatibility matrix"
-  - job-description: "Testing of the Elasticsearch %BRANCH% branch backwards compatibility matrix.\n"
   - matrix-yaml-file: ".ci/bwcVersions"
   - matrix-variable: BWC_VERSION
   - gradle-args: "-Dbwc.checkout.align=true v$BWC_VERSION#bwcTest"

.ci/jobs.t/elastic+elasticsearch+periodic+java-fips-matrix.yml

@@ -2,7 +2,8 @@
 - job:
     name: "elastic+elasticsearch+%BRANCH%+periodic+java-fips-matrix"
     display-name: "elastic / elasticsearch # %BRANCH% - java fips compatibility matrix"
-    description: "Testing of the Elasticsearch %BRANCH% branch java FIPS compatibility matrix.\n"
+    description: "This job has been migrated to Buildkite.\n"
+    disabled: true
     project-type: matrix
     child-workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+periodic+java-fips-matrix
     node: master

.ci/jobs.t/elastic+elasticsearch+periodic+java-matrix.yml

@@ -2,7 +2,8 @@
 - job:
     name: "elastic+elasticsearch+%BRANCH%+periodic+java-matrix"
     display-name: "elastic / elasticsearch # %BRANCH% - java compatibility matrix"
-    description: "Testing of the Elasticsearch %BRANCH% branch java compatibility matrix.\n"
+    description: "This job has been migrated to Buildkite.\n"
+    disabled: true
     project-type: matrix
     child-workspace: /dev/shm/elastic+elasticsearch+%BRANCH%+periodic+java-matrix
     node: master

.ci/jobs.t/elastic+elasticsearch+periodic+packaging-tests.yml

@@ -2,7 +2,8 @@
 - job:
     name: elastic+elasticsearch+%BRANCH%+periodic+packaging-tests
     display-name: "elastic / elasticsearch # %BRANCH% - packaging tests"
-    description: "Testing of the Elasticsearch %BRANCH% branch packaging tests.\n"
+    description: "This job has been migrated to Buildkite.\n"
+    disabled: true
     project-type: multijob
     node: master
     vault: []

.ci/templates.t/matrix-gradle-unix-disabled.yml

@@ -0,0 +1,32 @@
+---
+- job:
+    name: "{job-name}"
+    display-name: "{job-display-name}"
+    description: "This job has been migrated to Buildkite.\n"
+    disabled: true
+    project-type: matrix
+    child-workspace: /dev/shm/{job-name}
+    node: master
+    scm:
+      - git:
+          wipe-workspace: false
+    axes:
+      - axis:
+          type: slave
+          name: nodes
+          values:
+            - "general-purpose"
+      - axis:
+          type: yaml
+          filename: "{matrix-yaml-file}"
+          name: "{matrix-variable}"
+    builders:
+      - inject:
+          properties-file: ".ci/java-versions.properties"
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            JAVA11_HOME=$HOME/.java/java11
+            JAVA16_HOME=$HOME/.java/openjdk16
+      - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr
+          $WORKSPACE/.ci/scripts/run-gradle.sh {gradle-args}

TESTING.asciidoc

@@ -108,6 +108,8 @@ password: `elastic-password`.
 - In order to set an Elasticsearch setting, provide a setting with the following prefix: `-Dtests.es.`
 - In order to pass a JVM setting, e.g. to disable assertions: `-Dtests.jvm.argline="-da"`
 - In order to use HTTPS: ./gradlew run --https
+- In order to start a mock logging APM server on port 9999 and configure ES cluster to connect to it,
+use `./gradlew run --with-apm-server`
 
 ==== Customizing the test cluster for ./gradlew run
 
@@ -487,136 +489,16 @@ If in doubt about which command to use, simply run <gradle path>:check
 
 == Testing packaging
 
-The packaging tests use Vagrant virtual machines or cloud instances to verify
+The packaging tests are run on different build vm cloud instances to verify
 that installing and running Elasticsearch distributions works correctly on
 supported operating systems. These tests should really only be run on ephemeral
 systems because they're destructive; that is, these tests install and remove
 packages and freely modify system settings, so you will probably regret it if
 you execute them on your development machine.
 
-When you run a packaging test, Gradle will set up the target VM and mount your
-repository directory in the VM. Once this is done, a Gradle task will issue a
-Vagrant command to run a *nested* Gradle task on the VM. This nested Gradle
-runs the actual "destructive" test classes.
-
-. Install Virtual Box and Vagrant.
-+
-. (Optional) Install https://github.com/fgrehm/vagrant-cachier[vagrant-cachier] to squeeze
-a bit more performance out of the process:
-+
---------------------------------------
-vagrant plugin install vagrant-cachier
---------------------------------------
-+
-. You can run all of the OS packaging tests with `./gradlew packagingTest`.
-This task includes our legacy `bats` tests. To run only the OS tests that are
-written in Java, run `.gradlew distroTest`, will cause Gradle to build the tar,
-zip, and deb packages and all the plugins. It will then run the tests on every
-available system. This will take a very long time.
-+
-Fortunately, the various systems under test have their own Gradle tasks under
-`qa/os`. To find the systems tested, do a listing of the `qa/os` directory.
-To find out what packaging combinations can be tested on a system, run
-the `tasks` task. For example:
-+
-----------------------------------
-./gradlew :qa:os:ubuntu-1804:tasks
-----------------------------------
-+
-If you want a quick test of the tarball and RPM packagings for Centos 7, you
-would run:
-+
--------------------------------------------------------------------------------------------------
-./gradlew :qa:os:centos-7:distroTest.default-rpm :qa:os:centos-7:distroTest.default-linux-archive
--------------------------------------------------------------------------------------------------
-
-Note that if you interrupt Gradle in the middle of running these tasks, any boxes started
-will remain running and you'll have to stop them manually with `./gradlew --stop` or
-`vagrant halt`.
-
-All the regular vagrant commands should just work so you can get a shell in a
-VM running trusty by running
-`vagrant up ubuntu-1804 --provider virtualbox && vagrant ssh ubuntu-1804`.
-
-=== Testing packaging on Windows
-
-The packaging tests also support Windows Server 2012R2 and Windows Server 2016.
-Unfortunately we're not able to provide boxes for them in open source use
-because of licensing issues. Any Virtualbox image that has WinRM and Powershell
-enabled for remote users should work.
-
-Specify the image IDs of the Windows boxes to gradle with the following project
-properties. They can be set in `~/.gradle/gradle.properties` like
-
-------------------------------------
-vagrant.windows-2012r2.id=my-image-id
-vagrant.windows-2016.id=another-image-id
-------------------------------------
-
-or passed on the command line like `-Pvagrant.windows-2012r2.id=my-image-id`
-`-Pvagrant.windows-2016=another-image-id`
-
-These properties are required for Windows support in all gradle tasks that
-handle packaging tests. Either or both may be specified.
-
-If you're running vagrant commands outside of gradle, specify the Windows boxes
-with the environment variables
-
-* `VAGRANT_WINDOWS_2012R2_BOX`
-* `VAGRANT_WINDOWS_2016_BOX`
-
-=== Testing VMs are disposable
-
-It's important to think of VMs like cattle. If they become lame you just shoot
-them and let vagrant reprovision them. Say you've hosed your precise VM:
+=== Reproducing packaging tests
 
-----------------------------------------------------
-vagrant ssh ubuntu-1604 -c 'sudo rm -rf /bin'; echo oops
-----------------------------------------------------
-
-All you've got to do to get another one is
-
-----------------------------------------------
-vagrant destroy -f ubuntu-1604 && vagrant up ubuntu-1604 --provider virtualbox
-----------------------------------------------
-
-The whole process takes a minute and a half on a modern laptop, two and a half
-without vagrant-cachier.
-
-It's possible that some downloads will fail and it'll be impossible to restart
-them. This is a bug in vagrant. See the instructions here for how to work
-around it:
-https://github.com/mitchellh/vagrant/issues/4479
-
-Some vagrant commands will work on all VMs at once:
-
-------------------
-vagrant halt
-vagrant destroy -f
-------------------
-
-`vagrant up` would normally start all the VMs but we've prevented that because
-that'd consume a ton of ram.
-
-=== Iterating on packaging tests
-
-Because our packaging tests are capable of testing many combinations of OS
-(e.g., Windows, Linux, etc.), package type (e.g., zip file, RPM, etc.),
-Elasticsearch distribution type (e.g., default or OSS), and so forth, it's
-faster to develop against smaller subsets of the tests. For example, to run
-tests for the default archive distribution on Fedora 28:
-
------------------------------------------------------------
-./gradlew :qa:os:fedora-28:distroTest.default-linux-archive
------------------------------------------------------------
-
-These test tasks can use the `--tests`, `--info`, and `--debug` parameters just like
-non-OS tests can. For example:
-
------------------------------------------------------------
-./gradlew :qa:os:fedora-28:distroTest.default-linux-archive \
-  --tests "com.elasticsearch.packaging.test.ArchiveTests"
------------------------------------------------------------
+To reproduce or debug packaging tests failures we recommend using using our provided https://github.com/elastic/elasticsearch-infra/blob/master/buildkite-tools/README.md[*buildkite tools*]
 
 == Testing backwards compatibility