Merge branch 'main' into main

Signed-off-by: Rishabh Singh <sngri@amazon.com>

.github/ISSUE_TEMPLATE/plugin_onboarding_template.yaml

@@ -7,7 +7,7 @@ body:
   - type: markdown
     attributes:
       value: |
-        Thanks for taking the time to submit on-boarding request!
+        Thanks for taking the time to submit on-boarding request! Please note that the estimate time for on-barding a plugin to the distribution is 2 weeks from the time of the request.
   - type: textarea
     attributes:
       description: 'Provide the name of your plugin.'

.github/workflows/python-tests.yml

@@ -46,6 +46,8 @@ jobs:
   python-tests-linux:
     needs: Get-CI-Image-Tag
     runs-on: ubuntu-latest
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     container:
       # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
       # this image tag is subject to change as more dependencies and updates will arrive over time

ONBOARDING.md

@@ -10,7 +10,7 @@
 
 ## Plugin Onboarding
 
-This document describes the steps required to onboard a new plugin to the release workflow for continuous integration and testing. Make sure that you also consider whether the plugin requires documentation to implement or use it. If so, see [Contributing.md](https://github.com/opensearch-project/documentation-website/blob/main/CONTRIBUTING.md) in the documentation-website GitHub repository.
+This document describes the steps required to onboard a new plugin to the release workflow for continuous integration and testing. **Please note that the estimate time for on-barding a plugin to the distribution is 2 weeks from the time of the request.** Make sure that you also consider whether the plugin requires documentation to implement or use it. If so, see [Contributing.md](https://github.com/opensearch-project/documentation-website/blob/main/CONTRIBUTING.md) in the documentation-website GitHub repository.
 
 ### Onboard to OpenSearch Meta
 

docker/ci/config/windows-setup.ps1

@@ -187,8 +187,8 @@ scoop install gh
 gh version
 
 # Install dev tools
-# Lock to 3.23.3
-scoop install https://raw.githubusercontent.com/ScoopInstaller/Main/56eed69c3bf04110e306f77ad45cfc8c1c5bb9bc/bucket/cmake.json
+# Lock to 3.26.4
+scoop install https://raw.githubusercontent.com/ScoopInstaller/Main/22a3d6e16c3688926f575eb80e3cc20286b8cf50/bucket/cmake.json
 cmake --version
 
 # Install zip

docker/ci/dockerfiles/current/build.al2.opensearch.x64.arm64.dockerfile

@@ -112,7 +112,7 @@ RUN git clone -b v0.3.27 --single-branch https://github.com/xianyi/OpenBLAS.git
     fi && \
     make PREFIX=/usr/local install
 ENV LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH"
-RUN pip3 install cmake==3.23.3
+RUN pip3 install cmake==3.26.4
 
 # Change User
 USER $CONTAINER_USER

docker/ci/dockerfiles/current/build.almalinux8.opensearch-dashboards.x64.arm64.ppc64le.dockerfile

@@ -80,7 +80,7 @@ USER $CONTAINER_USER
 WORKDIR $CONTAINER_USER_HOME
 
 # Install fpm for opensearch dashboards core
-RUN gem install dotenv -v 2.8.1 && gem install fpm -v 1.14.2
+RUN gem install dotenv -v 2.8.1 && gem install public_suffix -v 5.1.1 && gem install fpm -v 1.14.2
 ENV PATH=$CONTAINER_USER_HOME/.gem/gems/fpm-1.14.2/bin:$PATH
 
 # Hard code node version and yarn version for now

docker/ci/dockerfiles/current/build.almalinux8.opensearch.x64.arm64.ppc64le.dockerfile

@@ -86,7 +86,7 @@ RUN dnf install -y 'dnf-command(config-manager)' && \
     dnf config-manager --set-enabled powertools && \
     dnf install epel-release -y && dnf repolist && \
     dnf install openblas-static lapack gcc-gfortran -y && dnf clean all
-RUN pip3 install cmake==3.23.3
+RUN pip3 install cmake==3.26.4
 # Upgrade gcc
 # The setup part is partially based on Austin Dewey's article:
 # https://austindewey.com/2019/03/26/enabling-software-collections-binaries-on-a-docker-image/
@@ -102,6 +102,6 @@ USER $CONTAINER_USER
 WORKDIR $CONTAINER_USER_HOME
 
 # Install fpm for opensearch dashboards core
-RUN gem install dotenv -v 2.8.1 && gem install fpm -v 1.14.2
+RUN gem install dotenv -v 2.8.1 && gem install public_suffix -v 5.1.1 && gem install fpm -v 1.14.2
 ENV PATH=$CONTAINER_USER_HOME/.gem/gems/fpm-1.14.2/bin:$PATH
 RUN fpm -v

docker/ci/dockerfiles/current/test.almalinux8.opensearch-dashboards.x64.arm64.ppc64le.dockerfile

@@ -130,7 +130,7 @@ RUN update-alternatives --set python /usr/bin/python3.9 && \
 
 # Add other dependencies
 RUN dnf install -y epel-release && dnf clean all && dnf install -y jq && dnf clean all && rm -rf /var/cache/dnf/* && \
-    pip3 install cmake==3.23.3
+    pip3 install cmake==3.26.4
 
 # Tools setup
 COPY --chown=0:0 config/yq-setup.sh config/gh-setup.sh /tmp/

docker/ci/dockerfiles/current/test.almalinux8.systemd-base.x64.arm64.ppc64le.dockerfile

@@ -152,7 +152,7 @@ RUN update-alternatives --set python /usr/bin/python3.9 && \
 
 # Add other dependencies
 RUN dnf install -y epel-release && dnf clean all && dnf install -y jq && dnf clean all && \
-    pip3 install cmake==3.23.3
+    pip3 install cmake==3.26.4
 
 # We use the version test to check if packages installed correctly
 # And get added to the PATH

docker/ci/dockerfiles/current/test.ubuntu2004.systemd-base.x64.arm64.ppc64le.dockerfile

@@ -114,7 +114,7 @@ RUN apt-get update -y && apt-get upgrade -y && apt-get install -y curl git gnupg
 # Install pip packages
 RUN curl -SL https://bootstrap.pypa.io/get-pip.py | python && \
     pip3 install pip==23.1.2 && pip3 install pipenv==2023.6.12 awscli==1.32.17 && \
-    pip3 install cmake==3.23.3
+    pip3 install cmake==3.26.4
 
 # Create user group
 RUN apt-get install -y sudo && \

jenkins/check-for-build.jenkinsfile

@@ -23,10 +23,11 @@ pipeline {
     }
     triggers {
         parameterizedCron '''
+            H */6 * * * %INPUT_MANIFEST=1.3.18/opensearch-dashboards-1.3.18.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip
             H 1 * * * %INPUT_MANIFEST=2.15.1/opensearch-2.15.1.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip
             H 1 * * * %INPUT_MANIFEST=2.16.0/opensearch-dashboards-2.16.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip
             H 1 * * * %INPUT_MANIFEST=2.16.0/opensearch-2.16.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip
-            H 1 * * * %INPUT_MANIFEST=1.3.18/opensearch-1.3.18.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip
+            H */6 * * * %INPUT_MANIFEST=1.3.18/opensearch-1.3.18.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip
             H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip
             H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-dashboards-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip
         '''

jenkins/docker/docker-build.jenkinsfile

@@ -19,7 +19,7 @@ def docker_args = [
 
 def agent_nodes = [
     "linux": "Jenkins-Agent-Ubuntu2004-X64-M52xlarge-Docker-Builder",
-    "windows": "Jenkins-Agent-Windows2019-X64-C54xlarge-Docker-Builder",
+    "windows": "Jenkins-Agent-Windows2019-X64-M54xlarge-Docker-Builder",
 ]
 
 pipeline {

manifests/1.3.18/opensearch-1.3.18-test.yml

@@ -0,0 +1,84 @@
+---
+schema-version: '1.0'
+name: OpenSearch
+ci:
+  image:
+    name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v3
+    args: -e JAVA_HOME=/opt/java/openjdk-11
+components:
+  - name: index-management
+    integ-test:
+      build-dependencies:
+        - job-scheduler
+      test-configs:
+        - with-security
+        - without-security
+      additional-cluster-configs:
+        path.repo: [/tmp]
+    bwc-test:
+      test-configs:
+        - with-security
+
+  - name: anomaly-detection
+    integ-test:
+      build-dependencies:
+        - job-scheduler
+      test-configs:
+        - with-security
+        - without-security
+    bwc-test:
+      test-configs:
+        - with-security
+
+  - name: asynchronous-search
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+
+  - name: alerting
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+      additional-cluster-configs:
+        plugins.destination.host.deny_list: [10.0.0.0/8, 127.0.0.1]
+    bwc-test:
+      test-configs:
+        - with-security
+
+  - name: sql
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+      additional-cluster-configs:
+        script.context.field.max_compilations_rate: 1000/1m
+    bwc-test:
+      test-configs:
+        - with-security
+
+  - name: k-NN
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+
+  - name: opensearch-reports
+    integ-test:
+      test-configs:
+        - without-security
+
+  - name: opensearch-observability
+    integ-test:
+      test-configs:
+        - without-security
+    bwc-test:
+      test-configs:
+        - with-security
+
+  - name: ml-commons
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security

manifests/1.3.18/opensearch-dashboards-1.3.18-test.yml

@@ -0,0 +1,47 @@
+---
+schema-version: '1.0'
+name: OpenSearch Dashboards
+ci:
+  image:
+    name: opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v4
+components:
+  - name: anomalyDetectionDashboards
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+  - name: ganttChartDashboards
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+  - name: observabilityDashboards
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+  - name: alertingDashboards
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+  - name: indexManagementDashboards
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+  - name: reportsDashboards
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+  - name: securityDashboards
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security
+  - name: queryWorkbenchDashboards
+    integ-test:
+      test-configs:
+        - with-security
+        - without-security

manifests/1.3.18/opensearch-dashboards-1.3.18.yml

@@ -0,0 +1,39 @@
+---
+schema-version: '1.1'
+build:
+  name: OpenSearch Dashboards
+  version: 1.3.18
+ci:
+  image:
+    name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-dashboards-build-v4
+components:
+  - name: OpenSearch-Dashboards
+    repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git
+    ref: '1.3'
+  - name: functionalTestDashboards
+    repository: https://github.com/opensearch-project/opensearch-dashboards-functional-test.git
+    ref: '1.3'
+  - name: anomalyDetectionDashboards
+    repository: https://github.com/opensearch-project/anomaly-detection-dashboards-plugin
+    ref: '1.3'
+  - name: ganttChartDashboards
+    repository: https://github.com/opensearch-project/dashboards-visualizations.git
+    ref: '1.3'
+  - name: observabilityDashboards
+    repository: https://github.com/opensearch-project/dashboards-observability.git
+    ref: '1.3'
+  - name: alertingDashboards
+    repository: https://github.com/opensearch-project/alerting-dashboards-plugin.git
+    ref: '1.3'
+  - name: indexManagementDashboards
+    repository: https://github.com/opensearch-project/index-management-dashboards-plugin
+    ref: '1.3'
+  - name: reportsDashboards
+    repository: https://github.com/opensearch-project/dashboards-reporting.git
+    ref: '1.3'
+  - name: securityDashboards
+    repository: https://github.com/opensearch-project/security-dashboards-plugin.git
+    ref: '1.3'
+  - name: queryWorkbenchDashboards
+    repository: https://github.com/opensearch-project/dashboards-query-workbench.git
+    ref: '1.3'

src/test_workflow/benchmark_test/benchmark_test_suite.py

@@ -74,7 +74,8 @@ def __init__(
                 self.command += f" --telemetry-params '{self.args.telemetry_params}'"
 
         if self.security:
-            self.command += f' --client-options="timeout:300,use_ssl:true,verify_certs:false,basic_auth_user:\'{self.args.username}\',basic_auth_password:\'{self.password}\'" --results-file=final_result.md'
+            self.command += (f' --client-options="timeout:300,use_ssl:true,verify_certs:false,basic_auth_user:\'{self.args.username}\','
+                             f'basic_auth_password:\'{self.password}\'" --results-file=final_result.md')
         else:
             self.command += ' --client-options="timeout:300" --results-file=final_result.md'
 

tests/jenkins/TestBenchmarkPullRequest.groovy

@@ -16,7 +16,6 @@ import static org.hamcrest.CoreMatchers.equalTo
 import static org.hamcrest.CoreMatchers.hasItem
 import static org.hamcrest.CoreMatchers.hasItems
 import static org.hamcrest.MatcherAssert.assertThat
-import hudson.model.Cause.UserIdCause
 
 import static com.lesfurets.jenkins.unit.global.lib.LibraryConfiguration.library
 import static com.lesfurets.jenkins.unit.global.lib.GitSource.gitSource
@@ -36,8 +35,6 @@ class TestBenchmarkPullRequest extends BuildPipelineTest {
                         .retriever(gitSource('https://github.com/opensearch-project/opensearch-build-libraries.git'))
                         .build()
         )
-        //binding.getVariable('currentBuild').rawBuild = [:]
-        //binding.getVariable('currentBuild').rawBuild.getCauses = { return "jenkins.branch.GenericCause@123abc" }
 
         helper.registerAllowedMethod("s3Download", [Map])
         helper.registerAllowedMethod('unstash', [String.class], null)