[Backport 2.0][CI][BWC] add BWC tests to github actions (#1624) (#1659)

.github/workflows/pr_check_workflow.yml

@@ -23,9 +23,11 @@ env:
 jobs:
   build-lint-test:
     runs-on: ubuntu-latest
+    container:
+      image: docker://opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2
+      options: --user 1001
     name: Build and Verify
     steps:
-
       - name: Checkout code
         uses: actions/checkout@v2
 
@@ -61,6 +63,9 @@ jobs:
 
   functional-tests:
     runs-on: ubuntu-latest
+    container:
+      image: docker://opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2
+      options: --user 1001
     name: Run functional tests
     strategy: 
       matrix:
@@ -82,9 +87,9 @@ jobs:
           npm uninstall -g yarn
           npm i -g yarn@1.22.10
 
-      # github virtual env is the latest chrome
+      # image has the latest chrome v99
       - name: Setup chromedriver
-        run: yarn add --dev chromedriver@100.0.0
+        run: yarn add --dev chromedriver@99.0.0
 
       - name: Run bootstrap
         run: yarn osd bootstrap
@@ -100,3 +105,126 @@ jobs:
           CI_PARALLEL_PROCESS_NUMBER: ciGroup${{ matrix.group }}
           JOB: ci${{ matrix.group }}
           CACHE_DIR: ciGroup${{ matrix.group }}
+
+  build-min-artifact-tests:
+    runs-on: ubuntu-latest
+    container:
+      image: docker://opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2
+      options: --user 1001
+    name: Build min release artifacts
+    defaults:
+      run:
+        working-directory: ./artifacts
+    strategy: 
+      matrix:
+        include:
+          - name: Linux x64
+            ext: tar.gz
+            suffix: linux-x64
+            script: build-platform --linux --skip-os-packages
+          - name: Linux ARM64
+            ext: tar.gz
+            suffix: linux-arm64
+            script: build-platform --linux-arm --skip-os-packages
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          path: ./artifacts
+
+      - name: Setup Node
+        uses: actions/setup-node@v2
+        with:
+          node-version-file: "./artifacts/.nvmrc"
+          registry-url: 'https://registry.npmjs.org'
+
+      - name: Setup Yarn
+        run: |
+          npm uninstall -g yarn
+          npm i -g yarn@1.22.10
+
+      - name: Get package version
+        run: |
+          echo "VERSION=$(yarn --silent pkg-version)" >> $GITHUB_ENV
+
+      - name: Get artifact build name
+        run: |
+          echo "ARTIFACT_BUILD_NAME=opensearch-dashboards-${{ env.VERSION }}-${{ matrix.suffix }}.${{ matrix.ext }}" >> $GITHUB_ENV
+
+      - name: Run bootstrap
+        run: yarn osd bootstrap
+
+      - name: Build `${{ matrix.name }}`
+        run: yarn ${{ matrix.script }} --release
+
+      - uses: actions/upload-artifact@v3
+        if: success()
+        with:
+          name: ${{ matrix.suffix }}-${{ env.VERSION }}
+          path: ./artifacts/target/${{ env.ARTIFACT_BUILD_NAME }}
+          retention-days: 1
+
+  bwc-tests:
+    needs: [ build-min-artifact-tests ]
+    runs-on: ubuntu-latest
+    container:
+      image: docker://opensearchstaging/ci-runner:ci-runner-rockylinux8-opensearch-dashboards-integtest-v2
+      options: --user 1001
+    name: Run backwards compatibility tests
+    defaults:
+      run:
+        working-directory: ./artifacts
+    strategy: 
+      matrix:
+        version: [ osd-1.3.2 ]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          path: ./artifacts
+
+      - run: echo Running backwards compatibility tests for version ${{ matrix.version }}
+      - run: echo [NOTE] These tests will be ran using Linux x64 release builds without security
+
+      - name: Setup Node
+        uses: actions/setup-node@v2
+        with:
+          node-version-file: "./artifacts/.nvmrc"
+          registry-url: 'https://registry.npmjs.org'
+
+      - name: Setup Yarn
+        run: |
+          npm uninstall -g yarn
+          npm i -g yarn@1.22.10
+
+      - name: Get package version
+        run: |
+          echo "VERSION=$(yarn --silent pkg-version)" >> $GITHUB_ENV
+
+      - name: Set OpenSearch URL
+        run: |
+          echo "OPENSEARCH_URL=https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/${{ env.VERSION }}/latest/linux/x64/tar/dist/opensearch/opensearch-${{ env.VERSION }}-linux-x64.tar.gz" >> $GITHUB_ENV
+
+      - name: Verify if OpenSearch is available for version
+        id: verify-opensearch-exists
+        run: |
+          if curl -I -L ${{ env.OPENSEARCH_URL }}; then
+            echo "::set-output name=version-exists::true"
+          fi
+
+      - name: Skipping tests
+        if: steps.verify-opensearch-exists.outputs.version-exists != 'true'
+        run: echo Tests were skipped because an OpenSearch release build does not exist for this version yet!
+
+      - name: Download OpenSearch Dashboards
+        uses: actions/download-artifact@v3
+        id: download
+        with:
+          name: linux-x64-${{ env.VERSION }}
+          path: ./artifacts/
+        if: steps.verify-opensearch-exists.outputs.version-exists == 'true'
+
+      - name: Run tests
+        if: steps.verify-opensearch-exists.outputs.version-exists == 'true'
+        run: |
+          yarn test:bwc -s false -o ${{ env.OPENSEARCH_URL }} -d ${{ steps.download.outputs.download-path }}/opensearch-dashboards-${{ env.VERSION }}-linux-x64.tar.gz -v ${{ matrix.version }}

TESTING.md

@@ -67,6 +67,12 @@ To run specific versions' backwards compatibility tests, pass the versions to th
 
 `yarn test:bwc -o [test path to opensearch.tar.gz] -d [test path to opensearch-dashboards.tar.gz] -v "[test versions]"`
 
+To generate test data that will be utilized for backwards compatibility tests:
+
+`yarn test:bwc -o [test path to opensearch.tar.gz] -d [test path to opensearch-dashboards.tar.gz] -g true`
+
+This will create an archive of the data based on the OpenSearch Dashboards version you have provided. For example, if a tarball of 2.0.0 was passed then an `osd-2.0.0.zip` will be created. This command is intended to be executed when needed per a version. For example, when end-users cannot migrate directly from `vPrevious` to `vNext`. If `osd-vCurrent.zip` does not exist, then this command be ran and the output sourced controlled for future use.
+
 ### Additional checks
 Make sure you run lint checker before submitting a pull request. To run lint checker:
 `node scripts/precommit_hook.js --fix`

bwctest.sh

@@ -13,7 +13,7 @@
 
 set -e
 
-DEFAULT_VERSIONS="osd-1.1.0,odfe-1.13.2,odfe-1.0.2"
+DEFAULT_VERSIONS="osd-1.3.2,"
 
 function usage() {
     echo ""
@@ -75,6 +75,7 @@ done
 [ -z "$SECURITY_ENABLED" ] && SECURITY_ENABLED="false"
 [ -z "$CREDENTIAL" ] && CREDENTIAL="admin:admin"
 [ -z "$CI" ] && CI=1
+[ -z "$BWC_VERSIONS" ] && BWC_VERSIONS=$DEFAULT_VERSIONS
 
 # If no OpenSearch build was passed then this constructs the version
 if [ -z "$OPENSEARCH" ]; then
@@ -96,4 +97,4 @@ if [ -z "$OPENSEARCH" ]; then
     OPENSEARCH="https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/$VERSION/latest/$PLATFORM/$ARCH/tar/dist/opensearch/opensearch-$VERSION-$PLATFORM-$ARCH.tar.gz"
 fi
 
-source scripts/bwctest_osd.sh -b $BIND_ADDRESS -p $BIND_PORT -s $SECURITY_ENABLED -c $CREDENTIAL -o $OPENSEARCH -d $DASHBOARDS -v $DEFAULT_VERSIONS
+source scripts/bwctest_osd.sh -b $BIND_ADDRESS -p $BIND_PORT -s $SECURITY_ENABLED -c $CREDENTIAL -o $OPENSEARCH -d $DASHBOARDS -v $BWC_VERSIONS

cypress/integration/with-security/check_loaded_data.js

@@ -47,7 +47,9 @@ describe('check previously loaded data', () => {
         .get('[data-test-subj="dashboardListingTitleLink-[Flights]-Global-Flight-Dashboard"]')
         .click();
       commonUI.removeAllFilters();
-      commonUI.setDateRange('Dec 1, 2021 @ 00:00:00.000', 'Nov 1, 2021 @ 00:00:00.000');
+      commonUI.setDateRange(
+        `Dec 31, ${new Date().getFullYear()} @ 00:00:00.000', 'Nov 1, 2016 @ 00:00:00.000`
+      );
     });
 
     it('Global Flight Dashboard is loaded and funtions correctly', () => {
@@ -77,7 +79,9 @@ describe('check previously loaded data', () => {
         .get('[data-test-subj="dashboardListingTitleLink-[eCommerce]-Revenue-Dashboard"]')
         .click();
       commonUI.removeAllFilters();
-      commonUI.setDateRange('Nov 1, 2021 @ 00:00:00.000', 'Nov 1, 2016 @ 00:00:00.000');
+      commonUI.setDateRange(
+        `Dec 31, ${new Date().getFullYear()} @ 00:00:00.000', 'Nov 1, 2016 @ 00:00:00.000`
+      );
     });
 
     it('eCommerce Revenue Dashboard is loaded and functions correctly', () => {

cypress/integration/with-security/helpers/generate_data.js

@@ -0,0 +1,86 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import {
+  MiscUtils,
+  LoginPage,
+} from '@opensearch-dashboards-test/opensearch-dashboards-test-library';
+
+const miscUtils = new MiscUtils(cy);
+const loginPage = new LoginPage(cy);
+
+describe('Generating BWC test data with security', () => {
+  beforeEach(() => {
+    miscUtils.visitPage('app/management/opensearch-dashboards/settings');
+    loginPage.enterUserName('admin');
+    loginPage.enterPassword('admin');
+    loginPage.submit();
+  });
+
+  afterEach(() => {
+    cy.clearCookies();
+  });
+
+  it('tenant-switch-modal page should show and be clicked', () => {
+    cy.get('[data-test-subj="tenant-switch-modal"]');
+    cy.get('[data-test-subj="confirm"]').click();
+  });
+
+  it('adds advanced settings', () => {
+    miscUtils.visitPage('app/management/opensearch-dashboards/settings');
+    cy.get('[data-test-subj="advancedSetting-editField-theme:darkMode"]').click();
+    cy.get('[data-test-subj="advancedSetting-editField-timeline:default_columns"]').type(
+      '{selectAll}4'
+    );
+    cy.get('[data-test-subj="advancedSetting-editField-timeline:max_buckets"]').type(
+      '{selectAll}4'
+    );
+    cy.get('[data-test-subj="advancedSetting-editField-defaultRoute"]')
+      .clear()
+      .type('/app/opensearch_dashboards_overview#/');
+    cy.get('[data-test-subj="advancedSetting-saveButton"]').click({ force: true });
+    cy.reload();
+  });
+
+  it('adds sample data', () => {
+    miscUtils.addSampleData();
+  });
+
+  it('adds filters and queries', () => {
+    miscUtils.visitPage('app/dashboards#');
+    cy.get('[data-test-subj="dashboardListingTitleLink-[Logs]-Web-Traffic"]').click();
+    cy.get('[data-test-subj="queryInput"]').clear().type('resp=200');
+    cy.get('[data-test-subj="addFilter"]').click();
+    cy.get('[data-test-subj="filterFieldSuggestionList"]')
+      .find('[data-test-subj="comboBoxInput"]')
+      .type('machine.os{downArrow}{enter}');
+    cy.get('[data-test-subj="filterOperatorList"]')
+      .find('[data-test-subj="comboBoxInput"]')
+      .type('{downArrow}{enter}');
+    cy.get('[data-test-subj="filterParams"]')
+      .find('input.euiFieldText[placeholder="Enter a value"]')
+      .type('osx');
+    cy.get('[data-test-subj="createCustomLabel"]').click();
+    cy.get('[class="globalFilterItem__editorForm"]').find('input').last().type('osx filter');
+    cy.get('[data-test-subj="saveFilter"]').click();
+    cy.get('[data-test-subj="saved-query-management-popover-button"]').click();
+    cy.get('[data-test-subj="saved-query-management-popover"]')
+      .find('[data-test-subj="saved-query-management-save-button"]')
+      .click();
+    cy.get('[data-test-subj="saveQueryFormTitle"]').type('test-query');
+    cy.get('[data-test-subj="savedQueryFormSaveButton"]').click();
+  });
+
+  it('adds Timeline visualization', () => {
+    miscUtils.visitPage('app/visualize#');
+    cy.get('[data-test-subj="visualizationLandingPage"]')
+      .find('[data-test-subj="newItemButton"]')
+      .click();
+    cy.get('[data-test-subj="visType-timelion"]').click();
+    cy.get('[data-test-subj="visualizeSaveButton"]').click();
+    cy.get('[data-test-subj="savedObjectTitle"]').type('test-timeline');
+    cy.get('[data-test-subj="confirmSaveSavedObjectButton"]').click();
+  });
+});

cypress/integration/without-security/check_loaded_data.js

@@ -38,7 +38,9 @@ describe('check previously loaded data', () => {
         .get('[data-test-subj="dashboardListingTitleLink-[Flights]-Global-Flight-Dashboard"]')
         .click();
       commonUI.removeAllFilters();
-      commonUI.setDateRange('Dec 1, 2021 @ 00:00:00.000', 'Nov 1, 2021 @ 00:00:00.000');
+      commonUI.setDateRange(
+        `Dec 31, ${new Date().getFullYear()} @ 00:00:00.000', 'Nov 1, 2016 @ 00:00:00.000`
+      );
     });
 
     it('Global Flight Dashboard is loaded when clicked', () => {
@@ -71,7 +73,9 @@ describe('check previously loaded data', () => {
         .get('[data-test-subj="dashboardListingTitleLink-[eCommerce]-Revenue-Dashboard"]')
         .click();
       commonUI.removeAllFilters();
-      commonUI.setDateRange('Nov 1, 2021 @ 00:00:00.000', 'Nov 1, 2016 @ 00:00:00.000');
+      commonUI.setDateRange(
+        `Dec 31, ${new Date().getFullYear()} @ 00:00:00.000', 'Nov 1, 2016 @ 00:00:00.000`
+      );
     });
 
     it('eCommerce Revenue Dashboard is loaded when clicked', () => {

cypress/integration/without-security/helpers/generate_data.js

@@ -0,0 +1,69 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { MiscUtils } from '@opensearch-dashboards-test/opensearch-dashboards-test-library';
+
+const miscUtils = new MiscUtils(cy);
+
+describe('Generating BWC test data without security', () => {
+  before(() => {
+    miscUtils.visitPage('app');
+  });
+  it('adds advanced settings', () => {
+    miscUtils.visitPage('app/management/opensearch-dashboards/settings');
+    cy.get('[data-test-subj="advancedSetting-editField-theme:darkMode"]').click();
+    cy.get('[data-test-subj="advancedSetting-editField-timeline:default_columns"]').type(
+      '{selectAll}4'
+    );
+    cy.get('[data-test-subj="advancedSetting-editField-timeline:max_buckets"]').type(
+      '{selectAll}4'
+    );
+    cy.get('[data-test-subj="advancedSetting-editField-defaultRoute"]')
+      .clear()
+      .type('/app/opensearch_dashboards_overview#/');
+    cy.get('[data-test-subj="advancedSetting-saveButton"]').click({ force: true });
+    cy.reload();
+  });
+
+  it('adds sample data', () => {
+    miscUtils.addSampleData();
+  });
+
+  it('adds filters and queries', () => {
+    miscUtils.visitPage('app/dashboards#');
+    cy.get('[data-test-subj="dashboardListingTitleLink-[Logs]-Web-Traffic"]').click();
+    cy.get('[data-test-subj="queryInput"]').clear().type('resp=200');
+    cy.get('[data-test-subj="addFilter"]').click();
+    cy.get('[data-test-subj="filterFieldSuggestionList"]')
+      .find('[data-test-subj="comboBoxInput"]')
+      .type('machine.os{downArrow}{enter}');
+    cy.get('[data-test-subj="filterOperatorList"]')
+      .find('[data-test-subj="comboBoxInput"]')
+      .type('{downArrow}{enter}');
+    cy.get('[data-test-subj="filterParams"]')
+      .find('input.euiFieldText[placeholder="Enter a value"]')
+      .type('osx');
+    cy.get('[data-test-subj="createCustomLabel"]').click();
+    cy.get('[class="globalFilterItem__editorForm"]').find('input').last().type('osx filter');
+    cy.get('[data-test-subj="saveFilter"]').click();
+    cy.get('[data-test-subj="saved-query-management-popover-button"]').click();
+    cy.get('[data-test-subj="saved-query-management-popover"]')
+      .find('[data-test-subj="saved-query-management-save-button"]')
+      .click();
+    cy.get('[data-test-subj="saveQueryFormTitle"]').type('test-query');
+    cy.get('[data-test-subj="savedQueryFormSaveButton"]').click();
+  });
+
+  it('adds Timeline visualization', () => {
+    miscUtils.visitPage('app/visualize#');
+    cy.get('[data-test-subj="visualizationLandingPage"]')
+      .find('[data-test-subj="newItemButton"]')
+      .click();
+    cy.get('[data-test-subj="visType-timelion"]').click();
+    cy.get('[data-test-subj="visualizeSaveButton"]').click();
+    cy.get('[data-test-subj="savedObjectTitle"]').type('test-timeline');
+    cy.get('[data-test-subj="confirmSaveSavedObjectButton"]').click();
+  });
+});