From 2acc98e5eb021194f2226de06de259fac67c11b5 Mon Sep 17 00:00:00 2001 From: avifenesh Date: Sun, 21 Jul 2024 16:01:05 +0000 Subject: [PATCH 01/12] limit pr run to ubuntu latest X first and last version supported, added schedule and label runs Signed-off-by: avifenesh --- .github/json_matrices/engine-matrix.json | 27 ++-- .github/workflows/java.yml | 98 +++++++++---- .github/workflows/node.yml | 160 +++++++++++++++------ .github/workflows/python.yml | 169 +++++++++++++++++++---- 4 files changed, 347 insertions(+), 107 deletions(-) diff --git a/.github/json_matrices/engine-matrix.json b/.github/json_matrices/engine-matrix.json index bf755b782e..0e47a0b0ef 100644 --- a/.github/json_matrices/engine-matrix.json +++ b/.github/json_matrices/engine-matrix.json @@ -1,10 +1,21 @@ [ - { - "type": "valkey", - "version": "7.2.5" - }, - { - "type": "valkey", - "version": "8.0.0-rc1" - } + { + "type": "valkey", + "version": "8.0.0-rc1" + "run": "always" + }, + { + "type": "valkey", + "version": "7.2.5", + "run": "always" + }, + { + "type": "valkey", + "version": "redis-7.0.15" + }, + { + "type": "valkey", + "version": "redis-6.2.14", + "run": "always" + } ] diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 5106aec4ce..c2a181fd7d 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -26,46 +26,90 @@ on: - .github/json_matrices/build-matrix.json workflow_dispatch: + pull_request_target: + types: [labeled] + paths: + - glide-core/src/** + - submodules/** + - java/** + - .github/workflows/java.yml + - .github/workflows/install-shared-dependencies/action.yml + - .github/workflows/test-benchmark/action.yml + - .github/workflows/lint-rust/action.yml + - .github/workflows/install-valkey/action.yml + - .github/json_matrices/build-matrix.json + schedule: + - cron: "0 0 * * *" + concurrency: group: java-${{ github.head_ref || github.ref }} cancel-in-progress: true jobs: - load-engine-matrix: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.load-engine-matrix.outputs.matrix }} - steps: - - name: Checkout - uses: actions/checkout@v4 + check-label: + runs-on: ubuntu-latest + # If the trigger is a label we want to check if it is a CoreChanges label, + # if it is we want to run the job in the AWS_ACTIONS environment so its need to be approved + environment: ${{ github.event_name == 'pull_request_target' && 'AWS_ACTIONS' || '' }} + steps: + - name: Check for CoreChanges label + if: github.event_name == 'pull_request_target' && github.event.label.name != 'CoreChanges' + run: | + echo "This workflow only runs for the CoreChanges label on pull requests" + exit 1 - - name: Load the engine matrix - id: load-engine-matrix - shell: bash - run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + load-engine-matrix: + runs-on: ubuntu-latest + # We want to run this job only if the previous job outputs should_run=true, + # Which means that if the trigger was a label we run this job just if the label was CoreChanges + needs: check-label + outputs: + matrix: ${{ steps.load-engine-matrix.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Load the engine matrix + id: load-engine-matrix + shell: bash + run: | + if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" ]]; then + echo "matrix=$(jq -c '[.[] | select(.run == "always")]' < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + else + echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + fi + + create-host-matrix: + runs-on: ubuntu-latest + needs: check-label + outputs: + matrix: ${{ steps.create-host-matrix.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Create host matrix + id: create-host-matrix + shell: bash + run: | + if [[ "${{ github.event_name }}" == "pull_request" || "${{github.event_name }}" == "push" ]]; then + echo 'matrix={"include":[{"OS":"ubuntu","RUNNER":"ubuntu-latest","TARGET":"x86_64-unknown-linux-gnu"}]}' >> $GITHUB_OUTPUT + else + echo 'matrix={"include":[{"OS":"ubuntu","RUNNER":"ubuntu-latest","TARGET":"x86_64-unknown-linux-gnu"},{"OS":"macos","RUNNER":"macos-latest","TARGET":"aarch64-apple-darwin"}]}' >> $GITHUB_OUTPUT + fi build-and-test-java-client: - needs: load-engine-matrix + needs: [load-engine-matrix, create-host-matrix] timeout-minutes: 35 strategy: # Run all jobs fail-fast: false matrix: java: - # - 11 + - 11 - 17 engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} - host: - - { - OS: ubuntu, - RUNNER: ubuntu-latest, - TARGET: x86_64-unknown-linux-gnu - } - # - { - # OS: macos, - # RUNNER: macos-latest, - # TARGET: aarch64-apple-darwin - # } + host: ${{ fromJson(needs.create-host-matrix.outputs.matrix).include }} runs-on: ${{ matrix.host.RUNNER }} @@ -122,15 +166,16 @@ jobs: java/client/build/reports/spotbugs/** build-amazonlinux-latest: - if: github.repository_owner == 'valkey-io' + if: github.repository_owner == 'valkey-io' && (github.event_name == 'schedule' || github.event.label.name == 'CoreChanges') strategy: # Run all jobs fail-fast: false matrix: java: - # - 11 + - 11 - 17 runs-on: ubuntu-latest + needs: check-label container: amazonlinux:latest timeout-minutes: 35 steps: @@ -186,6 +231,7 @@ jobs: lint-rust: timeout-minutes: 15 runs-on: ubuntu-latest + needs: check-label steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 32db45e5c5..cf8693878b 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -30,6 +30,23 @@ on: - .github/json_matrices/build-matrix.json workflow_dispatch: + pull_request_target: + types: [labeled] + paths: + - glide-core/src/** + - submodules/** + - node/** + - utils/cluster_manager.py + - .github/workflows/node.yml + - .github/workflows/build-node-wrapper/action.yml + - .github/workflows/install-shared-dependencies/action.yml + - .github/workflows/test-benchmark/action.yml + - .github/workflows/lint-rust/action.yml + - .github/workflows/install-valkey/action.yml + - .github/json_matrices/build-matrix.json + schedule: + - cron: "0 1 * * *" + concurrency: group: node-${{ github.head_ref || github.ref }} cancel-in-progress: true @@ -38,37 +55,81 @@ env: CARGO_TERM_COLOR: always jobs: - load-engine-matrix: + check-label: runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.load-engine-matrix.outputs.matrix }} + # If the trigger is a label we want to check if it is a CoreChanges label, + # if it is we want to run the job in the AWS_ACTIONS environment so its need to be approved + environment: ${{ github.event_name == 'pull_request_target' && 'AWS_ACTIONS' || '' }} steps: - - name: Checkout - uses: actions/checkout@v4 + - name: Check for CoreChanges label + if: github.event_name == 'pull_request_target' && github.event.label.name != 'CoreChanges' + run: | + echo "This workflow only runs for the CoreChanges label on pull requests" + exit 1 + + load-engine-matrix: + runs-on: ubuntu-latest + # We want to run this job only if the previous job outputs should_run=true, + # Which means that if the trigger was a label we run this job just if the label was CoreChanges + needs: check-label + outputs: + matrix: ${{ steps.load-engine-matrix.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Load the engine matrix + id: load-engine-matrix + shell: bash + # We want to run the full matrix only for scheduled runs and CoreChanges + # For other events we want to run only the always matrix which is first and last version supported + run: | + if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" ]]; then + echo "matrix=$(jq -c '[.[] | select(.run == "always")]' < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + else + echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + fi + create-version-matrix: + runs-on: ubuntu-latest + needs: check-label + outputs: + matrix: ${{ steps.create-version-matrix.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Create version matrix + id: create-version-matrix + shell: bash + # We want to run the full matrix only for scheduled runs and CoreChanges + # For other events we want to run only the always matrix which is first and last version supported + run: | + if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" ]]; then + echo 'matrix={"node":["16.x","20.x"]}' >> $GITHUB_OUTPUT + else + echo 'matrix={"node":["16.x","17.x","18.x","19.x","20.x"]}' >> $GITHUB_OUTPUT + fi - - name: Load the engine matrix - id: load-engine-matrix - shell: bash - run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT test-ubuntu-latest: runs-on: ubuntu-latest - needs: load-engine-matrix - timeout-minutes: 25 + needs: [load-engine-matrix, create-version-matrix] + timeout-minutes: 15 strategy: fail-fast: false matrix: engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} - + node: ${{ fromJson(needs.create-version-matrix.outputs.matrix).node }} + steps: - uses: actions/checkout@v4 with: submodules: recursive - - name: Use Node.js 16.x + - name: Setup Node uses: actions/setup-node@v3 with: - node-version: 16.x + node-version: ${{ matrix.node }} - name: Build Node wrapper uses: ./.github/workflows/build-node-wrapper @@ -118,6 +179,7 @@ jobs: lint-rust: timeout-minutes: 15 runs-on: ubuntu-latest + needs: check-label steps: - uses: actions/checkout@v4 with: @@ -128,39 +190,41 @@ jobs: cargo-toml-folder: ./node/rust-client name: lint node rust - # build-macos-latest: - # runs-on: macos-latest - # timeout-minutes: 25 - # steps: - # - uses: actions/checkout@v4 - # with: - # submodules: recursive - # - name: Set up Homebrew - # uses: Homebrew/actions/setup-homebrew@master - - # - name: Install NodeJS - # run: | - # brew update - # brew upgrade || true - # brew install node - - # - name: Downgrade npm major version to 8 - # run: | - # npm i -g npm@8 - - # - name: Build Node wrapper - # uses: ./.github/workflows/build-node-wrapper - # with: - # os: "macos" - # named_os: "darwin" - # arch: "arm64" - # target: "aarch64-apple-darwin" - # github-token: ${{ secrets.GITHUB_TOKEN }} - # engine-version: "7.2.5" + build-macos-latest: + if: github.event_name == 'schedule' || github.event.label.name == 'CoreChanges' + runs-on: macos-latest + needs: check-label + timeout-minutes: 25 + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Set up Homebrew + uses: Homebrew/actions/setup-homebrew@master + + - name: Install NodeJS + run: | + brew update + brew upgrade || true + brew install node + + - name: Downgrade npm major version to 8 + run: | + npm i -g npm@8 - # - name: Test compatibility - # run: npm test -- -t "set and get flow works" - # working-directory: ./node + - name: Build Node wrapper + uses: ./.github/workflows/build-node-wrapper + with: + os: "macos" + named_os: "darwin" + arch: "arm64" + target: "aarch64-apple-darwin" + github-token: ${{ secrets.GITHUB_TOKEN }} + engine-version: "7.2.5" + + - name: Test compatibility + run: npm test -- -t "set and get flow works" + working-directory: ./node # - name: Upload test reports # if: always() @@ -174,8 +238,10 @@ jobs: # benchmarks/results/** build-amazonlinux-latest: + if: github.event_name == 'schedule' || github.event.label.name == 'CoreChanges' runs-on: ubuntu-latest container: amazonlinux:latest + needs: check-label timeout-minutes: 15 steps: - name: Install git @@ -221,7 +287,9 @@ jobs: benchmarks/results/** build-and-test-linux-musl-on-x86: + if: github.event_name == 'schedule' || github.event.label.name == 'CoreChanges' name: Build and test Node wrapper on Linux musl + needs: check-label runs-on: ubuntu-latest container: image: node:alpine diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 2511c8c1f4..0b88b03c88 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -35,6 +35,23 @@ on: - .github/workflows/start-self-hosted-runner/action.yml workflow_dispatch: + pull_request_target: + types: [labeled] + paths: + - python/** + - glide-core/src/** + - submodules/** + - utils/cluster_manager.py + - .github/workflows/python.yml + - .github/workflows/build-python-wrapper/action.yml + - .github/workflows/install-shared-dependencies/action.yml + - .github/workflows/test-benchmark/action.yml + - .github/workflows/lint-rust/action.yml + - .github/workflows/install-valkey/action.yml + - .github/json_matrices/build-matrix.json + schedule: + - cron: "0 2 * * *" + concurrency: group: python-${{ github.head_ref || github.ref }} cancel-in-progress: true @@ -45,45 +62,71 @@ permissions: id-token: write jobs: + check-label: + runs-on: ubuntu-latest + # If the trigger is a label we want to check if it is a CoreChanges label, + # if it is we want to run the job in the AWS_ACTIONS environment so its need to be approved + environment: ${{ github.event_name == 'pull_request_target' && 'AWS_ACTIONS' || '' }} + steps: + - name: Check for CoreChanges label + if: github.event_name == 'pull_request_target' && github.event.label.name != 'CoreChanges' + run: | + echo "This workflow only runs for the CoreChanges label on pull requests" + exit 1 + load-engine-matrix: + runs-on: ubuntu-latest + # We want to run this job only if the previous job outputs should_run=true, + # Which means that if the trigger was a label we run this job just if the label was CoreChanges + needs: check-label + outputs: + matrix: ${{ steps.load-engine-matrix.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Load the engine matrix + id: load-engine-matrix + shell: bash + # We want to run the full matrix only for scheduled runs and CoreChanges + # For other events we want to run only the always matrix which is first and last version supported + run: | + if [[ "${{ github.event_name }}" == "pull_request" || "${{github.event_name }}" == "push" ]]; then + echo "matrix=$(jq -c '[.[] | select(.run == "always")]' < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + else + echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + fi + create-version-matrix: runs-on: ubuntu-latest + needs: check-label outputs: - matrix: ${{ steps.load-engine-matrix.outputs.matrix }} + matrix: ${{ steps.create-version-matrix.outputs.matrix }} steps: - name: Checkout uses: actions/checkout@v4 - - - name: Load the engine matrix - id: load-engine-matrix + + - name: Create version matrix + id: create-version-matrix + shell: bash - run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT - - test: - runs-on: ${{ matrix.host.RUNNER }} - needs: load-engine-matrix + # We want to run the full matrix only for scheduled runs and CoreChanges + # For other events we want to run only the always matrix which is first and last version supported + run: | + if [[ "${{ github.event_name }}" == "pull_request" || "${{github.event_name }}" == "push" ]]; then + echo 'matrix={"python":["3.8","3.12"]}' >> $GITHUB_OUTPUT + else + echo 'matrix={"python":["3.8","3.9","3.10","3.11","3.12"]}' >> $GITHUB_OUTPUT + fi + + test-ubuntu-latest: + runs-on: ubuntu-latest + needs: [load-engine-matrix, create-version-matrix] timeout-minutes: 35 strategy: fail-fast: false matrix: engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} - python: - # - "3.8" - # - "3.9" - # - "3.10" - # - "3.11" - - "3.12" - host: - - { - OS: ubuntu, - RUNNER: ubuntu-latest, - TARGET: x86_64-unknown-linux-gnu - } - # - { - # OS: macos, - # RUNNER: macos-latest, - # TARGET: aarch64-apple-darwin - # } - + python: ${{ fromJson(needs.create-version-matrix.outputs.matrix).python }} steps: - uses: actions/checkout@v4 with: @@ -140,6 +183,15 @@ jobs: python/python/tests/pytest_report.html utils/clusters/** benchmarks/results/** + test-pubsub-ubuntu-latest: + runs-on: ubuntu-latest + needs: [load-engine-matrix, create-version-matrix] + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} + python: ${{ fromJson(needs.create-version-matrix.outputs.matrix).python }} test-pubsub: runs-on: ${{ matrix.host.RUNNER }} @@ -203,6 +255,7 @@ jobs: lint: runs-on: ubuntu-latest + needs: check-label timeout-minutes: 15 steps: - uses: actions/checkout@v4 @@ -241,10 +294,72 @@ jobs: working-directory: ./python run: | black --check --diff . + test-macos-latest: + if: github.event_name == 'schedule' || github.event.label.name == 'CoreChanges' + runs-on: macos-latest + needs: load-engine-matrix + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Set up Homebrew + uses: Homebrew/actions/setup-homebrew@master + + - name: Build Python wrapper + uses: ./.github/workflows/build-python-wrapper + with: + os: "macos" + target: "aarch64-apple-darwin" + github-token: ${{ secrets.GITHUB_TOKEN }} + engine-version: ${{ matrix.engine.version }} + + - name: Test with pytest + working-directory: ./python + run: | + source .env/bin/activate + pytest --asyncio-mode=auto + + test-pubsub-macos-latest: + if: github.event_name == 'schedule' || github.event.label.name == 'CoreChanges' + runs-on: macos-latest + needs: load-engine-matrix + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Set up Homebrew + uses: Homebrew/actions/setup-homebrew@master + + - name: Build Python wrapper + uses: ./.github/workflows/build-python-wrapper + with: + os: "macos" + target: "aarch64-apple-darwin" + github-token: ${{ secrets.GITHUB_TOKEN }} + engine-version: ${{ matrix.engine.version }} + + - name: Test pubsub with pytest + working-directory: ./python + run: | + source .env/bin/activate + cd python/tests/ + pytest --asyncio-mode=auto -k test_pubsub build-amazonlinux-latest: + if: github.event_name == 'schedule' || github.event.label.name == 'CoreChanges' runs-on: ubuntu-latest container: amazonlinux:latest + needs: check-label timeout-minutes: 15 steps: - name: Install git From ea259747317384117c5fd8a0be527c1f1a9304ed Mon Sep 17 00:00:00 2001 From: avifenesh Date: Sun, 28 Jul 2024 12:38:57 +0000 Subject: [PATCH 02/12] limit CI Signed-off-by: avifenesh --- .github/DEVELOPER.md | 130 +++++++++++ .github/json_matrices/build-matrix.json | 18 +- .github/json_matrices/engine-matrix.json | 8 +- .../supported-languages-versions.json | 17 ++ .../workflows/create-test-matrices/action.yml | 54 +++++ .github/workflows/csharp.yml | 23 +- .github/workflows/java.yml | 112 +++------- .github/workflows/lint-rust/action.yml | 2 - .github/workflows/node.yml | 179 ++++----------- .github/workflows/python.yml | 207 +++--------------- .prettierignore | 1 + .vscode/settings.json | 2 +- 12 files changed, 339 insertions(+), 414 deletions(-) create mode 100644 .github/DEVELOPER.md create mode 100644 .github/json_matrices/supported-languages-versions.json create mode 100644 .github/workflows/create-test-matrices/action.yml create mode 100644 .prettierignore diff --git a/.github/DEVELOPER.md b/.github/DEVELOPER.md new file mode 100644 index 0000000000..53e4116220 --- /dev/null +++ b/.github/DEVELOPER.md @@ -0,0 +1,130 @@ +# CI/CD Workflow Guide + +TODO: Add a description of the CI/CD workflow and its components. + +### Overview + +Our CI/CD pipeline tests and builds our project across multiple languages, versions, and environments. This guide outlines the key components and processes of our workflow. + +### Workflow Triggers + +* Push to `main` branch +* Pull requests +* Scheduled runs (daily) +* Manual trigger (workflow_dispatch) + +### Language-Specific Workflows + +Each language has its own workflow file with similar structure but language-specific steps, for example python.yml for Python, or java.yml for Java. + +### Shared Components + +#### Matrix Files + +While workflows are language-specific, the matrix files are shared across all workflows. +Workflows are starting by loading the matrix files from the `.github/json_matrices` directory. + +* `engine-matrix.json`: Defines the versions of Valkey engine to test against. +* `build-matrix.json`: Defines the host environments for testing. +* `supported-languages-version.json`: Defines the supported versions of languages. + +All matrices have a `run` like field which specifies if the configuration should be tested on every workflow run. +This allows for flexible control over which configurations are tested in different scenarios, optimizing CI/CD performance and resource usage. + +#### Engine Matrix (engine-matrix.json) + +Defines the versions of Valkey engine to test against: + +```json +[ + { "type": "valkey", "version": "7.2.5", "run": "always" }, +] +``` +* `type`: The type of engine (e.g., Valkey, Redis). +* `version`: The version of the engine. +* `run`: Specifies if the engine version should be tested on every workflow. + +#### Build Matrix (build-matrix.json) + +Defines the host environments for testing: + +```json +[ + { + "OS": "ubuntu", + "RUNNER": "ubuntu-latest", + "TARGET": "x86_64-unknown-linux-gnu", + "run": ["always", "python", "node", "java"] + } + // ... other configurations +] +``` +* `OS`: The operating system of the host. +* `RUNNER`: The GitHub runner to use. +* `TARGET`: The target environment. +* `run`: Specifies which language workflows should use this host configuration, always means run on each workflow trigger. + +#### Supported Languages Version (supported-languages-version.json) + +Defines the supported versions of languages: + +```json +[ + { + "language": "java", + "versions": ["11", "17"], + "always-run-versions": ["17"] + }, + // ... other configurations +] +``` +* `language`: The language for which the version is supported. +* `versions`: The full versions supported of the language which will test against scheduled. +* `always-run-versions`: The versions which will be tested in every workflow run. + +#### Triggering Workflows + +Push to main or create a pull request to run workflows automatically. +Use workflow_dispatch for manual triggers, accepting inputs of full-matrix which is a boolean value to run all configurations. +Scheduled runs are triggered daily to ensure regular testing of all configurations. + +### Mutual vs. Language-Specific Components + +#### Mutual + +`Matrix files` - `.github/json_matrices` +`Shared dependencies installation` - `.github/workflows/install-shared-dependencies/action.yml` +`Linting Rust` - `.github/workflows/lint-rust/action.yml` + +#### Language-Specific + +`Package manager commands` +`Testing frameworks` +`Build processes` + +### Customizing Workflows + +Modify `[language].yml` files to adjust language-specific steps. +Update matrix files to change tested versions or environments. +Adjust cron schedules in workflow files for different timing of scheduled runs. + +### Workflow Matrices + +We use dynamic matrices for our CI/CD workflows, which are created using the `create-test-matrices` action. This action is defined in `.github/workflows/create-test-matrices/action.yml`. + +#### How it works + +1. The action is called with a `language-name` input and `dispatch-run-full-matrix` input. +2. It reads the `engine-matrix.json`, `build-matrix.json`, and `supported-languages-version.json` files. +3. It filters the matrices based on the inputs and the event type. +2. It generates three matrices: + - Engine matrix: Defines the types and versions of the engine to test against, for example Valkey 7.2.5. + - Host matrix: Defines the host platforms to run the tests on, for example Ubuntu on ARM64. + - Language-version matrix: Defines the supported versions of languages, for example python 3.8. + +#### Outputs +- `engine-matrix-output`: The generated engine matrix. +- `host-matrix-output`: The generated host matrix. +- `language-version-matrix-output`: The generated language version matrix. + +This dynamic matrix generation allows for flexible and efficient CI/CD workflows, adapting the test configurations based on the type of change and the specific language being tested. diff --git a/.github/json_matrices/build-matrix.json b/.github/json_matrices/build-matrix.json index e8589b84f8..f776ad3c64 100644 --- a/.github/json_matrices/build-matrix.json +++ b/.github/json_matrices/build-matrix.json @@ -5,7 +5,8 @@ "RUNNER": "ubuntu-latest", "ARCH": "x64", "TARGET": "x86_64-unknown-linux-gnu", - "PACKAGE_MANAGERS": ["pypi", "npm"] + "PACKAGE_MANAGERS": ["pypi", "npm"], + "run": ["always", "python", "node", "java"] }, { "OS": "ubuntu", @@ -14,7 +15,8 @@ "ARCH": "arm64", "TARGET": "aarch64-unknown-linux-gnu", "PACKAGE_MANAGERS": ["pypi", "npm"], - "CONTAINER": "2_28" + "CONTAINER": "2_28", + "run": ["python", "node", "java"] }, { "OS": "macos", @@ -22,7 +24,8 @@ "RUNNER": "macos-12", "ARCH": "x64", "TARGET": "x86_64-apple-darwin", - "PACKAGE_MANAGERS": ["pypi", "npm"] + "PACKAGE_MANAGERS": ["pypi", "npm"], + "run": ["python", "node", "java"] }, { "OS": "macos", @@ -30,7 +33,8 @@ "RUNNER": "macos-latest", "ARCH": "arm64", "TARGET": "aarch64-apple-darwin", - "PACKAGE_MANAGERS": ["pypi", "npm"] + "PACKAGE_MANAGERS": ["pypi", "npm"], + "run": ["python", "node", "java"] }, { "OS": "ubuntu", @@ -40,7 +44,8 @@ "RUNNER": ["self-hosted", "Linux", "ARM64"], "IMAGE": "node:alpine", "CONTAINER_OPTIONS": "--user root --privileged --rm", - "PACKAGE_MANAGERS": ["npm"] + "PACKAGE_MANAGERS": ["npm"], + "run": ["node"] }, { "OS": "ubuntu", @@ -50,6 +55,7 @@ "RUNNER": "ubuntu-latest", "IMAGE": "node:alpine", "CONTAINER_OPTIONS": "--user root --privileged", - "PACKAGE_MANAGERS": ["npm"] + "PACKAGE_MANAGERS": ["npm"], + "run": ["node"] } ] diff --git a/.github/json_matrices/engine-matrix.json b/.github/json_matrices/engine-matrix.json index 0e47a0b0ef..441ba9cc33 100644 --- a/.github/json_matrices/engine-matrix.json +++ b/.github/json_matrices/engine-matrix.json @@ -10,12 +10,12 @@ "run": "always" }, { - "type": "valkey", - "version": "redis-7.0.15" + "type": "redis", + "version": "7.0" }, { - "type": "valkey", - "version": "redis-6.2.14", + "type": "redis", + "version": "6.2", "run": "always" } ] diff --git a/.github/json_matrices/supported-languages-versions.json b/.github/json_matrices/supported-languages-versions.json new file mode 100644 index 0000000000..bc6fd5f472 --- /dev/null +++ b/.github/json_matrices/supported-languages-versions.json @@ -0,0 +1,17 @@ +[ + { + "language": "java", + "versions": ["11", "17"], + "always-run-versions": ["17"] + }, + { + "language": "python", + "versions": ["3.8", "3.9", "3.10", "3.11", "3.12"], + "always-run-versions": ["3.8", "3.12"] + }, + { + "language": "node", + "full-versions": ["16.x", "17.x", "18.x", "19.x", "20.x"], + "always-run-versions": ["16.x", "20.x"] + } +] diff --git a/.github/workflows/create-test-matrices/action.yml b/.github/workflows/create-test-matrices/action.yml new file mode 100644 index 0000000000..61516b1cc9 --- /dev/null +++ b/.github/workflows/create-test-matrices/action.yml @@ -0,0 +1,54 @@ +inputs: + language-name: + description: "Language name" + required: true + dispatch-run-full-matrix: + description: "Run the full matrix" + required: false + default: "false" +outputs: + engine-matrix-output: + description: "Engine matrix" + value: ${{ steps.load-engine-matrix.outputs.engine-matrix }} + host-matrix-output: + description: "Host matrix" + value: ${{ steps.load-host-matrix.outputs.host-matrix }} + version-matrix-output: + description: "Version matrix" + value: ${{ steps.create-version-matrix.outputs.version-matrix }} + +runs: + using: "composite" + steps: + - name: Load engine matrix + id: load-engine-matrix + shell: bash + run: | + if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{inputs.dispatch-run-full-matrix}}" == "false" ]]; then + echo "engine-matrix=$(jq -c '[.[] | select(.run == "always")]' < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + else + echo "engine-matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + fi + echo "engine-matrix=$(cat $GITHUB_OUTPUT)" + + - name: Load host matrix + id: load-host-matrix + shell: bash + run: | + if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{inputs.dispatch-run-full-matrix}}" == "false" ]]; then + echo 'host-matrix={"include":'"$(jq -c '[.[] | select(.run | type == "array" and contains(["always"]))]' .github/json_matrices/build-matrix.json)"'}' >> $GITHUB_OUTPUT + else + echo 'host-matrix={"include":'"$(jq -c '[.[] | select(.run | type == "array" and contains(["${{ inputs.language-name }}"]))]' .github/json_matrices/build-matrix.json)"'}' >> $GITHUB_OUTPUT + fi + echo "host-matrix=$(cat $GITHUB_OUTPUT)" + + - name: Create version matrix + id: create-version-matrix + shell: bash + run: | + if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{inputs.dispatch-run-full-matrix}}" == "false" ]]; then + echo 'version-matrix={"include":'"$(jq -c '[.[] | select(.language == "${{ inputs.language-name }}") | .["always-run-versions"] | map({version: .})]' .github/json_matrices/supported-languages-versions.json)"'}' >> $GITHUB_OUTPUT + else + echo 'version-matrix={"include":'"$(jq -c '[.[] | select(.language == "${{ inputs.language-name }}") | (.versions // .["full-versions"]) | map({version: .})]' .github/json_matrices/supported-languages-versions.json)"'}' >> $GITHUB_OUTPUT + fi + echo "version-matrix=$(cat $GITHUB_OUTPUT)" diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml index 36b380c3e0..fa1bb6d4e3 100644 --- a/.github/workflows/csharp.yml +++ b/.github/workflows/csharp.yml @@ -37,16 +37,16 @@ jobs: load-engine-matrix: runs-on: ubuntu-latest outputs: - matrix: ${{ steps.load-engine-matrix.outputs.matrix }} + matrix: ${{ steps.load-engine-matrix.outputs.matrix }} steps: - name: Checkout uses: actions/checkout@v4 - + - name: Load the engine matrix id: load-engine-matrix shell: bash run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT - + run-tests: needs: load-engine-matrix timeout-minutes: 25 @@ -56,19 +56,19 @@ jobs: engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} dotnet: # - '6.0' - - '8.0' + - "8.0" host: - { - OS: ubuntu, - RUNNER: ubuntu-latest, - TARGET: x86_64-unknown-linux-gnu - } + OS: ubuntu, + RUNNER: ubuntu-latest, + TARGET: x86_64-unknown-linux-gnu, + } # - { # OS: macos, # RUNNER: macos-latest, # TARGET: aarch64-apple-darwin # } - + runs-on: ${{ matrix.host.RUNNER }} steps: @@ -80,7 +80,7 @@ jobs: uses: actions/setup-dotnet@v4 with: dotnet-version: ${{ matrix.dotnet }} - + - name: Install shared software dependencies uses: ./.github/workflows/install-shared-dependencies with: @@ -112,7 +112,7 @@ jobs: benchmarks/results/* utils/clusters/** -# TODO Add amazonlinux + # TODO Add amazonlinux lint-rust: timeout-minutes: 10 @@ -125,3 +125,4 @@ jobs: - uses: ./.github/workflows/lint-rust with: cargo-toml-folder: ./csharp/lib + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index c2a181fd7d..8a7178117e 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -25,19 +25,12 @@ on: - .github/workflows/install-valkey/action.yml - .github/json_matrices/build-matrix.json workflow_dispatch: + inputs: + full-matrix: + description: "Run the full matrix" + required: false + default: "false" - pull_request_target: - types: [labeled] - paths: - - glide-core/src/** - - submodules/** - - java/** - - .github/workflows/java.yml - - .github/workflows/install-shared-dependencies/action.yml - - .github/workflows/test-benchmark/action.yml - - .github/workflows/lint-rust/action.yml - - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json schedule: - cron: "0 0 * * *" @@ -46,71 +39,32 @@ concurrency: cancel-in-progress: true jobs: - check-label: - runs-on: ubuntu-latest - # If the trigger is a label we want to check if it is a CoreChanges label, - # if it is we want to run the job in the AWS_ACTIONS environment so its need to be approved - environment: ${{ github.event_name == 'pull_request_target' && 'AWS_ACTIONS' || '' }} - steps: - - name: Check for CoreChanges label - if: github.event_name == 'pull_request_target' && github.event.label.name != 'CoreChanges' - run: | - echo "This workflow only runs for the CoreChanges label on pull requests" - exit 1 - - load-engine-matrix: - runs-on: ubuntu-latest - # We want to run this job only if the previous job outputs should_run=true, - # Which means that if the trigger was a label we run this job just if the label was CoreChanges - needs: check-label - outputs: - matrix: ${{ steps.load-engine-matrix.outputs.matrix }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Load the engine matrix - id: load-engine-matrix - shell: bash - run: | - if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" ]]; then - echo "matrix=$(jq -c '[.[] | select(.run == "always")]' < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT - else - echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT - fi - - create-host-matrix: - runs-on: ubuntu-latest - needs: check-label - outputs: - matrix: ${{ steps.create-host-matrix.outputs.matrix }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Create host matrix - id: create-host-matrix - shell: bash - run: | - if [[ "${{ github.event_name }}" == "pull_request" || "${{github.event_name }}" == "push" ]]; then - echo 'matrix={"include":[{"OS":"ubuntu","RUNNER":"ubuntu-latest","TARGET":"x86_64-unknown-linux-gnu"}]}' >> $GITHUB_OUTPUT - else - echo 'matrix={"include":[{"OS":"ubuntu","RUNNER":"ubuntu-latest","TARGET":"x86_64-unknown-linux-gnu"},{"OS":"macos","RUNNER":"macos-latest","TARGET":"aarch64-apple-darwin"}]}' >> $GITHUB_OUTPUT - fi - - build-and-test-java-client: - needs: [load-engine-matrix, create-host-matrix] + get-matrices: + runs-on: ubuntu-latest + # Avoid running on schedule for forks + if: (github.repository_owner == 'valkey-io' || github.event_name != 'schedule') + outputs: + engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} + host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} + version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output }} + steps: + - uses: actions/checkout@v4 + - id: get-matrices + uses: ./.github/workflows/create-test-matrices + with: + language-name: java + dispatch-run-full-matrix: ${{ github.event.inputs.full-matrix || 'false' }} + + test-java: + needs: get-matrices timeout-minutes: 35 strategy: # Run all jobs fail-fast: false matrix: - java: - - 11 - - 17 - engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} - host: ${{ fromJson(needs.create-host-matrix.outputs.matrix).include }} - + java: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} + engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} + host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output).include }} runs-on: ${{ matrix.host.RUNNER }} steps: @@ -166,16 +120,15 @@ jobs: java/client/build/reports/spotbugs/** build-amazonlinux-latest: - if: github.repository_owner == 'valkey-io' && (github.event_name == 'schedule' || github.event.label.name == 'CoreChanges') + if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && github.event.inputs.full-matrix == 'true') + needs: get-matrices strategy: # Run all jobs fail-fast: false matrix: - java: - - 11 - - 17 + java: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} + engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} runs-on: ubuntu-latest - needs: check-label container: amazonlinux:latest timeout-minutes: 35 steps: @@ -201,7 +154,7 @@ jobs: os: "amazon-linux" target: "x86_64-unknown-linux-gnu" github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: "7.2.5" + engine-version: ${{ matrix.engine.version }} - name: Install protoc (protobuf) uses: arduino/setup-protoc@v3 @@ -222,7 +175,7 @@ jobs: continue-on-error: true uses: actions/upload-artifact@v4 with: - name: test-reports-${{ matrix.java }}-amazon-linux + name: test-reports-${{ matrix.java }}-${{ matrix.engine }}-amazon-linux path: | java/client/build/reports/** java/integTest/build/reports/** @@ -231,7 +184,6 @@ jobs: lint-rust: timeout-minutes: 15 runs-on: ubuntu-latest - needs: check-label steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/lint-rust/action.yml b/.github/workflows/lint-rust/action.yml index 11ca944f71..9c85e3cc18 100644 --- a/.github/workflows/lint-rust/action.yml +++ b/.github/workflows/lint-rust/action.yml @@ -24,8 +24,6 @@ runs: github-token: ${{ inputs.github-token }} - uses: Swatinem/rust-cache@v2 - with: - github-token: ${{ inputs.github-token }} - run: cargo fmt --all -- --check working-directory: ${{ inputs.cargo-toml-folder }} diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index cf8693878b..1ae1973838 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -29,21 +29,12 @@ on: - .github/workflows/install-valkey/action.yml - .github/json_matrices/build-matrix.json workflow_dispatch: + inputs: + full-matrix: + description: "Run the full matrix" + required: false + default: "false" - pull_request_target: - types: [labeled] - paths: - - glide-core/src/** - - submodules/** - - node/** - - utils/cluster_manager.py - - .github/workflows/node.yml - - .github/workflows/build-node-wrapper/action.yml - - .github/workflows/install-shared-dependencies/action.yml - - .github/workflows/test-benchmark/action.yml - - .github/workflows/lint-rust/action.yml - - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json schedule: - cron: "0 1 * * *" @@ -55,87 +46,50 @@ env: CARGO_TERM_COLOR: always jobs: - check-label: - runs-on: ubuntu-latest - # If the trigger is a label we want to check if it is a CoreChanges label, - # if it is we want to run the job in the AWS_ACTIONS environment so its need to be approved - environment: ${{ github.event_name == 'pull_request_target' && 'AWS_ACTIONS' || '' }} - steps: - - name: Check for CoreChanges label - if: github.event_name == 'pull_request_target' && github.event.label.name != 'CoreChanges' - run: | - echo "This workflow only runs for the CoreChanges label on pull requests" - exit 1 - - load-engine-matrix: + get-matrices: runs-on: ubuntu-latest - # We want to run this job only if the previous job outputs should_run=true, - # Which means that if the trigger was a label we run this job just if the label was CoreChanges - needs: check-label + # Avoid running on schedule for forks + if: (github.repository_owner == 'valkey-io' || github.event_name != 'schedule') outputs: - matrix: ${{ steps.load-engine-matrix.outputs.matrix }} - steps: - - name: Checkout - uses: actions/checkout@v4 + engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} + host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} + version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output }} - - name: Load the engine matrix - id: load-engine-matrix - shell: bash - # We want to run the full matrix only for scheduled runs and CoreChanges - # For other events we want to run only the always matrix which is first and last version supported - run: | - if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" ]]; then - echo "matrix=$(jq -c '[.[] | select(.run == "always")]' < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT - else - echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT - fi - create-version-matrix: - runs-on: ubuntu-latest - needs: check-label - outputs: - matrix: ${{ steps.create-version-matrix.outputs.matrix }} steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Create version matrix - id: create-version-matrix - shell: bash - # We want to run the full matrix only for scheduled runs and CoreChanges - # For other events we want to run only the always matrix which is first and last version supported - run: | - if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" ]]; then - echo 'matrix={"node":["16.x","20.x"]}' >> $GITHUB_OUTPUT - else - echo 'matrix={"node":["16.x","17.x","18.x","19.x","20.x"]}' >> $GITHUB_OUTPUT - fi - + - uses: actions/checkout@v4 + - id: get-matrices + uses: ./.github/workflows/create-test-matrices + with: + language-name: node + dispatch-run-full-matrix: ${{ github.event.inputs.full-matrix || 'false' }} - test-ubuntu-latest: + test-node: runs-on: ubuntu-latest - needs: [load-engine-matrix, create-version-matrix] - timeout-minutes: 15 + needs: [get-matrices] + timeout-minutes: 25 strategy: fail-fast: false matrix: - engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} - node: ${{ fromJson(needs.create-version-matrix.outputs.matrix).node }} - + engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} + host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output) }} + node: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output)}} steps: - uses: actions/checkout@v4 with: submodules: recursive - name: Setup Node - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 + env: + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true with: node-version: ${{ matrix.node }} - name: Build Node wrapper uses: ./.github/workflows/build-node-wrapper with: - os: "ubuntu" - target: "x86_64-unknown-linux-gnu" + os: ${{ matrix.host.OS }} + target: ${{ matrix.host.TARGET }} github-token: ${{ secrets.GITHUB_TOKEN }} engine-version: ${{ matrix.engine.version }} @@ -145,18 +99,18 @@ jobs: - name: test hybrid node modules - commonjs run: | - npm install --package-lock-only - npm ci - npm run build-and-test + npm install --package-lock-only + npm ci + npm run build-and-test working-directory: ./node/hybrid-node-tests/commonjs-test env: JEST_HTML_REPORTER_OUTPUT_PATH: test-report-commonjs.html - name: test hybrid node modules - ecma run: | - npm install --package-lock-only - npm ci - npm run build-and-test + npm install --package-lock-only + npm ci + npm run build-and-test working-directory: ./node/hybrid-node-tests/ecmascript-test env: JEST_HTML_REPORTER_OUTPUT_PATH: test-report-ecma.html @@ -170,7 +124,7 @@ jobs: continue-on-error: true uses: actions/upload-artifact@v4 with: - name: test-report-node-${{ matrix.engine.type }}-${{ matrix.engine.version }}-ubuntu + name: test-report-node-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.node }}-ubuntu path: | node/test-report*.html utils/clusters/** @@ -179,7 +133,6 @@ jobs: lint-rust: timeout-minutes: 15 runs-on: ubuntu-latest - needs: check-label steps: - uses: actions/checkout@v4 with: @@ -190,58 +143,10 @@ jobs: cargo-toml-folder: ./node/rust-client name: lint node rust - build-macos-latest: - if: github.event_name == 'schedule' || github.event.label.name == 'CoreChanges' - runs-on: macos-latest - needs: check-label - timeout-minutes: 25 - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - name: Set up Homebrew - uses: Homebrew/actions/setup-homebrew@master - - - name: Install NodeJS - run: | - brew update - brew upgrade || true - brew install node - - - name: Downgrade npm major version to 8 - run: | - npm i -g npm@8 - - - name: Build Node wrapper - uses: ./.github/workflows/build-node-wrapper - with: - os: "macos" - named_os: "darwin" - arch: "arm64" - target: "aarch64-apple-darwin" - github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: "7.2.5" - - - name: Test compatibility - run: npm test -- -t "set and get flow works" - working-directory: ./node - - # - name: Upload test reports - # if: always() - # continue-on-error: true - # uses: actions/upload-artifact@v4 - # with: - # name: test-report-node-${{ matrix.engine.type }}-${{ matrix.engine.version }}-macos - # path: | - # node/test-report*.html - # utils/clusters/** - # benchmarks/results/** - build-amazonlinux-latest: - if: github.event_name == 'schedule' || github.event.label.name == 'CoreChanges' + if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule') runs-on: ubuntu-latest container: amazonlinux:latest - needs: check-label timeout-minutes: 15 steps: - name: Install git @@ -271,8 +176,8 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} engine-version: "7.2.5" - - name: Test compatibility - run: npm test -- -t "set and get flow works" + - name: test + run: npm test working-directory: ./node - name: Upload test reports @@ -287,9 +192,8 @@ jobs: benchmarks/results/** build-and-test-linux-musl-on-x86: - if: github.event_name == 'schedule' || github.event.label.name == 'CoreChanges' + if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && github.event.inputs.full-matrix == 'true') name: Build and test Node wrapper on Linux musl - needs: check-label runs-on: ubuntu-latest container: image: node:alpine @@ -322,9 +226,8 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} engine-version: "7.2.5" - - name: Test compatibility - shell: bash - run: npm test -- -t "set and get flow works" + - name: test + run: npm test working-directory: ./node - name: Upload test reports diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 0b88b03c88..493e30614c 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -34,21 +34,12 @@ on: - .github/json_matrices/engine-matrix.json - .github/workflows/start-self-hosted-runner/action.yml workflow_dispatch: + inputs: + full-matrix: + description: "Run the full matrix" + required: false + default: "false" - pull_request_target: - types: [labeled] - paths: - - python/** - - glide-core/src/** - - submodules/** - - utils/cluster_manager.py - - .github/workflows/python.yml - - .github/workflows/build-python-wrapper/action.yml - - .github/workflows/install-shared-dependencies/action.yml - - .github/workflows/test-benchmark/action.yml - - .github/workflows/lint-rust/action.yml - - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json schedule: - cron: "0 2 * * *" @@ -62,78 +53,39 @@ permissions: id-token: write jobs: - check-label: - runs-on: ubuntu-latest - # If the trigger is a label we want to check if it is a CoreChanges label, - # if it is we want to run the job in the AWS_ACTIONS environment so its need to be approved - environment: ${{ github.event_name == 'pull_request_target' && 'AWS_ACTIONS' || '' }} - steps: - - name: Check for CoreChanges label - if: github.event_name == 'pull_request_target' && github.event.label.name != 'CoreChanges' - run: | - echo "This workflow only runs for the CoreChanges label on pull requests" - exit 1 - - load-engine-matrix: - runs-on: ubuntu-latest - # We want to run this job only if the previous job outputs should_run=true, - # Which means that if the trigger was a label we run this job just if the label was CoreChanges - needs: check-label - outputs: - matrix: ${{ steps.load-engine-matrix.outputs.matrix }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Load the engine matrix - id: load-engine-matrix - shell: bash - # We want to run the full matrix only for scheduled runs and CoreChanges - # For other events we want to run only the always matrix which is first and last version supported - run: | - if [[ "${{ github.event_name }}" == "pull_request" || "${{github.event_name }}" == "push" ]]; then - echo "matrix=$(jq -c '[.[] | select(.run == "always")]' < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT - else - echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT - fi - create-version-matrix: + get-matrices: runs-on: ubuntu-latest - needs: check-label + # Avoid running on schedule for forks + if: (github.repository_owner == 'valkey-io' || github.event_name != 'schedule') outputs: - matrix: ${{ steps.create-version-matrix.outputs.matrix }} + engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} + host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} + version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output}} steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Create version matrix - id: create-version-matrix - - shell: bash - # We want to run the full matrix only for scheduled runs and CoreChanges - # For other events we want to run only the always matrix which is first and last version supported - run: | - if [[ "${{ github.event_name }}" == "pull_request" || "${{github.event_name }}" == "push" ]]; then - echo 'matrix={"python":["3.8","3.12"]}' >> $GITHUB_OUTPUT - else - echo 'matrix={"python":["3.8","3.9","3.10","3.11","3.12"]}' >> $GITHUB_OUTPUT - fi + - uses: actions/checkout@v4 + - id: get-matrices + uses: ./.github/workflows/create-test-matrices + with: + language-name: python + dispatch-run-full-matrix: ${{ github.event.inputs.full-matrix || 'false' }} - test-ubuntu-latest: - runs-on: ubuntu-latest - needs: [load-engine-matrix, create-version-matrix] + test-python: + runs-on: ${{ matrix.host.RUNNER }} + needs: get-matrices timeout-minutes: 35 strategy: fail-fast: false matrix: - engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} - python: ${{ fromJson(needs.create-version-matrix.outputs.matrix).python }} + engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} + python: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} + host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output).include }} steps: - uses: actions/checkout@v4 with: submodules: recursive - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} @@ -183,52 +135,26 @@ jobs: python/python/tests/pytest_report.html utils/clusters/** benchmarks/results/** - test-pubsub-ubuntu-latest: - runs-on: ubuntu-latest - needs: [load-engine-matrix, create-version-matrix] - timeout-minutes: 35 - strategy: - fail-fast: false - matrix: - engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} - python: ${{ fromJson(needs.create-version-matrix.outputs.matrix).python }} - - test-pubsub: + test-pubsub-python: runs-on: ${{ matrix.host.RUNNER }} - needs: load-engine-matrix + needs: get-matrices timeout-minutes: 35 strategy: fail-fast: false matrix: - engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} - python: - # - "3.8" - # - "3.9" - # - "3.10" - # - "3.11" - - "3.12" - host: - - { - OS: ubuntu, - RUNNER: ubuntu-latest, - TARGET: x86_64-unknown-linux-gnu - } - # - { - # OS: macos, - # RUNNER: macos-latest, - # TARGET: aarch64-apple-darwin - # } - + engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} + python: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} + host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output).include }} steps: - uses: actions/checkout@v4 with: submodules: recursive - + - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - + - name: Build Python wrapper uses: ./.github/workflows/build-python-wrapper with: @@ -236,14 +162,14 @@ jobs: target: ${{ matrix.host.TARGET }} github-token: ${{ secrets.GITHUB_TOKEN }} engine-version: ${{ matrix.engine.version }} - + - name: Test pubsub with pytest working-directory: ./python run: | source .env/bin/activate cd python/tests/ pytest --asyncio-mode=auto -k test_pubsub --html=pytest_report.html --self-contained-html - + - name: Upload test reports if: always() continue-on-error: true @@ -255,7 +181,6 @@ jobs: lint: runs-on: ubuntu-latest - needs: check-label timeout-minutes: 15 steps: - uses: actions/checkout@v4 @@ -294,72 +219,10 @@ jobs: working-directory: ./python run: | black --check --diff . - test-macos-latest: - if: github.event_name == 'schedule' || github.event.label.name == 'CoreChanges' - runs-on: macos-latest - needs: load-engine-matrix - timeout-minutes: 35 - strategy: - fail-fast: false - matrix: - engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - name: Set up Homebrew - uses: Homebrew/actions/setup-homebrew@master - - - name: Build Python wrapper - uses: ./.github/workflows/build-python-wrapper - with: - os: "macos" - target: "aarch64-apple-darwin" - github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: ${{ matrix.engine.version }} - - - name: Test with pytest - working-directory: ./python - run: | - source .env/bin/activate - pytest --asyncio-mode=auto - - test-pubsub-macos-latest: - if: github.event_name == 'schedule' || github.event.label.name == 'CoreChanges' - runs-on: macos-latest - needs: load-engine-matrix - timeout-minutes: 35 - strategy: - fail-fast: false - matrix: - engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - name: Set up Homebrew - uses: Homebrew/actions/setup-homebrew@master - - - name: Build Python wrapper - uses: ./.github/workflows/build-python-wrapper - with: - os: "macos" - target: "aarch64-apple-darwin" - github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: ${{ matrix.engine.version }} - - - name: Test pubsub with pytest - working-directory: ./python - run: | - source .env/bin/activate - cd python/tests/ - pytest --asyncio-mode=auto -k test_pubsub - build-amazonlinux-latest: - if: github.event_name == 'schedule' || github.event.label.name == 'CoreChanges' + if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && github.event.inputs.full-matrix == 'true') runs-on: ubuntu-latest container: amazonlinux:latest - needs: check-label timeout-minutes: 15 steps: - name: Install git diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000000..dd449725e1 --- /dev/null +++ b/.prettierignore @@ -0,0 +1 @@ +*.md diff --git a/.vscode/settings.json b/.vscode/settings.json index ef488543df..0ff2c47feb 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -27,7 +27,7 @@ "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true, "[github-actions-workflow]": { - "editor.defaultFormatter": "redhat.vscode-yaml" + "editor.defaultFormatter": "esbenp.prettier-vscode" }, "yaml.schemas": { "https://json.schemastore.org/github-workflow.json": [ From c35cacf60325e01f0d5eebe62ce88f6d4edb699a Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Thu, 29 Aug 2024 19:09:49 -0700 Subject: [PATCH 03/12] pr comments Signed-off-by: Yury-Fridlyand --- .github/DEVELOPER.md | 51 ++++++++++--------- .github/json_matrices/build-matrix.json | 13 ++--- ....json => supported-language-versions.json} | 2 +- .../workflows/create-test-matrices/action.yml | 44 +++++++++++----- .github/workflows/java.yml | 16 +++--- .github/workflows/node.yml | 14 ++--- .github/workflows/python.yml | 21 +++++--- 7 files changed, 96 insertions(+), 65 deletions(-) rename .github/json_matrices/{supported-languages-versions.json => supported-language-versions.json} (84%) diff --git a/.github/DEVELOPER.md b/.github/DEVELOPER.md index 53e4116220..f3d05950a0 100644 --- a/.github/DEVELOPER.md +++ b/.github/DEVELOPER.md @@ -24,9 +24,9 @@ Each language has its own workflow file with similar structure but language-spec While workflows are language-specific, the matrix files are shared across all workflows. Workflows are starting by loading the matrix files from the `.github/json_matrices` directory. -* `engine-matrix.json`: Defines the versions of Valkey engine to test against. -* `build-matrix.json`: Defines the host environments for testing. -* `supported-languages-version.json`: Defines the supported versions of languages. +* `engine-matrix.json`: Defines the versions of the engine to test against. +* `build-matrix.json`: Defines the host environments for testing. +* `supported-language-versions.json`: Defines the supported versions of languages. All matrices have a `run` like field which specifies if the configuration should be tested on every workflow run. This allows for flexible control over which configurations are tested in different scenarios, optimizing CI/CD performance and resource usage. @@ -38,11 +38,14 @@ Defines the versions of Valkey engine to test against: ```json [ { "type": "valkey", "version": "7.2.5", "run": "always" }, + // ... other configurations ] ``` -* `type`: The type of engine (e.g., Valkey, Redis). -* `version`: The version of the engine. -* `run`: Specifies if the engine version should be tested on every workflow. + +* `type`: The type of engine (e.g., Valkey, Redis). +* `version`: Specifies the engine version that the workflow should checkout. + For example, "7.2.5" represents a release tag, while "7.0" denotes a branch name. The workflow should use this parameter to checkout the specific release version or branch to build the engine with the appropriate version. +* `run`: Specifies if the engine version should be tested on every workflow. #### Build Matrix (build-matrix.json) @@ -59,10 +62,10 @@ Defines the host environments for testing: // ... other configurations ] ``` -* `OS`: The operating system of the host. -* `RUNNER`: The GitHub runner to use. -* `TARGET`: The target environment. -* `run`: Specifies which language workflows should use this host configuration, always means run on each workflow trigger. +* `OS`: The operating system of the host. +* `RUNNER`: The GitHub runner to use. +* `TARGET`: The target environment as defined in Rust. To see a list of available targets, run `rustup target list`. +* `run`: Specifies which language workflows should use this host configuration. The value `always` indicates that the configuration should be used for every workflow trigger. #### Supported Languages Version (supported-languages-version.json) @@ -78,33 +81,33 @@ Defines the supported versions of languages: // ... other configurations ] ``` -* `language`: The language for which the version is supported. -* `versions`: The full versions supported of the language which will test against scheduled. -* `always-run-versions`: The versions which will be tested in every workflow run. +* `language`: The language for which the version is supported. +* `versions`: The full versions supported of the language which will test against scheduled. +* `always-run-versions`: The versions that will always be tested, regardless of the workflow trigger. #### Triggering Workflows -Push to main or create a pull request to run workflows automatically. -Use workflow_dispatch for manual triggers, accepting inputs of full-matrix which is a boolean value to run all configurations. -Scheduled runs are triggered daily to ensure regular testing of all configurations. +- Push to `main` by merging a PR or create a new pull request to run workflows automatically. +- Use `workflow_dispatch` for manual triggers, accepting a boolean configuration parameter to run all configurations. +- Scheduled runs are triggered daily to ensure regular testing of all configurations. ### Mutual vs. Language-Specific Components #### Mutual -`Matrix files` - `.github/json_matrices` -`Shared dependencies installation` - `.github/workflows/install-shared-dependencies/action.yml` -`Linting Rust` - `.github/workflows/lint-rust/action.yml` +- Matrix files - `.github/json_matrices/` +- Shared dependencies installation - `.github/workflows/install-shared-dependencies/action.yml` +- Rust linters - `.github/workflows/lint-rust/action.yml` #### Language-Specific -`Package manager commands` -`Testing frameworks` -`Build processes` +- Package manager commands +- Testing frameworks +- Build processes ### Customizing Workflows -Modify `[language].yml` files to adjust language-specific steps. +Modify `.yml` files to adjust language-specific steps. Update matrix files to change tested versions or environments. Adjust cron schedules in workflow files for different timing of scheduled runs. @@ -117,7 +120,7 @@ We use dynamic matrices for our CI/CD workflows, which are created using the `cr 1. The action is called with a `language-name` input and `dispatch-run-full-matrix` input. 2. It reads the `engine-matrix.json`, `build-matrix.json`, and `supported-languages-version.json` files. 3. It filters the matrices based on the inputs and the event type. -2. It generates three matrices: +4. It generates three matrices: - Engine matrix: Defines the types and versions of the engine to test against, for example Valkey 7.2.5. - Host matrix: Defines the host platforms to run the tests on, for example Ubuntu on ARM64. - Language-version matrix: Defines the supported versions of languages, for example python 3.8. diff --git a/.github/json_matrices/build-matrix.json b/.github/json_matrices/build-matrix.json index f776ad3c64..9dbc749fd0 100644 --- a/.github/json_matrices/build-matrix.json +++ b/.github/json_matrices/build-matrix.json @@ -6,7 +6,8 @@ "ARCH": "x64", "TARGET": "x86_64-unknown-linux-gnu", "PACKAGE_MANAGERS": ["pypi", "npm"], - "run": ["always", "python", "node", "java"] + "run": "always", + "languages": ["python", "node", "java"] }, { "OS": "ubuntu", @@ -16,7 +17,7 @@ "TARGET": "aarch64-unknown-linux-gnu", "PACKAGE_MANAGERS": ["pypi", "npm"], "CONTAINER": "2_28", - "run": ["python", "node", "java"] + "languages": ["python", "node", "java"] }, { "OS": "macos", @@ -25,7 +26,7 @@ "ARCH": "x64", "TARGET": "x86_64-apple-darwin", "PACKAGE_MANAGERS": ["pypi", "npm"], - "run": ["python", "node", "java"] + "languages": ["python", "node", "java"] }, { "OS": "macos", @@ -34,7 +35,7 @@ "ARCH": "arm64", "TARGET": "aarch64-apple-darwin", "PACKAGE_MANAGERS": ["pypi", "npm"], - "run": ["python", "node", "java"] + "languages": ["python", "node", "java"] }, { "OS": "ubuntu", @@ -45,7 +46,7 @@ "IMAGE": "node:alpine", "CONTAINER_OPTIONS": "--user root --privileged --rm", "PACKAGE_MANAGERS": ["npm"], - "run": ["node"] + "languages": ["node"] }, { "OS": "ubuntu", @@ -56,6 +57,6 @@ "IMAGE": "node:alpine", "CONTAINER_OPTIONS": "--user root --privileged", "PACKAGE_MANAGERS": ["npm"], - "run": ["node"] + "languages": ["node"] } ] diff --git a/.github/json_matrices/supported-languages-versions.json b/.github/json_matrices/supported-language-versions.json similarity index 84% rename from .github/json_matrices/supported-languages-versions.json rename to .github/json_matrices/supported-language-versions.json index bc6fd5f472..bc1b4388cc 100644 --- a/.github/json_matrices/supported-languages-versions.json +++ b/.github/json_matrices/supported-language-versions.json @@ -11,7 +11,7 @@ }, { "language": "node", - "full-versions": ["16.x", "17.x", "18.x", "19.x", "20.x"], + "versions": ["16.x", "17.x", "18.x", "19.x", "20.x"], "always-run-versions": ["16.x", "20.x"] } ] diff --git a/.github/workflows/create-test-matrices/action.yml b/.github/workflows/create-test-matrices/action.yml index 61516b1cc9..8f37ce989b 100644 --- a/.github/workflows/create-test-matrices/action.yml +++ b/.github/workflows/create-test-matrices/action.yml @@ -2,10 +2,19 @@ inputs: language-name: description: "Language name" required: true - dispatch-run-full-matrix: + type: choice + options: + - java + - node + - python + - go + - C# + run-full-matrix: description: "Run the full matrix" required: false + type: boolean default: "false" + outputs: engine-matrix-output: description: "Engine matrix" @@ -15,7 +24,7 @@ outputs: value: ${{ steps.load-host-matrix.outputs.host-matrix }} version-matrix-output: description: "Version matrix" - value: ${{ steps.create-version-matrix.outputs.version-matrix }} + value: ${{ steps.create-lang-version-matrix.outputs.version-matrix }} runs: using: "composite" @@ -24,10 +33,13 @@ runs: id: load-engine-matrix shell: bash run: | - if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{inputs.dispatch-run-full-matrix}}" == "false" ]]; then - echo "engine-matrix=$(jq -c '[.[] | select(.run == "always")]' < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + : Select server engines to run tests against + if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{ inputs.dispatch-run-full-matrix }}" == "false" ]]; then + : Pick engines marked as '"run": "always"' only - on PR, push or manually triggered job which does not require full matrix + echo "engine-matrix=$(jq -c '[.[] | select(.run == "always")]' < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT else - echo "engine-matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + : Pick all engines - on cron (schedule) or if manually triggered job requires a full matrix + echo "engine-matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT fi echo "engine-matrix=$(cat $GITHUB_OUTPUT)" @@ -35,20 +47,26 @@ runs: id: load-host-matrix shell: bash run: | - if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{inputs.dispatch-run-full-matrix}}" == "false" ]]; then - echo 'host-matrix={"include":'"$(jq -c '[.[] | select(.run | type == "array" and contains(["always"]))]' .github/json_matrices/build-matrix.json)"'}' >> $GITHUB_OUTPUT + : Select runners (VMs) to run tests on + if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{ inputs.dispatch-run-full-matrix }}" == "false" ]]; then + : Pick runners marked as '"run": "always"' only - on PR, push or manually triggered job which does not require full matrix + echo 'host-matrix=$(jq -c '[.[] | select(.run == "always")]' < .github/json_matrices/build-matrix.json)' >> $GITHUB_OUTPUT else - echo 'host-matrix={"include":'"$(jq -c '[.[] | select(.run | type == "array" and contains(["${{ inputs.language-name }}"]))]' .github/json_matrices/build-matrix.json)"'}' >> $GITHUB_OUTPUT + : Pick all runners assigned for the chosen client (language) - on cron (schedule) or if manually triggered job requires a full matrix + echo 'host-matrix=$(jq -c '[.[] | select(.languages | type == "array" and contains(["${{ inputs.language-name }}"]))]' < .github/json_matrices/build-matrix.json) >> $GITHUB_OUTPUT fi echo "host-matrix=$(cat $GITHUB_OUTPUT)" - - name: Create version matrix - id: create-version-matrix + - name: Create language version matrix + id: create-lang-version-matrix shell: bash run: | - if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{inputs.dispatch-run-full-matrix}}" == "false" ]]; then - echo 'version-matrix={"include":'"$(jq -c '[.[] | select(.language == "${{ inputs.language-name }}") | .["always-run-versions"] | map({version: .})]' .github/json_matrices/supported-languages-versions.json)"'}' >> $GITHUB_OUTPUT + : Select language (framework/SDK) versions to run tests on + if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{ inputs.dispatch-run-full-matrix }}" == "false" ]]; then + : Pick language versions listed in 'always-run-versions' only - on PR, push or manually triggered job which does not require full matrix + echo 'version-matrix=$(jq -c '[.[] | select(.language == "${{ inputs.language-name }}") | .["always-run-versions"] | map({version: .})]' < .github/json_matrices/supported-language-versions.json) >> $GITHUB_OUTPUT else - echo 'version-matrix={"include":'"$(jq -c '[.[] | select(.language == "${{ inputs.language-name }}") | (.versions // .["full-versions"]) | map({version: .})]' .github/json_matrices/supported-languages-versions.json)"'}' >> $GITHUB_OUTPUT + : Pick language versions listed in 'versions' - on cron (schedule) or if manually triggered job requires a full matrix + echo 'version-matrix=$(jq -c '[.[] | select(.language == "${{ inputs.language-name }}") | .versions | map({version: .})]' < .github/json_matrices/supported-language-versions.json) >> $GITHUB_OUTPUT fi echo "version-matrix=$(cat $GITHUB_OUTPUT)" diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 8a7178117e..bad15d1233 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -27,12 +27,13 @@ on: workflow_dispatch: inputs: full-matrix: - description: "Run the full matrix" + description: "Run the full engine, host, and language version matrix" required: false + type: boolean default: "false" schedule: - - cron: "0 0 * * *" + - cron: "0 5 * * *" concurrency: group: java-${{ github.head_ref || github.ref }} @@ -42,7 +43,7 @@ jobs: get-matrices: runs-on: ubuntu-latest # Avoid running on schedule for forks - if: (github.repository_owner == 'valkey-io' || github.event_name != 'schedule') + if: (github.event_name == 'schedule' && github.repository_owner == 'valkey-io') || github.event_name == 'push' || github.event_name == 'pull_request' outputs: engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} @@ -53,7 +54,7 @@ jobs: uses: ./.github/workflows/create-test-matrices with: language-name: java - dispatch-run-full-matrix: ${{ github.event.inputs.full-matrix || 'false' }} + run-full-matrix: ${{ github.event.inputs.full-matrix || 'false' }} test-java: needs: get-matrices @@ -64,7 +65,7 @@ jobs: matrix: java: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} - host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output).include }} + host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output) }} runs-on: ${{ matrix.host.RUNNER }} steps: @@ -120,7 +121,7 @@ jobs: java/client/build/reports/spotbugs/** build-amazonlinux-latest: - if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && github.event.inputs.full-matrix == 'true') + if: github.repository_owner == 'valkey-io' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && github.event.inputs.full-matrix == 'true' needs: get-matrices strategy: # Run all jobs @@ -128,6 +129,7 @@ jobs: matrix: java: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} + runs-on: ubuntu-latest container: amazonlinux:latest timeout-minutes: 35 @@ -192,4 +194,4 @@ jobs: - uses: ./.github/workflows/lint-rust with: cargo-toml-folder: ./java - name: lint java rust + name: lint rust diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 1ae1973838..26a000bc61 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -31,12 +31,13 @@ on: workflow_dispatch: inputs: full-matrix: - description: "Run the full matrix" + description: "Run the full engine, host, and language version matrix" required: false + type: boolean default: "false" schedule: - - cron: "0 1 * * *" + - cron: "0 6 * * *" concurrency: group: node-${{ github.head_ref || github.ref }} @@ -49,7 +50,7 @@ jobs: get-matrices: runs-on: ubuntu-latest # Avoid running on schedule for forks - if: (github.repository_owner == 'valkey-io' || github.event_name != 'schedule') + if: (github.event_name == 'schedule' && github.repository_owner == 'valkey-io') || github.event_name == 'push' || github.event_name == 'pull_request' outputs: engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} @@ -61,7 +62,7 @@ jobs: uses: ./.github/workflows/create-test-matrices with: language-name: node - dispatch-run-full-matrix: ${{ github.event.inputs.full-matrix || 'false' }} + run-full-matrix: ${{ github.event.inputs.full-matrix || 'false' }} test-node: runs-on: ubuntu-latest @@ -72,7 +73,7 @@ jobs: matrix: engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output) }} - node: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output)}} + node: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} steps: - uses: actions/checkout@v4 with: @@ -144,7 +145,8 @@ jobs: name: lint node rust build-amazonlinux-latest: - if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule') + if: github.repository_owner == 'valkey-io' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && github.event.inputs.full-matrix == 'true' + # TODO matrix runs-on: ubuntu-latest container: amazonlinux:latest timeout-minutes: 15 diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 493e30614c..854a3085ea 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -36,12 +36,13 @@ on: workflow_dispatch: inputs: full-matrix: - description: "Run the full matrix" + description: "Run the full engine, host, and language version matrix" required: false + type: boolean default: "false" schedule: - - cron: "0 2 * * *" + - cron: "0 4 * * *" concurrency: group: python-${{ github.head_ref || github.ref }} @@ -56,18 +57,18 @@ jobs: get-matrices: runs-on: ubuntu-latest # Avoid running on schedule for forks - if: (github.repository_owner == 'valkey-io' || github.event_name != 'schedule') + if: (github.event_name == 'schedule' && github.repository_owner == 'valkey-io') || github.event_name == 'push' || github.event_name == 'pull_request' outputs: engine-matrix-output: ${{ steps.get-matrices.outputs.engine-matrix-output }} host-matrix-output: ${{ steps.get-matrices.outputs.host-matrix-output }} - version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output}} + version-matrix-output: ${{ steps.get-matrices.outputs.version-matrix-output }} steps: - uses: actions/checkout@v4 - id: get-matrices uses: ./.github/workflows/create-test-matrices with: language-name: python - dispatch-run-full-matrix: ${{ github.event.inputs.full-matrix || 'false' }} + run-full-matrix: ${{ github.event.inputs.full-matrix || 'false' }} test-python: runs-on: ${{ matrix.host.RUNNER }} @@ -78,7 +79,7 @@ jobs: matrix: engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} python: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} - host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output).include }} + host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output) }} steps: - uses: actions/checkout@v4 with: @@ -135,6 +136,7 @@ jobs: python/python/tests/pytest_report.html utils/clusters/** benchmarks/results/** + test-pubsub-python: runs-on: ${{ matrix.host.RUNNER }} needs: get-matrices @@ -144,7 +146,8 @@ jobs: matrix: engine: ${{ fromJson(needs.get-matrices.outputs.engine-matrix-output) }} python: ${{ fromJson(needs.get-matrices.outputs.version-matrix-output) }} - host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output).include }} + host: ${{ fromJson(needs.get-matrices.outputs.host-matrix-output) }} + steps: - uses: actions/checkout@v4 with: @@ -219,8 +222,10 @@ jobs: working-directory: ./python run: | black --check --diff . + build-amazonlinux-latest: - if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && github.event.inputs.full-matrix == 'true') + if: github.repository_owner == 'valkey-io' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && github.event.inputs.full-matrix == 'true' + # TODO matrix runs-on: ubuntu-latest container: amazonlinux:latest timeout-minutes: 15 From 8100bf1cd5cbd44f1cd9747601972292bc6ab944 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Thu, 29 Aug 2024 19:13:20 -0700 Subject: [PATCH 04/12] dbg matrices Signed-off-by: Yury-Fridlyand --- .github/workflows/java.yml | 89 -------------- .github/workflows/node.yml | 121 ------------------- .github/workflows/python.yml | 219 ----------------------------------- 3 files changed, 429 deletions(-) diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index bad15d1233..abca85aae3 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -73,53 +73,6 @@ jobs: with: submodules: recursive - - uses: gradle/actions/wrapper-validation@v3 - - - name: Set up JDK ${{ matrix.java }} - uses: actions/setup-java@v4 - with: - distribution: "temurin" - java-version: ${{ matrix.java }} - - - name: Install shared software dependencies - uses: ./.github/workflows/install-shared-dependencies - with: - os: ${{ matrix.host.OS }} - target: ${{ matrix.host.TARGET }} - github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: ${{ matrix.engine.version }} - - - name: Install protoc (protobuf) - uses: arduino/setup-protoc@v3 - with: - version: "26.1" - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Build java client - working-directory: java - run: ./gradlew --continue build -x javadoc - - - name: Ensure no skipped files by linter - working-directory: java - run: ./gradlew spotlessDiagnose | grep 'All formatters are well behaved for all files' - - - uses: ./.github/workflows/test-benchmark - with: - language-flag: -java - - - name: Upload test & spotbugs reports - if: always() - continue-on-error: true - uses: actions/upload-artifact@v4 - with: - name: test-reports-java-${{ matrix.java }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.host.RUNNER }} - path: | - java/client/build/reports/** - java/integTest/build/reports/** - utils/clusters/** - benchmarks/results/** - java/client/build/reports/spotbugs/** - build-amazonlinux-latest: if: github.repository_owner == 'valkey-io' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && github.event.inputs.full-matrix == 'true' needs: get-matrices @@ -145,44 +98,6 @@ jobs: - uses: actions/checkout@v4 - - name: Checkout submodules - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - git submodule update --init --recursive - - - name: Install shared software dependencies - uses: ./.github/workflows/install-shared-dependencies - with: - os: "amazon-linux" - target: "x86_64-unknown-linux-gnu" - github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: ${{ matrix.engine.version }} - - - name: Install protoc (protobuf) - uses: arduino/setup-protoc@v3 - with: - version: "26.1" - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Install Java - run: | - yum install -y java-${{ matrix.java }}-amazon-corretto-devel.x86_64 - - - name: Build java wrapper - working-directory: java - run: ./gradlew --continue build -x javadoc - - - name: Upload test & spotbugs reports - if: always() - continue-on-error: true - uses: actions/upload-artifact@v4 - with: - name: test-reports-${{ matrix.java }}-${{ matrix.engine }}-amazon-linux - path: | - java/client/build/reports/** - java/integTest/build/reports/** - java/client/build/reports/spotbugs/** - lint-rust: timeout-minutes: 15 runs-on: ubuntu-latest @@ -191,7 +106,3 @@ jobs: with: submodules: recursive - - uses: ./.github/workflows/lint-rust - with: - cargo-toml-folder: ./java - name: lint rust diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 26a000bc61..18631fd1fa 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -79,58 +79,6 @@ jobs: with: submodules: recursive - - name: Setup Node - uses: actions/setup-node@v4 - env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - with: - node-version: ${{ matrix.node }} - - - name: Build Node wrapper - uses: ./.github/workflows/build-node-wrapper - with: - os: ${{ matrix.host.OS }} - target: ${{ matrix.host.TARGET }} - github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: ${{ matrix.engine.version }} - - - name: test - run: npm test - working-directory: ./node - - - name: test hybrid node modules - commonjs - run: | - npm install --package-lock-only - npm ci - npm run build-and-test - working-directory: ./node/hybrid-node-tests/commonjs-test - env: - JEST_HTML_REPORTER_OUTPUT_PATH: test-report-commonjs.html - - - name: test hybrid node modules - ecma - run: | - npm install --package-lock-only - npm ci - npm run build-and-test - working-directory: ./node/hybrid-node-tests/ecmascript-test - env: - JEST_HTML_REPORTER_OUTPUT_PATH: test-report-ecma.html - - - uses: ./.github/workflows/test-benchmark - with: - language-flag: -node - - - name: Upload test reports - if: always() - continue-on-error: true - uses: actions/upload-artifact@v4 - with: - name: test-report-node-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.node }}-ubuntu - path: | - node/test-report*.html - utils/clusters/** - benchmarks/results/** - lint-rust: timeout-minutes: 15 runs-on: ubuntu-latest @@ -139,11 +87,6 @@ jobs: with: submodules: recursive - - uses: ./.github/workflows/lint-rust - with: - cargo-toml-folder: ./node/rust-client - name: lint node rust - build-amazonlinux-latest: if: github.repository_owner == 'valkey-io' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && github.event.inputs.full-matrix == 'true' # TODO matrix @@ -161,38 +104,6 @@ jobs: - uses: actions/checkout@v4 - - name: Checkout submodules - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - git submodule update --init --recursive - - - name: Install NodeJS - run: | - yum install -y nodejs - - - name: Build Node wrapper - uses: ./.github/workflows/build-node-wrapper - with: - os: "amazon-linux" - target: "x86_64-unknown-linux-gnu" - github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: "7.2.5" - - - name: test - run: npm test - working-directory: ./node - - - name: Upload test reports - if: always() - continue-on-error: true - uses: actions/upload-artifact@v4 - with: - name: test-report-node-amazonlinux - path: | - node/test-report*.html - utils/clusters/** - benchmarks/results/** - build-and-test-linux-musl-on-x86: if: (github.repository_owner == 'valkey-io' && github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && github.event.inputs.full-matrix == 'true') name: Build and test Node wrapper on Linux musl @@ -210,35 +121,3 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - - name: Setup musl on Linux - uses: ./.github/workflows/setup-musl-on-linux - with: - workspace: $GITHUB_WORKSPACE - npm-scope: ${{ secrets.NPM_SCOPE }} - npm-auth-token: ${{ secrets.NPM_AUTH_TOKEN }} - - - name: Build Node wrapper - uses: ./.github/workflows/build-node-wrapper - with: - os: ubuntu - named_os: linux - arch: x64 - target: x86_64-unknown-linux-musl - github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: "7.2.5" - - - name: test - run: npm test - working-directory: ./node - - - name: Upload test reports - if: always() - continue-on-error: true - uses: actions/upload-artifact@v4 - with: - name: test-report-node-linux-musl - path: | - node/test-report*.html - utils/clusters/** - benchmarks/results/** diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 854a3085ea..3afcb166e4 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -85,58 +85,6 @@ jobs: with: submodules: recursive - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python }} - - - name: Install dependencies - working-directory: ./python - run: | - python -m pip install --upgrade pip - pip install mypy-protobuf - - - name: Build Python wrapper - uses: ./.github/workflows/build-python-wrapper - with: - os: ${{ matrix.host.OS }} - target: ${{ matrix.host.TARGET }} - github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: ${{ matrix.engine.version }} - - - name: Type check with mypy - working-directory: ./python - run: | - # The type check should run inside the virtual env to get - # all installed dependencies and build files - source .env/bin/activate - pip install mypy types-protobuf - # Install the benchmark requirements - pip install -r ../benchmarks/python/requirements.txt - python -m mypy .. - - - name: Test with pytest - working-directory: ./python - run: | - source .env/bin/activate - cd python/tests/ - pytest --asyncio-mode=auto --html=pytest_report.html --self-contained-html - - - uses: ./.github/workflows/test-benchmark - with: - language-flag: -python - - - name: Upload test reports - if: always() - continue-on-error: true - uses: actions/upload-artifact@v4 - with: - name: test-report-python-${{ matrix.python }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.host.RUNNER }} - path: | - python/python/tests/pytest_report.html - utils/clusters/** - benchmarks/results/** - test-pubsub-python: runs-on: ${{ matrix.host.RUNNER }} needs: get-matrices @@ -153,35 +101,6 @@ jobs: with: submodules: recursive - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python }} - - - name: Build Python wrapper - uses: ./.github/workflows/build-python-wrapper - with: - os: ${{ matrix.host.OS }} - target: ${{ matrix.host.TARGET }} - github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: ${{ matrix.engine.version }} - - - name: Test pubsub with pytest - working-directory: ./python - run: | - source .env/bin/activate - cd python/tests/ - pytest --asyncio-mode=auto -k test_pubsub --html=pytest_report.html --self-contained-html - - - name: Upload test reports - if: always() - continue-on-error: true - uses: actions/upload-artifact@v4 - with: - name: pubsub-test-report-python-${{ matrix.python }}-${{ matrix.engine.type }}-${{ matrix.engine.version }}-${{ matrix.host.RUNNER }} - path: | - python/python/tests/pytest_report.html - lint: runs-on: ubuntu-latest timeout-minutes: 15 @@ -190,39 +109,6 @@ jobs: with: submodules: recursive - - name: lint rust - uses: ./.github/workflows/lint-rust - with: - cargo-toml-folder: ./python - - - name: Install dependencies - if: always() - working-directory: ./python - run: | - python -m pip install --upgrade pip - pip install flake8 isort black - - - name: Lint python with isort - if: always() - working-directory: ./python - run: | - isort . --profile black --check --diff - - - name: Lint python with flake8 - if: always() - working-directory: ./python - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --extend-ignore=E230 --exclude=python/glide/protobuf,.env/* - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=12 --max-line-length=127 --statistics --extend-ignore=E230 --exclude=python/glide/protobuf,.env/* - - - name: Lint python with black - if: always() - working-directory: ./python - run: | - black --check --diff . - build-amazonlinux-latest: if: github.repository_owner == 'valkey-io' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && github.event.inputs.full-matrix == 'true' # TODO matrix @@ -239,108 +125,3 @@ jobs: git --version - uses: actions/checkout@v4 - - - name: Checkout submodules - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - git submodule update --init --recursive - - - name: Install python - run: | - yum install -y python3 - - - name: Build Python wrapper - uses: ./.github/workflows/build-python-wrapper - with: - os: "amazon-linux" - target: "x86_64-unknown-linux-gnu" - github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: "7.2.5" - - - name: Test compatibility with pytest - working-directory: ./python - run: | - source .env/bin/activate - pytest --asyncio-mode=auto -m smoke_test --html=pytest_report.html --self-contained-html - - - name: Upload test reports - if: always() - continue-on-error: true - uses: actions/upload-artifact@v4 - with: - name: smoke-test-report-amazon-linux - path: | - python/python/tests/pytest_report.html - - start-self-hosted-runner: - if: github.event.pull_request.head.repo.owner.login == 'valkey-io' - runs-on: ubuntu-latest - environment: AWS_ACTIONS - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Start self hosted EC2 runner - uses: ./.github/workflows/start-self-hosted-runner - with: - role-to-assume: ${{ secrets.ROLE_TO_ASSUME }} - aws-region: ${{ secrets.AWS_REGION }} - ec2-instance-id: ${{ secrets.AWS_EC2_INSTANCE_ID }} - - test-modules: - needs: [start-self-hosted-runner, load-engine-matrix] - name: Running Module Tests - runs-on: ${{ matrix.host.RUNNER }} - timeout-minutes: 35 - strategy: - fail-fast: false - matrix: - engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }} - python: - - "3.12" - host: - - { - OS: "ubuntu", - NAMED_OS: "linux", - RUNNER: ["self-hosted", "Linux", "ARM64"], - TARGET: "aarch64-unknown-linux-gnu", - } - - steps: - - name: Setup self-hosted runner access - if: ${{ contains(matrix.host.RUNNER, 'self-hosted') }} - run: sudo chown -R $USER:$USER /home/ubuntu/actions-runner/_work/valkey-glide - - - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Setup Python for self-hosted Ubuntu runners - run: | - sudo apt update -y - sudo apt upgrade -y - sudo apt install python3 python3-venv python3-pip -y - - - name: Build Python wrapper - uses: ./.github/workflows/build-python-wrapper - with: - os: ${{ matrix.host.OS }} - target: ${{ matrix.host.TARGET }} - github-token: ${{ secrets.GITHUB_TOKEN }} - engine-version: ${{ matrix.engine.version }} - - - name: Test with pytest - working-directory: ./python - run: | - source .env/bin/activate - cd python/tests/ - pytest --asyncio-mode=auto --tls --cluster-endpoints=${{ secrets.MEMDB_MODULES_ENDPOINT }} -k server_modules --html=pytest_report.html --self-contained-html - - - name: Upload test reports - if: always() - continue-on-error: true - uses: actions/upload-artifact@v4 - with: - name: smoke-test-report-amazon-linux - path: | - python/python/tests/pytest_report.html From 9e75ddb37bd574d75e8bd7e9657dc853fd553762 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Thu, 29 Aug 2024 19:14:21 -0700 Subject: [PATCH 05/12] on all branches Signed-off-by: Yury-Fridlyand --- .github/workflows/java.yml | 1 - .github/workflows/node.yml | 1 - .github/workflows/python.yml | 1 - 3 files changed, 3 deletions(-) diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index abca85aae3..35f1405a75 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -2,7 +2,6 @@ name: Java CI on: push: - branches: ["main"] paths: - glide-core/src/** - submodules/** diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 18631fd1fa..73ee86b41d 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -2,7 +2,6 @@ name: Node on: push: - branches: ["main"] paths: - glide-core/src/** - submodules/** diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 3afcb166e4..f84ef00ec0 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -2,7 +2,6 @@ name: Python tests on: push: - branches: ["main"] paths: - python/** - glide-core/src/** From a9fdf9b438a8397e544e611e9b5ecd025afc0540 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Thu, 29 Aug 2024 19:21:39 -0700 Subject: [PATCH 06/12] fix `create-test-matrices` Signed-off-by: Yury-Fridlyand --- .../workflows/create-test-matrices/action.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/create-test-matrices/action.yml b/.github/workflows/create-test-matrices/action.yml index 8f37ce989b..51b639a773 100644 --- a/.github/workflows/create-test-matrices/action.yml +++ b/.github/workflows/create-test-matrices/action.yml @@ -33,12 +33,12 @@ runs: id: load-engine-matrix shell: bash run: | - : Select server engines to run tests against + echo 'Select server engines to run tests against' if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{ inputs.dispatch-run-full-matrix }}" == "false" ]]; then - : Pick engines marked as '"run": "always"' only - on PR, push or manually triggered job which does not require full matrix + echo 'Pick engines marked as `"run": "always"` only - on PR, push or manually triggered job which does not require full matrix' echo "engine-matrix=$(jq -c '[.[] | select(.run == "always")]' < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT else - : Pick all engines - on cron (schedule) or if manually triggered job requires a full matrix + echo 'Pick all engines - on cron (schedule) or if manually triggered job requires a full matrix' echo "engine-matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT fi echo "engine-matrix=$(cat $GITHUB_OUTPUT)" @@ -47,12 +47,12 @@ runs: id: load-host-matrix shell: bash run: | - : Select runners (VMs) to run tests on + echo 'Select runners (VMs) to run tests on' if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{ inputs.dispatch-run-full-matrix }}" == "false" ]]; then - : Pick runners marked as '"run": "always"' only - on PR, push or manually triggered job which does not require full matrix + echo 'Pick runners marked as '"run": "always"' only - on PR, push or manually triggered job which does not require full matrix' echo 'host-matrix=$(jq -c '[.[] | select(.run == "always")]' < .github/json_matrices/build-matrix.json)' >> $GITHUB_OUTPUT else - : Pick all runners assigned for the chosen client (language) - on cron (schedule) or if manually triggered job requires a full matrix + echo 'Pick all runners assigned for the chosen client (language) - on cron (schedule) or if manually triggered job requires a full matrix' echo 'host-matrix=$(jq -c '[.[] | select(.languages | type == "array" and contains(["${{ inputs.language-name }}"]))]' < .github/json_matrices/build-matrix.json) >> $GITHUB_OUTPUT fi echo "host-matrix=$(cat $GITHUB_OUTPUT)" @@ -61,12 +61,12 @@ runs: id: create-lang-version-matrix shell: bash run: | - : Select language (framework/SDK) versions to run tests on + echo 'Select language (framework/SDK) versions to run tests on' if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{ inputs.dispatch-run-full-matrix }}" == "false" ]]; then - : Pick language versions listed in 'always-run-versions' only - on PR, push or manually triggered job which does not require full matrix + echo 'Pick language versions listed in 'always-run-versions' only - on PR, push or manually triggered job which does not require full matrix' echo 'version-matrix=$(jq -c '[.[] | select(.language == "${{ inputs.language-name }}") | .["always-run-versions"] | map({version: .})]' < .github/json_matrices/supported-language-versions.json) >> $GITHUB_OUTPUT else - : Pick language versions listed in 'versions' - on cron (schedule) or if manually triggered job requires a full matrix + echo 'Pick language versions listed in 'versions' - on cron (schedule) or if manually triggered job requires a full matrix' echo 'version-matrix=$(jq -c '[.[] | select(.language == "${{ inputs.language-name }}") | .versions | map({version: .})]' < .github/json_matrices/supported-language-versions.json) >> $GITHUB_OUTPUT fi echo "version-matrix=$(cat $GITHUB_OUTPUT)" From 07d9e861ed58f4ab1b490616657e83a7e3e75317 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Thu, 29 Aug 2024 19:24:17 -0700 Subject: [PATCH 07/12] fix path filter Signed-off-by: Yury-Fridlyand --- .github/workflows/java.yml | 8 ++++++-- .github/workflows/node.yml | 8 ++++++-- .github/workflows/python.yml | 9 +++++---- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 35f1405a75..24cdd938ba 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -11,7 +11,9 @@ on: - .github/workflows/test-benchmark/action.yml - .github/workflows/lint-rust/action.yml - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json + - .github/workflows/create-test-matrices/action.yml + - .github/json_matrices/** + pull_request: paths: - glide-core/src/** @@ -22,7 +24,9 @@ on: - .github/workflows/test-benchmark/action.yml - .github/workflows/lint-rust/action.yml - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json + - .github/workflows/create-test-matrices/action.yml + - .github/json_matrices/** + workflow_dispatch: inputs: full-matrix: diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 73ee86b41d..1f6d92bd8f 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -13,7 +13,9 @@ on: - .github/workflows/test-benchmark/action.yml - .github/workflows/lint-rust/action.yml - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json + - .github/workflows/create-test-matrices/action.yml + - .github/json_matrices/** + pull_request: paths: - glide-core/src/** @@ -26,7 +28,9 @@ on: - .github/workflows/test-benchmark/action.yml - .github/workflows/lint-rust/action.yml - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json + - .github/workflows/create-test-matrices/action.yml + - .github/json_matrices/** + workflow_dispatch: inputs: full-matrix: diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index f84ef00ec0..19bab2e944 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -13,9 +13,9 @@ on: - .github/workflows/test-benchmark/action.yml - .github/workflows/lint-rust/action.yml - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json - - .github/json_matrices/engine-matrix.json - .github/workflows/start-self-hosted-runner/action.yml + - .github/workflows/create-test-matrices/action.yml + - .github/json_matrices/** pull_request: paths: @@ -29,9 +29,10 @@ on: - .github/workflows/test-benchmark/action.yml - .github/workflows/lint-rust/action.yml - .github/workflows/install-valkey/action.yml - - .github/json_matrices/build-matrix.json - - .github/json_matrices/engine-matrix.json - .github/workflows/start-self-hosted-runner/action.yml + - .github/workflows/create-test-matrices/action.yml + - .github/json_matrices/** + workflow_dispatch: inputs: full-matrix: From 526201b0b03afb401bd44f4b7e0f31baf0326993 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Thu, 29 Aug 2024 19:27:30 -0700 Subject: [PATCH 08/12] fix `create-test-matrices` Signed-off-by: Yury-Fridlyand --- .github/workflows/create-test-matrices/action.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/create-test-matrices/action.yml b/.github/workflows/create-test-matrices/action.yml index 51b639a773..7fbf90f7f9 100644 --- a/.github/workflows/create-test-matrices/action.yml +++ b/.github/workflows/create-test-matrices/action.yml @@ -36,10 +36,10 @@ runs: echo 'Select server engines to run tests against' if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{ inputs.dispatch-run-full-matrix }}" == "false" ]]; then echo 'Pick engines marked as `"run": "always"` only - on PR, push or manually triggered job which does not require full matrix' - echo "engine-matrix=$(jq -c '[.[] | select(.run == "always")]' < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + echo engine-matrix=$(jq -c '[.[] | select(.run == "always")]' < .github/json_matrices/engine-matrix.json) >> $GITHUB_OUTPUT else echo 'Pick all engines - on cron (schedule) or if manually triggered job requires a full matrix' - echo "engine-matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT + echo engine-matrix=$(jq -c . < .github/json_matrices/engine-matrix.json) >> $GITHUB_OUTPUT fi echo "engine-matrix=$(cat $GITHUB_OUTPUT)" @@ -50,10 +50,10 @@ runs: echo 'Select runners (VMs) to run tests on' if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{ inputs.dispatch-run-full-matrix }}" == "false" ]]; then echo 'Pick runners marked as '"run": "always"' only - on PR, push or manually triggered job which does not require full matrix' - echo 'host-matrix=$(jq -c '[.[] | select(.run == "always")]' < .github/json_matrices/build-matrix.json)' >> $GITHUB_OUTPUT + echo host-matrix=$(jq -c '[.[] | select(.run == "always")]' < .github/json_matrices/build-matrix.json) >> $GITHUB_OUTPUT else echo 'Pick all runners assigned for the chosen client (language) - on cron (schedule) or if manually triggered job requires a full matrix' - echo 'host-matrix=$(jq -c '[.[] | select(.languages | type == "array" and contains(["${{ inputs.language-name }}"]))]' < .github/json_matrices/build-matrix.json) >> $GITHUB_OUTPUT + echo host-matrix=$(jq -c '[.[] | select(.languages | type == "array" and contains(["${{ inputs.language-name }}"]))]' < .github/json_matrices/build-matrix.json) >> $GITHUB_OUTPUT fi echo "host-matrix=$(cat $GITHUB_OUTPUT)" @@ -64,9 +64,9 @@ runs: echo 'Select language (framework/SDK) versions to run tests on' if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{ inputs.dispatch-run-full-matrix }}" == "false" ]]; then echo 'Pick language versions listed in 'always-run-versions' only - on PR, push or manually triggered job which does not require full matrix' - echo 'version-matrix=$(jq -c '[.[] | select(.language == "${{ inputs.language-name }}") | .["always-run-versions"] | map({version: .})]' < .github/json_matrices/supported-language-versions.json) >> $GITHUB_OUTPUT + echo version-matrix=$(jq -c '[.[] | select(.language == "${{ inputs.language-name }}") | .["always-run-versions"] | map({version: .})]' < .github/json_matrices/supported-language-versions.json) >> $GITHUB_OUTPUT else echo 'Pick language versions listed in 'versions' - on cron (schedule) or if manually triggered job requires a full matrix' - echo 'version-matrix=$(jq -c '[.[] | select(.language == "${{ inputs.language-name }}") | .versions | map({version: .})]' < .github/json_matrices/supported-language-versions.json) >> $GITHUB_OUTPUT + echo version-matrix=$(jq -c '[.[] | select(.language == "${{ inputs.language-name }}") | .versions | map({version: .})]' < .github/json_matrices/supported-language-versions.json) >> $GITHUB_OUTPUT fi echo "version-matrix=$(cat $GITHUB_OUTPUT)" From b686e0975d9cdc4c1a4d5a78d58fdb86b2f6af96 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Thu, 29 Aug 2024 19:55:23 -0700 Subject: [PATCH 09/12] try run full matrix Signed-off-by: Yury-Fridlyand --- .github/workflows/java.yml | 2 +- .github/workflows/node.yml | 2 +- .github/workflows/python.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 24cdd938ba..4f5892123f 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -57,7 +57,7 @@ jobs: uses: ./.github/workflows/create-test-matrices with: language-name: java - run-full-matrix: ${{ github.event.inputs.full-matrix || 'false' }} + run-full-matrix: ${{ github.event.inputs.full-matrix || 'true' }} test-java: needs: get-matrices diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 1f6d92bd8f..21341af2aa 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -65,7 +65,7 @@ jobs: uses: ./.github/workflows/create-test-matrices with: language-name: node - run-full-matrix: ${{ github.event.inputs.full-matrix || 'false' }} + run-full-matrix: ${{ github.event.inputs.full-matrix || 'true' }} test-node: runs-on: ubuntu-latest diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 19bab2e944..6421c2c2b9 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -68,7 +68,7 @@ jobs: uses: ./.github/workflows/create-test-matrices with: language-name: python - run-full-matrix: ${{ github.event.inputs.full-matrix || 'false' }} + run-full-matrix: ${{ github.event.inputs.full-matrix || 'true' }} test-python: runs-on: ${{ matrix.host.RUNNER }} From 9a6f1786afe768f69b14b1edf3a479ad662dd276 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Thu, 29 Aug 2024 19:57:18 -0700 Subject: [PATCH 10/12] try run full matrix Signed-off-by: Yury-Fridlyand --- .github/workflows/create-test-matrices/action.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/create-test-matrices/action.yml b/.github/workflows/create-test-matrices/action.yml index 7fbf90f7f9..5548ad1ec9 100644 --- a/.github/workflows/create-test-matrices/action.yml +++ b/.github/workflows/create-test-matrices/action.yml @@ -34,7 +34,7 @@ runs: shell: bash run: | echo 'Select server engines to run tests against' - if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{ inputs.dispatch-run-full-matrix }}" == "false" ]]; then + if [[ "${{ inputs.dispatch-run-full-matrix }}" == "false" ]]; then echo 'Pick engines marked as `"run": "always"` only - on PR, push or manually triggered job which does not require full matrix' echo engine-matrix=$(jq -c '[.[] | select(.run == "always")]' < .github/json_matrices/engine-matrix.json) >> $GITHUB_OUTPUT else @@ -48,7 +48,7 @@ runs: shell: bash run: | echo 'Select runners (VMs) to run tests on' - if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{ inputs.dispatch-run-full-matrix }}" == "false" ]]; then + if [[ "${{ inputs.dispatch-run-full-matrix }}" == "false" ]]; then echo 'Pick runners marked as '"run": "always"' only - on PR, push or manually triggered job which does not require full matrix' echo host-matrix=$(jq -c '[.[] | select(.run == "always")]' < .github/json_matrices/build-matrix.json) >> $GITHUB_OUTPUT else @@ -62,7 +62,7 @@ runs: shell: bash run: | echo 'Select language (framework/SDK) versions to run tests on' - if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{ inputs.dispatch-run-full-matrix }}" == "false" ]]; then + if [[ "${{ inputs.dispatch-run-full-matrix }}" == "false" ]]; then echo 'Pick language versions listed in 'always-run-versions' only - on PR, push or manually triggered job which does not require full matrix' echo version-matrix=$(jq -c '[.[] | select(.language == "${{ inputs.language-name }}") | .["always-run-versions"] | map({version: .})]' < .github/json_matrices/supported-language-versions.json) >> $GITHUB_OUTPUT else From 474e0c063dbd0eae8057b8c18f16512b8cc55e50 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Thu, 29 Aug 2024 19:59:16 -0700 Subject: [PATCH 11/12] revert last 2 commits Signed-off-by: Yury-Fridlyand --- .github/workflows/create-test-matrices/action.yml | 6 +++--- .github/workflows/java.yml | 2 +- .github/workflows/node.yml | 2 +- .github/workflows/python.yml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/create-test-matrices/action.yml b/.github/workflows/create-test-matrices/action.yml index 5548ad1ec9..7fbf90f7f9 100644 --- a/.github/workflows/create-test-matrices/action.yml +++ b/.github/workflows/create-test-matrices/action.yml @@ -34,7 +34,7 @@ runs: shell: bash run: | echo 'Select server engines to run tests against' - if [[ "${{ inputs.dispatch-run-full-matrix }}" == "false" ]]; then + if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{ inputs.dispatch-run-full-matrix }}" == "false" ]]; then echo 'Pick engines marked as `"run": "always"` only - on PR, push or manually triggered job which does not require full matrix' echo engine-matrix=$(jq -c '[.[] | select(.run == "always")]' < .github/json_matrices/engine-matrix.json) >> $GITHUB_OUTPUT else @@ -48,7 +48,7 @@ runs: shell: bash run: | echo 'Select runners (VMs) to run tests on' - if [[ "${{ inputs.dispatch-run-full-matrix }}" == "false" ]]; then + if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{ inputs.dispatch-run-full-matrix }}" == "false" ]]; then echo 'Pick runners marked as '"run": "always"' only - on PR, push or manually triggered job which does not require full matrix' echo host-matrix=$(jq -c '[.[] | select(.run == "always")]' < .github/json_matrices/build-matrix.json) >> $GITHUB_OUTPUT else @@ -62,7 +62,7 @@ runs: shell: bash run: | echo 'Select language (framework/SDK) versions to run tests on' - if [[ "${{ inputs.dispatch-run-full-matrix }}" == "false" ]]; then + if [[ "${{ github.event_name }}" == "pull_request" || "${{ github.event_name }}" == "push" || "${{ inputs.dispatch-run-full-matrix }}" == "false" ]]; then echo 'Pick language versions listed in 'always-run-versions' only - on PR, push or manually triggered job which does not require full matrix' echo version-matrix=$(jq -c '[.[] | select(.language == "${{ inputs.language-name }}") | .["always-run-versions"] | map({version: .})]' < .github/json_matrices/supported-language-versions.json) >> $GITHUB_OUTPUT else diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 4f5892123f..24cdd938ba 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -57,7 +57,7 @@ jobs: uses: ./.github/workflows/create-test-matrices with: language-name: java - run-full-matrix: ${{ github.event.inputs.full-matrix || 'true' }} + run-full-matrix: ${{ github.event.inputs.full-matrix || 'false' }} test-java: needs: get-matrices diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 21341af2aa..1f6d92bd8f 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -65,7 +65,7 @@ jobs: uses: ./.github/workflows/create-test-matrices with: language-name: node - run-full-matrix: ${{ github.event.inputs.full-matrix || 'true' }} + run-full-matrix: ${{ github.event.inputs.full-matrix || 'false' }} test-node: runs-on: ubuntu-latest diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 6421c2c2b9..19bab2e944 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -68,7 +68,7 @@ jobs: uses: ./.github/workflows/create-test-matrices with: language-name: python - run-full-matrix: ${{ github.event.inputs.full-matrix || 'true' }} + run-full-matrix: ${{ github.event.inputs.full-matrix || 'false' }} test-python: runs-on: ${{ matrix.host.RUNNER }} From e7f0951449708abce3c26ab649070be69ca442d7 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Fri, 30 Aug 2024 10:17:43 -0700 Subject: [PATCH 12/12] add scheduler Signed-off-by: Yury-Fridlyand --- .github/workflows/nightly.yml | 42 +++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/nightly.yml diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 0000000000..058d6bd690 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,42 @@ +name: Nightly tests + +on: + workflow_dispatch: + inputs: + full-matrix: + description: "Run the full engine, host, and language version matrix" + required: false + type: boolean + default: "false" + name: + required: false + type: string + description: "Test run name" + + schedule: + - cron: "*/5 * * * *" +# - cron: "0 4 * * *" + +concurrency: + group: nightly-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +run-name: + ${{ inputs.name == '' && format('{0} @ {1}', github.ref_name, github.sha) || inputs.name }} + +jobs: + run-full-tests-for-clients: + timeout-minutes: 50 + #runs-on: ${{ matrix.host.RUNNER }} + strategy: + fail-fast: false + matrix: + client: + - java + - python + - node + - csharp + - go + - rust + uses: ./.github/workflows/${{ matrix.client }}.yml + name: Run CI for ${{ matrix.client }}