Merge remote-tracking branch 'upstream/main' into python/integ_yuryf_…

…lint_ci Signed-off-by: Yury-Fridlyand <yury.fridlyand@improving.com>
Bit-Quill · Jul 18, 2024 · 08ad6bb · 08ad6bb
2 parents bcfaaaf + 96a60f3
commit 08ad6bb
Show file tree

Hide file tree

Showing 397 changed files with 159,889 additions and 16,588 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml
@@ -73,10 +73,10 @@ body:
       required: true
 
   - type: input
-    id: redis-version
+    id: engine-version
     attributes:
-      label: Redis Version
-      description: E.g. 6.2, 7.0
+      label: Engine type and version
+      description: E.g. Valkey 7.0
     validations:
       required: true
 
@@ -111,7 +111,7 @@ body:
     attributes:
       label: Cluster information
       description: |
-        Redis cluster information, cluster topology, number of shards, number of replicas, used data types.
+       Cluster information, cluster topology, number of shards, number of replicas, used data types.
     validations:
       required: false
 
@@ -120,7 +120,7 @@ body:
     attributes:
       label: Logs
       description: |
-        Client and/or Redis logs.
+        Client and/or server logs.
     validations:
       required: false
 

diff --git a/.github/json_matrices/build-matrix.json b/.github/json_matrices/build-matrix.json
@@ -0,0 +1,55 @@
+[
+    {
+        "OS": "ubuntu",
+        "NAMED_OS": "linux",
+        "RUNNER": "ubuntu-latest",
+        "ARCH": "x64",
+        "TARGET": "x86_64-unknown-linux-gnu",
+        "PACKAGE_MANAGERS": ["pypi", "npm"]
+    },
+    {
+        "OS": "ubuntu",
+        "NAMED_OS": "linux",
+        "RUNNER": ["self-hosted", "Linux", "ARM64"],
+        "ARCH": "arm64",
+        "TARGET": "aarch64-unknown-linux-gnu",
+        "PACKAGE_MANAGERS": ["pypi", "npm"],
+        "CONTAINER": "2_28"
+    },
+    {
+        "OS": "macos",
+        "NAMED_OS": "darwin",
+        "RUNNER": "macos-12",
+        "ARCH": "x64",
+        "TARGET": "x86_64-apple-darwin",
+        "PACKAGE_MANAGERS": ["pypi", "npm"]
+    },
+    {
+        "OS": "macos",
+        "NAMED_OS": "darwin",
+        "RUNNER": "macos-latest",
+        "ARCH": "arm64",
+        "TARGET": "aarch64-apple-darwin",
+        "PACKAGE_MANAGERS": ["pypi", "npm"]
+    },
+    {
+        "OS": "ubuntu",
+        "NAMED_OS": "linux",
+        "ARCH": "arm64",
+        "TARGET": "aarch64-unknown-linux-musl",
+        "RUNNER": ["self-hosted", "Linux", "ARM64"],
+        "IMAGE": "node:alpine",
+        "CONTAINER_OPTIONS": "--user root --privileged --rm",
+        "PACKAGE_MANAGERS": ["npm"]
+    },
+    {
+        "OS": "ubuntu",
+        "NAMED_OS": "linux",
+        "ARCH": "x64",
+        "TARGET": "x86_64-unknown-linux-musl",
+        "RUNNER": "ubuntu-latest",
+        "IMAGE": "node:alpine",
+        "CONTAINER_OPTIONS": "--user root --privileged",
+        "PACKAGE_MANAGERS": ["npm"]
+    }
+]
diff --git a/.github/json_matrices/engine-matrix.json b/.github/json_matrices/engine-matrix.json
@@ -0,0 +1,6 @@
+[
+  {
+    "type": "valkey",
+    "version": "7.2.5"
+  }
+]
diff --git a/.github/workflows/build-node-wrapper/action.yml b/.github/workflows/build-node-wrapper/action.yml
@@ -7,8 +7,8 @@ inputs:
         type: string
         options:
             - amazon-linux
-            - macos-latest
-            - ubuntu-latest
+            - macos
+            - ubuntu
     named_os:
         description: "The name of the current operating system"
         required: false
@@ -29,6 +29,10 @@ inputs:
         description: "Specified target for rust toolchain, ex. x86_64-apple-darwin"
         type: string
         required: true
+    engine-version:
+        description: "Engine version to install"
+        required: true
+        type: string
     publish:
         description: "Enable building the wrapper in release mode"
         required: false
@@ -38,7 +42,7 @@ inputs:
         description: "The NPM scope"
         required: false
         type: string
-        default: "@aws"
+        default: "@valkey"
     github-token:
         description: "GITHUB_TOKEN, GitHub App installation access token"
         required: true
@@ -56,6 +60,7 @@ runs:
               os: ${{ inputs.os }}
               target: ${{ inputs.target }}
               github-token: ${{ inputs.github-token }}
+              engine-version: ${{ inputs.engine-version }}
 
         - name: Create package.json file
           uses: ./.github/workflows/node-create-package-file
@@ -65,14 +70,15 @@ runs:
             named_os: ${{ inputs.named_os }}
             arch: ${{ inputs.arch }}
             npm_scope: ${{ inputs.npm_scope }}
-
+            target: ${{ inputs.target }}
+
         - name: npm install
           shell: bash
           working-directory: ./node
           run: |
               rm -rf node_modules && npm install --frozen-lockfile
               cd rust-client
-              npm install
+              npm install --frozen-lockfile
 
         - name: Build
           shell: bash

diff --git a/.github/workflows/build-python-wrapper/action.yml b/.github/workflows/build-python-wrapper/action.yml
@@ -7,12 +7,16 @@ inputs:
         type: string
         options:
             - amazon-linux
-            - macos-latest
-            - ubuntu-latest
+            - macos
+            - ubuntu
     target:
         description: "Specified target for rust toolchain, ex. x86_64-apple-darwin"
         type: string
         required: true
+    engine-version:
+        description: "Engine version to install"
+        required: true
+        type: string
     publish:
         description: "Enable building the wrapper in release mode"
         required: false
@@ -34,13 +38,16 @@ runs:
               os: ${{ inputs.os }}
               target: ${{ inputs.target }}
               github-token: ${{ inputs.github-token }}
+              engine-version: ${{ inputs.engine-version }}
 
         - name: Install Python software dependencies
           shell: bash
           run: |
+              # Disregarding PEP 668 as it addresses package managers conflicts, which is not applicable in the CI scope.
+              INSTALL_FLAGS=`if [[ "${{ inputs.os }}" =~ .*"macos".*  ]]; then echo "--break-system-packages"; else echo ""; fi`
               python3 -m ensurepip --upgrade || true
-              python3 -m pip install --upgrade pip
-              python3 -m pip install virtualenv mypy-protobuf
+              python3 -m pip install --upgrade pip $INSTALL_FLAGS
+              python3 -m pip install virtualenv mypy-protobuf $INSTALL_FLAGS
 
         - name: Generate protobuf files
           shell: bash
@@ -58,6 +65,5 @@ runs:
               source "$HOME/.cargo/env"
               python3 -m venv .env
               source .env/bin/activate
-              python3 -m pip install --upgrade pip
               python3 -m pip install --no-cache-dir -r requirements.txt
               maturin develop
diff --git a/.github/workflows/csharp.yml b/.github/workflows/csharp.yml
@@ -9,19 +9,22 @@ on:
             - submodules/**
             - .github/workflows/csharp.yml
             - .github/workflows/install-shared-dependencies/action.yml
-            - .github/workflows/install-redis/action.yml
             - .github/workflows/test-benchmark/action.yml
             - .github/workflows/lint-rust/action.yml
+            - .github/workflows/install-valkey/action.yml
+            - .github/json_matrices/build-matrix.json
     pull_request:
         paths:
             - csharp/**
             - glide-core/src/**
             - submodules/**
             - .github/workflows/csharp.yml
             - .github/workflows/install-shared-dependencies/action.yml
-            - .github/workflows/install-redis/action.yml
             - .github/workflows/test-benchmark/action.yml
             - .github/workflows/lint-rust/action.yml
+            - .github/workflows/install-valkey/action.yml
+            - .github/json_matrices/build-matrix.json
+    workflow_dispatch:
 
 permissions:
     contents: read
@@ -31,45 +34,60 @@ concurrency:
     cancel-in-progress: true
 
 jobs:
+    load-engine-matrix:
+        runs-on: ubuntu-latest
+        outputs:
+          matrix: ${{ steps.load-engine-matrix.outputs.matrix }}
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+
+            - name: Load the engine matrix
+              id: load-engine-matrix
+              shell: bash
+              run: echo "matrix=$(jq -c . < .github/json_matrices/engine-matrix.json)" >> $GITHUB_OUTPUT
+
     run-tests:
+        needs: load-engine-matrix
         timeout-minutes: 25
         strategy:
             fail-fast: false
             matrix:
-                redis:
-                    - 6.2.14
-                    - 7.2.3
+                engine: ${{ fromJson(needs.load-engine-matrix.outputs.matrix) }}
                 dotnet:
-                    - '6.0'
+                    # - '6.0'
                     - '8.0'
-                os:
-                    - ubuntu-latest
-                    - macos-latest
-        runs-on: ${{ matrix.os }}
+                host:
+                    - {
+                        OS: ubuntu,
+                        RUNNER: ubuntu-latest,
+                        TARGET: x86_64-unknown-linux-gnu
+                    }
+                    # - {
+                    #     OS: macos,
+                    #     RUNNER: macos-latest,
+                    #     TARGET: aarch64-apple-darwin
+                    # }
+
+        runs-on: ${{ matrix.host.RUNNER }}
 
         steps:
             - uses: actions/checkout@v4
               with:
                   submodules: recursive
 
-            - name: Install redis
-              # TODO: make this step macos compatible: https://github.com/aws/glide-for-redis/issues/781
-              if: ${{ matrix.os == 'ubuntu-latest' }}
-              uses: ./.github/workflows/install-redis
+            - name: Set up dotnet ${{ matrix.dotnet }}
+              uses: actions/setup-dotnet@v4
               with:
-                  redis-version: ${{ matrix.redis }}
-
+                  dotnet-version: ${{ matrix.dotnet }}
+    
             - name: Install shared software dependencies
               uses: ./.github/workflows/install-shared-dependencies
               with:
-                  os: ${{ matrix.os }}
-                  target: ${{ matrix.os == 'ubuntu-latest' && 'x86_64-unknown-linux-gnu' || 'x86_64-apple-darwin' }}
+                  os: ${{ matrix.host.OS }}
+                  target: ${{ matrix.host.TARGET }}
                   github-token: ${{ secrets.GITHUB_TOKEN }}
-
-            - name: Set up dotnet ${{ matrix.dotnet }}
-              uses: actions/setup-dotnet@v4
-              with:
-                  dotnet-version: ${{ matrix.dotnet }}
+                  engine-version: ${{ matrix.engine.version }}
 
             - name: Format
               working-directory: ./csharp
@@ -81,19 +99,21 @@ jobs:
 
             - uses: ./.github/workflows/test-benchmark
               with:
-                  language-flag: -csharp
+                  language-flag: -csharp -dotnet-framework net${{ matrix.dotnet }}
 
             - name: Upload test reports
               if: always()
               continue-on-error: true
               uses: actions/upload-artifact@v4
               with:
-                  name: test-reports-dotnet-${{ matrix.dotnet }}-redis-${{ matrix.redis }}-${{ matrix.os }}
+                  name: test-reports-dotnet-${{ matrix.dotnet }}-redis-${{ matrix.redis }}-${{ matrix.host.RUNNER }}
                   path: |
                       csharp/TestReport.html
                       benchmarks/results/*
                       utils/clusters/**
 
+# TODO Add amazonlinux
+
     lint-rust:
         timeout-minutes: 10
         runs-on: ubuntu-latest