Merge branch 'release/v0.13' into p2p-stability

.github/workflows/binaries-x86-64.yml

@@ -3,10 +3,19 @@ name: Build binaries (x86-64)
 on:
   workflow_dispatch:
     inputs:
-      profile:
-        description: 'Profile (unused right now)'
+      rust_flags:
+        description: 'Rust flags'
         required: true
-        default: 'debug'
+        default: '-C opt-level=z -C target-cpu=x86-64 -C debuginfo=1'
+      rust_version:
+        description: 'Rust version'
+        required: true
+        default: '1.71.1'
+      strip_binaries:
+        description: 'Strip binaries'
+        required: true
+        default: 'true'
+        possible_values: ['true', 'false']
   push:
     branches:
       - master
@@ -20,80 +29,67 @@ on:
       - release/*
       - p2p-stability
 
-env:
-  rust_stable: 1.71.1
-
 jobs:
   build:
     name: Build binaries (x86-64)
     env:
-      # `-D warnings` means any warnings emitted will cause build to fail
-      RUSTFLAGS: "-C opt-level=z -C target-cpu=x86-64 -C debuginfo=1"
-    runs-on: [yagna-builder, ubuntu-22.04]
+      RUSTFLAGS: ${{ github.event.inputs.rust_flags || '-C opt-level=z -C target-cpu=x86-64 -C debuginfo=1' }}
+    runs-on: [ yagna-builder, ubuntu-22.04 ]
     steps:
-      - name: Clean runner data
-        run: sudo rm -rf ${{ github.workspace }}/*
-
       - name: Checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v4
 
-      - name: Extract branch name
-        id: extract_branch
-        run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
+      - name: Set input variables
+        id: variables
+        run: |
+          echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
+          echo "rust_version=${{ github.event.inputs.rust_version || '1.71.1' }}" >> $GITHUB_OUTPUT
+          echo "rust_flags_md5=`echo ${RUSTFLAGS} | md5sum | head -c 10`" >> $GITHUB_OUTPUT
 
       - name: Musl
         run: |
           sudo apt-get -y install musl musl-tools
           musl-gcc -v
 
-      - name: Install Rust ${{ env.rust_stable }}
-        uses: actions-rs/toolchain@v1
+      - name: Install Rust ${{ steps.variables.outputs.rust_version }}
+        uses: dtolnay/rust-toolchain@master
         with:
-          toolchain: ${{ env.rust_stable }}
+          toolchain: ${{ steps.variables.outputs.rust_version }}
           components: rustfmt, clippy
-          target: x86_64-unknown-linux-musl
-          override: true
-
-      - name: Set build target for cache
-        id: setup_cache_target
-        run: echo "build_target=${CARGO_BUILD_TARGET:-x86_64-unknown-linux-musl}" >> $GITHUB_OUTPUT
+          targets: x86_64-unknown-linux-musl
 
       - name: Load local cache
-        continue-on-error: true
-        run:
+        id: cache_action
+        env:
+          cache_dir: /opt/yagna_cache/${{ steps.variables.outputs.rust_version }}/${{ steps.variables.outputs.branch }}/${{ steps.variables.outputs.rust_flags_md5 }}/binaries_x86/target
+        run: |
           # Uncomment line if you want need to clear cache
-          # rm -rf "/opt/yagna_cache/${{ steps.extract_branch.outputs.branch }}/binaries_x86/target"
-          cp -rp "/opt/yagna_cache/${{ steps.extract_branch.outputs.branch }}/binaries_x86/target" target
+          # rm -rf "$cache_dir"
+          cp -rp "$cache_dir" target || echo "No local cache found"
+          echo "cache_dir=$cache_dir" >> $GITHUB_OUTPUT
 
       - name: Build binaries
         run: |
           cargo build --features static-openssl --target x86_64-unknown-linux-musl -p yagna -p ya-exe-unit -p gftp -p golemsp -p ya-provider
 
-      - name: Copy binaries
-        shell: bash
+      - name: Move target binaries
         run: |
           mkdir build
-          if [ "$RUNNER_OS" == "Linux" ]; then
-            cp target/x86_64-unknown-linux-musl/debug/{yagna,ya-provider,exe-unit,golemsp,gftp} build
-            strip -x build/*
-          elif [ "$RUNNER_OS" == "macOS" ]; then
-            cp target/debug/{yagna,gftp} build
-            strip -x build/*
-          elif [ "$RUNNER_OS" == "Windows" ]; then
-            cp target/debug/{yagna,gftp}.exe build
-          else
-            echo "$RUNNER_OS not supported"
-            exit 1
-          fi
+          mv target/x86_64-unknown-linux-musl/debug/{yagna,ya-provider,exe-unit,golemsp,gftp} build
+
+      - name: Strip binaries
+        if: ${{ github.event.inputs.strip_binaries != 'false' }}
+        run: |
+          strip -x build/*
 
       - name: Upload binaries
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v3
         with:
           name: Yagna ${{ runner.os }}
           path: build
 
       - name: Save local cache
         run: |
           # copy to /opt/yagna_cache and preserve permissions and timestamps
-          rsync -a --delete --mkpath target/ "/opt/yagna_cache/${{ steps.extract_branch.outputs.branch }}/binaries_x86/target"
+          rsync -a --delete --mkpath target/ "${{ steps.cache_action.outputs.cache_dir }}"
 

.github/workflows/clone_master_branch.yml

@@ -0,0 +1,26 @@
+name: Clone master branch
+on:
+  push:
+    branches:
+      - master
+jobs:
+  clone-master:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout with admin privileges
+        uses: actions/checkout@v4
+        with:
+          ssh-key: ${{secrets.ACTIONS_SSH_SECRET}}
+
+      - name: Set Git config
+        run: |
+            git config --local user.email "actions@github.com"
+            git config --local user.name "Github Actions"
+
+      - name: Merge master to master-fast
+        run: |
+            git fetch --unshallow
+            git checkout master-fast
+            git pull
+            git merge master -m "Auto-merge master to master-fast"
+            git push

.github/workflows/fast-ci.yml

@@ -1,3 +1,5 @@
+## This test makes sense if we have separate runner (currently we have only one)
+
 name: Yagna tests (unit + integration)
 
 on:

.github/workflows/fmt-clippy.yml

@@ -15,6 +15,7 @@ on:
   pull_request:
     branches:
       - master
+      - master-fast
       - release/*
       - p2p-stability
 

.github/workflows/integration-test-nightly.yml

@@ -41,7 +41,7 @@ jobs:
     strategy:
       matrix: ${{ fromJson(needs.prepare-matrix.outputs.matrix-json) }}
       fail-fast: false
-    runs-on: [goth, ubuntu-18.04]
+    runs-on: [goth2]
     name: Integration Tests (nightly) @ ${{ matrix.branch }}
     defaults:
       run:
@@ -54,38 +54,30 @@ jobs:
           ref: ${{ matrix.branch }}
 
       - name: Configure Python
-        uses: actions/setup-python@v2
+        continue-on-error: true
+        uses: actions/setup-python@v4
         with:
-          python-version: '3.8.0'
+          python-version: "3.10"
 
-      - name: Configure Poetry
-        uses: Gr1N/setup-poetry@v8
-        with:
-          poetry-version: 1.2.2
+      - name: Install poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+          ln -sf $HOME/.local/bin/poetry /usr/bin/poetry
 
       - name: Install dependencies
         run: |
-          poetry config experimental.new-installer false
           poetry install --no-root
 
-      - name: Disconnect Docker containers from default network
-        continue-on-error: true
-        # related to this issue: https://github.com/moby/moby/issues/23302
+      - name: Cleanup Docker
+        if: always()
         run: |
-          docker network inspect docker_default
-          sudo apt-get install -y jq
-          docker network inspect docker_default | jq ".[0].Containers | map(.Name)[]" | tee /dev/stderr | xargs --max-args 1 -- docker network disconnect -f docker_default
-
-      - name: Remove Docker containers
-        continue-on-error: true
-        run: docker rm -f $(docker ps -a -q)
-
-      - name: Restart Docker daemon
-        # related to this issue: https://github.com/moby/moby/issues/23302
-        run: sudo systemctl restart docker
+          c=$(docker ps -q) && [[ $c ]] && docker kill $c
+          docker system prune -af
 
       - name: Log in to GitHub Docker repository
-        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login docker.pkg.github.com -u ${{github.actor}} --password-stdin
+        run: |
+          rm -rf ~/.docker/config.json
+          echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{github.actor}} --password-stdin
 
       - name: Run test suite
         env:
@@ -117,7 +109,7 @@ jobs:
         # Python version below should agree with the version set up by this job.
         # In the future we'll be able to use the `--all` flag here to remove envs for
         # all Python versions (https://github.com/python-poetry/poetry/issues/3208).
-        run: poetry env remove python3.8
+        run: poetry env remove --all
 
       - name: Send Discord notification on failure
         uses: Ilshidur/action-discord@0.3.2

.github/workflows/integration-test.yml

@@ -3,10 +3,9 @@ name: Goth integration tests (hybrid-net)
 on:
   workflow_dispatch:
     inputs:
-      profile:
-        description: 'Profile (unused right now)'
+      workflow_number:
+        description: 'Id of workflow binaries-x86-64 to get binaries from'
         required: true
-        default: 'debug'
   push:
     branches:
       - master
@@ -24,6 +23,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Wait for build to succeed ( ubuntu )
+        if: github.event_name != 'workflow_dispatch'
         uses: tomchv/wait-my-workflow@v1.1.0
         id: wait-for-build-ubu
         with:
@@ -34,13 +34,15 @@ jobs:
           timeoutSeconds: 3600
 
       - name: Fail if build was not a success ( ubuntu )
+        if: github.event_name != 'workflow_dispatch'
         run: echo job status= ${{ steps.wait-for-build-ubu.outputs.conclusion }} && ${{ fromJSON('["false", "true"]')[steps.wait-for-build-ubu.outputs.conclusion == 'success'] }}
 
   integration-test-groups:
     name: Integration Tests (hybrid-net) group
-    runs-on: [goth2, ubuntu-22.04]
+    runs-on: [goth2]
     needs: test_check
     strategy:
+      fail-fast: false
       matrix:
         group: [1, 2]
     defaults:
@@ -54,40 +56,33 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha || github.sha }}
 
       - name: Configure Python
-        uses: actions/setup-python@v2
+        continue-on-error: true
+        uses: actions/setup-python@v4
         with:
           python-version: "3.10"
 
-      - name: Configure Poetry
-        uses: Gr1N/setup-poetry@v8
-        with:
-          poetry-version: 1.6.1
-          working-directory: "./goth_tests"
+      - name: Install poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+          ln -sf $HOME/.local/bin/poetry /usr/bin/poetry
 
       - name: Install dependencies
         run: |
           poetry install --no-root
 
-      - name: Disconnect Docker containers from default network
-        continue-on-error: true
-        # related to this issue: https://github.com/moby/moby/issues/23302
+      - name: Cleanup Docker
+        if: always()
         run: |
-          docker network inspect docker_default
-          sudo apt-get install -y jq
-          docker network inspect docker_default | jq ".[0].Containers | map(.Name)[]" | tee /dev/stderr | xargs --max-args 1 -- docker network disconnect -f docker_default
-
-      - name: Remove Docker containers
-        continue-on-error: true
-        run: docker rm -f $(docker ps -a -q)
-
-      - name: Restart Docker daemon
-        # related to this issue: https://github.com/moby/moby/issues/23302
-        run: sudo systemctl restart docker
+          c=$(docker ps -q) && [[ $c ]] && docker kill $c
+          docker system prune -af
 
       - name: Log in to GitHub Docker repository
-        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login docker.pkg.github.com -u ${{github.actor}} --password-stdin
+        run: |
+          rm -rf ~/.docker/config.json
+          echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{github.actor}} --password-stdin
 
       - name: Download artifact
+        if: github.event_name != 'workflow_dispatch'
         uses: dawidd6/action-download-artifact@v2
         with:
           workflow: binaries-x86-64.yml
@@ -96,6 +91,15 @@ jobs:
           name: "Yagna Linux"
           path: /tmp/yagna-build
 
+      - name: Download artifact from given run id
+        if: github.event_name == 'workflow_dispatch'
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          run_id: ${{github.event.inputs.workflow_number}}
+          workflow_conclusion: success
+          name: "Yagna Linux"
+          path: /tmp/yagna-build
+
       - name: Run test suite
         env:
           GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -117,11 +121,14 @@ jobs:
       # Only relevant for self-hosted runners
       - name: Remove Poetry virtual env
         if: always()
-        # Python version below should agree with the version set up by this job.
-        # In the future we'll be able to use the `--all` flag here to remove envs for
-        # all Python versions (https://github.com/python-poetry/poetry/issues/3208).
         run: poetry env remove --all
 
+      - name: Cleanup Docker
+        if: always()
+        run: |
+          c=$(docker ps -q) && [[ $c ]] && docker kill $c
+          docker system prune -af
+
   integration-test:
     name: Integration Tests (hybrid-net)
     runs-on: ubuntu-latest