goth + hybrid net (#1667)

* Make hybrid binaries for testing in goth

* Update integration test to run hybrid net goth

* Move hybrid net linux build and goth test to its own files / runs

* Add separate docker-compose file for hybrid net

* Fix cargo lock after rebase

* Cleanup test builds and prepare for PR

* Updated branch after goth 0.8.0 release

* Update goth to latest master after merging hybrid net features

* Test fix for public client / crypto modules

* Reference net branch from master

Co-authored-by: maaktweluit <maaktweluit@users.noreply.github.com>
Co-authored-by: nieznany.sprawiciel <nieznany.sprawiciel@gmail.com>

.github/workflows/binaries-x86-64-hybrid-net.yml

@@ -0,0 +1,75 @@
+name: Build binaries (x86-64) hybrid-net
+
+on:
+  push:
+    branches:
+      - master
+      - release/*
+  pull_request:
+    branches:
+      - master
+      - release/*
+
+jobs:
+  build:
+    name: Build binaries (x86-64) hybrid-net
+    env:
+      # `-D warnings` means any warnings emitted will cause build to fail
+      RUSTFLAGS: "-D warnings -C opt-level=z -C target-cpu=x86-64 -C debuginfo=1"
+      X86_64_PC_WINDOWS_MSVC_OPENSSL_DIR: c:/vcpkg/installed/x64-windows
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v1
+
+      - name: Install last stable Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          components: rustfmt, clippy
+
+      - name: Check lockfile
+        uses: actions-rs/cargo@v1
+        with:
+          command: tree
+          args: --locked
+
+      - name: Install openssl ( Windows only )
+        if: runner.os == 'Windows'
+        run: |
+          vcpkg install openssl:x64-windows openssl:x64-windows-static
+          vcpkg list
+          vcpkg integrate install
+
+      - name: Build binaries
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          args: --workspace --features hybrid-net
+
+      - name: Copy binaries
+        shell: bash
+        run: |
+          mkdir build
+          if [ "$RUNNER_OS" == "Linux" ]; then
+            cp target/debug/{yagna,ya-provider,exe-unit,golemsp,gftp} build
+            strip -x build/*
+          elif [ "$RUNNER_OS" == "macOS" ]; then
+            cp target/debug/{yagna,gftp} build
+            strip -x build/*
+          elif [ "$RUNNER_OS" == "Windows" ]; then
+            cp target/debug/{yagna,gftp}.exe build
+          else
+            echo "$RUNNER_OS not supported"
+            exit 1
+          fi
+
+      - name: Upload binaries
+        uses: actions/upload-artifact@v1
+        with:
+          name: Yagna ${{ runner.os }} hybrid-net
+          path: build

.github/workflows/integration-test-hybrid-net.yml

@@ -0,0 +1,111 @@
+name: Goth integration tests hybrid-net
+
+on:
+  push:
+    branches:
+      - master
+      - release/*
+  pull_request:
+    branches:
+      - master
+      - release/*
+
+jobs:
+  test_check:
+    name: Check test results
+    runs-on: ubuntu-latest
+    steps:
+      - name: Wait for build to succeed ( ubuntu )
+        uses: tomchv/wait-my-workflow@v1.1.0
+        id: wait-for-build-ubu
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          checkName: "Build binaries (x86-64) hybrid-net (ubuntu-latest)"
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          timeoutSeconds: 7200
+
+      - name: Fail if build was not a success ( ubuntu )
+        run: echo job status= ${{ steps.wait-for-build-ubu.outputs.conclusion }} && ${{ fromJSON('["false", "true"]')[steps.wait-for-build-ubu.outputs.conclusion == 'success'] }}
+
+  integration-test:
+    name: Integration Tests
+    runs-on: goth
+    needs: test_check
+    defaults:
+      run:
+        working-directory: './goth_tests'
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+
+      - name: Configure Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.8.0'
+
+      - name: Configure Poetry
+        uses: Gr1N/setup-poetry@v4
+        with:
+          poetry-version: 1.1.4
+          working-directory: './goth_tests'
+
+      - name: Install dependencies
+        run: poetry install --no-root
+
+      - name: Disconnect Docker containers from default network
+        continue-on-error: true
+        # related to this issue: https://github.com/moby/moby/issues/23302
+        run: |
+          docker network inspect docker_default
+          sudo apt-get install -y jq
+          docker network inspect docker_default | jq ".[0].Containers | map(.Name)[]" | tee /dev/stderr | xargs --max-args 1 -- docker network disconnect -f docker_default
+
+      - name: Remove Docker containers
+        continue-on-error: true
+        run: docker rm -f $(docker ps -a -q)
+
+      - name: Restart Docker daemon
+        # related to this issue: https://github.com/moby/moby/issues/23302
+        run: sudo systemctl restart docker
+
+      - name: Log in to GitHub Docker repository
+        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login docker.pkg.github.com -u ${{github.actor}} --password-stdin
+
+      - name: Download artifact
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: binaries-x86-64-hybrid-net.yml
+          commit: ${{github.event.pull_request.head.sha || github.sha}}
+          workflow_conclusion: success
+          name: 'Yagna Linux hybrid-net'
+          path: /tmp/yagna-build
+
+      - name: Run test suite
+        env:
+          GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          poetry run poe goth-assets
+          poetry run poe goth-tests --config-override docker-compose.build-environment.binary-path=/tmp/yagna-build --config-override docker-compose.build-environment.compose-file=docker-compose-hybrid-net.yml
+
+      - name: Upload test logs
+        uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: goth-logs
+          path: /tmp/goth-tests
+
+      # Only relevant for self-hosted runners
+      - name: Remove test logs
+        if: always()
+        run: rm -rf /tmp/goth-tests
+
+      # Only relevant for self-hosted runners
+      - name: Remove Poetry virtual env
+        if: always()
+        # Python version below should agree with the version set up by this job.
+        # In the future we'll be able to use the `--all` flag here to remove envs for
+        # all Python versions (https://github.com/python-poetry/poetry/issues/3208).
+        run: poetry env remove python3.8

core/net/Cargo.toml

@@ -11,8 +11,8 @@ hybrid-net = ["ya-relay-client", "ya-relay-proto", "ya-sb-proto", "bytes", "eths
 
 [dependencies]
 ya-core-model = { version = "^0.4", features=["net", "identity"] }
-ya-relay-client = { git = "https://github.com/golemfactory/ya-relay.git", rev = "26a4099c7c55f8ae53ae459e4c58f3c639a183af", optional = true }
-ya-relay-proto = { git = "https://github.com/golemfactory/ya-relay.git", rev = "26a4099c7c55f8ae53ae459e4c58f3c639a183af", features = ["codec"], optional = true }
+ya-relay-client = { git = "https://github.com/golemfactory/ya-relay.git", rev = "1052108b786c75c996f9c6db8bc52d8ddab04c45", optional = true }
+ya-relay-proto = { git = "https://github.com/golemfactory/ya-relay.git", rev = "1052108b786c75c996f9c6db8bc52d8ddab04c45", features = ["codec"], optional = true }
 ya-sb-proto = { version = "0.4", optional = true }
 ya-service-api = "0.1"
 ya-service-api-interfaces = "0.1"

goth_tests/pyproject.toml

@@ -23,7 +23,7 @@ pytest-asyncio = "^0.14"
 # goth = "^0.7"
 # to use development goth version uncomment below
 # goth = { git = "https://github.com/golemfactory/goth.git", branch = "your-branch-here" }
-goth = { git = "https://github.com/golemfactory/goth.git", rev = "7882c57cdba8db5715d081f447aced1ce12a0412" }
+goth = { git = "https://github.com/golemfactory/goth.git", rev = "8a01f6317f6efb592e4e6d7d0e0137ae8ae785e7" }
 
 [tool.poetry.dev-dependencies]
 black = "^20.8b1"