From ad4c34961b41ea3b69c8571e74b068d660ea32bf Mon Sep 17 00:00:00 2001 From: epikichi Date: Tue, 27 Sep 2022 06:59:55 -0400 Subject: [PATCH 1/6] initial workflow rework --- .github/workflows/build.yml | 32 +++++ .github/workflows/ci.yml | 48 +++++++ .github/workflows/cla.yml | 13 +- .github/workflows/deploy.devnet.yml | 164 +++++++++++++++++++++++ .github/workflows/deploy.testnet.yml | 186 ++++++++++++++++++++++++++ .github/workflows/deploy_edgenet.yaml | 15 ++- .github/workflows/lint.yml | 28 ++++ .github/workflows/loadbot.yml | 154 +++++++++++++++++++++ .github/workflows/main.yml | 83 ------------ .github/workflows/pandoras_box.yml | 157 ++++++++++++++++++++++ .github/workflows/release.yml | 4 +- .github/workflows/security.yml | 26 ++++ .github/workflows/test.yml | 36 +++++ 13 files changed, 850 insertions(+), 96 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/deploy.devnet.yml create mode 100644 .github/workflows/deploy.testnet.yml create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/loadbot.yml delete mode 100644 .github/workflows/main.yml create mode 100644 .github/workflows/pandoras_box.yml create mode 100644 .github/workflows/security.yml create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000000..edb5f74542 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,32 @@ +--- +name: Build +on: # yamllint disable-line rule:truthy + workflow_dispatch: + workflow_call: + +jobs: + go_build: + name: Polygon Edge + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Setup Go environment + uses: actions/setup-go@v3.3.0 + with: + go-version: 1.17.x + + - name: Build Polygon Edge + run: go build -ldflags="-s -w -linkmode external -extldflags "-static" -X \"github.com/0xPolygon/polygon-edge/versioning.Version=${GITHUB_REF_NAME}@${GITHUB_SHA}\"" && tar -czvf polygon-edge.tar.gz polygon-edge + env: + CC: gcc + CXX: g++ + GOARC: amd64 + GOOS: linux + + - name: 'Upload Artifact' + uses: actions/upload-artifact@v3 + with: + name: polygon-edge + path: polygon-edge.tar.gz + retention-days: 3 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..e76c1f1c26 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,48 @@ +--- +name: Pull Request CI +on: # yamllint disable-line rule:truthy + workflow_dispatch: {} + pull_request: + branches: + - 'develop' + paths: + - '**.go' + +jobs: + build: + name: Build + uses: 0xpolygon/polygon-edge/.github/workflows/build.yml@develop + + test: + name: Test + uses: 0xpolygon/polygon-edge/.github/workflows/test.yml@develop + needs: build + + gosec: + name: Gosec + runs-on: ubuntu-latest + env: + GO111MODULE: on # yamllint disable-line rule:truthy + steps: + - name: Checkout Source + uses: actions/checkout@v2 + - name: Run Gosec Security Scanner + uses: securego/gosec@master + with: + args: ./... + + notify-deployer: + needs: test + runs-on: ubuntu-latest + if: github.event_name != 'pull_request' + steps: + - name: Extract branch name + shell: bash + run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + id: extract_branch + - name: Notify Deployer + env: + ARTIFACT_NAME: polygon-edge_${{ github.sha }}_${{ steps.extract_branch.outputs.branch }} + COMMITER: ${{ github.actor }} + run: | + curl -X POST ${{ secrets.DEPLOYER_API }} -H 'Accept: application/vnd.github.everest-preview+json' -u ${{ secrets.DEPLOYER_ACCESS_TOKEN }} --data '{"event_type": "new_artefact", "client_payload": { "commiter":"'"$COMMITER"'","commit":"'"$GITHUB_SHA"'","branch":"'"$GITHUB_REF"'","repository": "'"$GITHUB_REPOSITORY"'","artefact_name":"'"$ARTIFACT_NAME"'" }}' diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml index 8b8f3fc953..1a3e412870 100644 --- a/.github/workflows/cla.yml +++ b/.github/workflows/cla.yml @@ -1,9 +1,14 @@ +--- name: "CLA Assistant" -on: +on: # yamllint disable-line rule:truthy issue_comment: - types: [created] + types: + - created pull_request_target: - types: [opened,closed,synchronize] + types: + - opened + - closed + - synchronize jobs: CLAssistant: @@ -16,7 +21,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # the below token should have repo scope and must be manually added by you in the repository's secret - PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }} + PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} with: path-to-signatures: 'cla.json' path-to-document: 'https://github.com/0xPolygon/polygon-edge/blob/develop/CLA.md' diff --git a/.github/workflows/deploy.devnet.yml b/.github/workflows/deploy.devnet.yml new file mode 100644 index 0000000000..da8172b6fb --- /dev/null +++ b/.github/workflows/deploy.devnet.yml @@ -0,0 +1,164 @@ +--- +name: DevNet Workflow + +on: # yamllint disable-line rule:truthy + workflow_dispatch: + push: + branches: + - develop + +permissions: + id-token: write + contents: read + security-events: write + +jobs: + gosec: + name: Gosec and Publish + uses: 0xpolygon/polygon-edge/.github/workflows/security.yml@develop + + build: + name: Build + uses: 0xpolygon/polygon-edge/.github/workflows/build.yml@develop + + test: + name: Test + uses: 0xpolygon/polygon-edge/.github/workflows/test.yml@develop + needs: build + + deploy_devnet: + name: Deploy DevNet + needs: build + concurrency: deploy_devnet + environment: + name: devnet + url: https://rpc.us-east-1.testing.psdk.io/ + runs-on: ubuntu-latest + steps: + - name: Download Polygon Edge Artifact + uses: actions/download-artifact@v3 + with: + name: polygon-edge + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Notify Slack + uses: slackapi/slack-github-action@v1.22.0 + if: false + continue-on-error: true + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEPLOYMENT_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + with: + payload: | + { + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "Devnet Deployment - Started" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Devnet Deployment Status: ${{ job.status }}\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Job Status>" + } + } + ] + } + + - name: Publish Binary package to S3 + run: | + aws s3 cp ./polygon-edge.tar.gz s3://${{ secrets.POLYGON_EDGE_ARTIFACT_BUCKET }} --metadata "{\"commit_sha\":\"${GITHUB_SHA}\"}" + + - name: Deploy DevNet + continue-on-error: true + env: + VALIDATOR_ASGS: ${{ secrets.VALIDATOR_ASGS }} + run: | + # Parallel deploy all nodes, relying on ALB Health Checks + echo "Deploying new Devnet" + echo "--------------------" + + # Should finish relatively fast. Can add a check to guard against pre-mature cycle + echo "Cleaning up the data directories and stopping Polygon-Edge Devnet service..." + aws ssm send-command --document-name polygon-edge-validators-maintenance-clean-data --targets Key=tag:aws:autoscaling:groupName,Values=$VALIDATOR_ASGS >> /dev/null + + for vasg in ${VALIDATOR_ASGS//,/ } + do + instance_id=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-names $vasg --query 'AutoScalingGroups[0].Instances[0].InstanceId' --output text) + echo "Draining $instance_id in $vasg" + irid=$(aws autoscaling start-instance-refresh --auto-scaling-group-name $vasg --query InstanceRefreshId --output text) + done + + echo "Waiting for instance to clear health check..." + pending=true + + until [ $pending != true ] + do + pending=false + sleep 10 + for vasg in ${VALIDATOR_ASGS//,/ } + do + refresh_status=$(aws autoscaling describe-instance-refreshes --auto-scaling-group-name $vasg --query 'InstanceRefreshes[0].Status' --output text) + echo "ASG: $vasg" + echo "Status: $refresh_status" + + if [ "$refresh_status" == "Successful" ] + then + VALIDATOR_ASGS=${VALIDATOR_ASGS//$vasg/} + instance_id=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-names $vasg --query 'AutoScalingGroups[0].Instances[0].InstanceId' --output text) + + echo "-------------------------------------------------" + echo "New validator: $instance_id successfully deployed" + echo "-------------------------------------------------" + fi + pending=true + done + echo "" + done + echo "Devnet Deployment Complete" + + - name: Notify Slack - Failures + uses: slackapi/slack-github-action@v1.22.0 + if: ${{ job.status != 'success' }} + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEPLOYMENT_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + with: + payload: | + { + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "Devnet Deployment - Failed" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Job Status>" + } + }, + { + "type": "divider" + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Commit SHA: <${{ github.server_url }}/${{ github.repository }}/tree/${{ github.sha }}|${{ github.sha }}>\nBranch: <${{ github.server_url }}/${{ github.repository }}/tree/${{ github.ref_name }}|${{ github.ref_name}}>" + } + } + ] + } diff --git a/.github/workflows/deploy.testnet.yml b/.github/workflows/deploy.testnet.yml new file mode 100644 index 0000000000..c5fe9d3530 --- /dev/null +++ b/.github/workflows/deploy.testnet.yml @@ -0,0 +1,186 @@ +--- +name: TestNet Workflow +on: # yamllint disable-line rule:truthy + workflow_dispatch: + push: + branches: + - 'release/**' + +permissions: + id-token: write + contents: read + security-events: write + +jobs: + gosec: + name: Gosec and Publish + uses: 0xpolygon/polygon-edge/.github/workflows/security.yml@develop + + build: + uses: 0xpolygon/polygon-edge/.github/workflows/build.yml@develop + name: Build + + test: + uses: 0xpolygon/polygon-edge/.github/workflows/test.yml@develop + name: Test + needs: build + + deploy_testnet: + name: Update TestNet + needs: build + concurrency: deploy_testnet + environment: + name: testnet + url: https://rpc.us-east-1.tn.testing.psdk.io/ + runs-on: ubuntu-latest + steps: + - name: Download Polygon Edge Artifact + uses: actions/download-artifact@v3 + with: + name: polygon-edge + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Notify Slack + uses: slackapi/slack-github-action@v1.22.0 + continue-on-error: true + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEPLOYMENT_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + with: + payload: | + { + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "Testnet Deployment - Started" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Testnet Deployment Status: ${{ job.status }}\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Job Status>" + } + } + ] + } + + - name: Publish Binary package to S3 + run: | + aws s3 cp ./polygon-edge.tar.gz s3://${{ secrets.POLYGON_EDGE_ARTIFACT_BUCKET }} --metadata "{\"commit_sha\":\"${GITHUB_SHA}\"}" + - name: Deploy to Testnet + continue-on-error: true + env: + VALIDATOR_ASGS: ${{ secrets.VALIDATOR_ASGS }} + run: | + echo "Deploying new TestNet..." + for vasg in ${VALIDATOR_ASGS//,/ } + do + instance_id=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-names $vasg --query 'AutoScalingGroups[0].Instances[0].InstanceId' --output text) + + echo "Draining $instance_id in $vasg" + irid=$(aws autoscaling start-instance-refresh --auto-scaling-group-name $vasg --query InstanceRefreshId --output text) + + echo "Waiting for instance to clear health check..." + refresh_status=Pending + until [ $refresh_status != "InProgress" ] && [ $refresh_status != "Pending" ] + do + sleep 10 + refresh_status=$(aws autoscaling describe-instance-refreshes --instance-refresh-ids $irid --auto-scaling-group-name $vasg --query 'InstanceRefreshes[0].Status' --output text) + + echo "ASG: $vasg" + echo "Status: $refresh_status" + done + + if [ "$refresh_status" != "Successful" ] + then + echo "instance refresh failed." + echo "autoscaling group: $vasg" + echo "instance refresh id: $irid" + exit 1 + fi + + instance_id=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-names $vasg --query 'AutoScalingGroups[0].Instances[0].InstanceId' --output text) + echo "-------------------------------------------------" + echo "New validator: $instance_id successfully deployed" + echo "-------------------------------------------------" + + done + echo "Testnet Deployment Complete" + + - name: Notify Slack + uses: slackapi/slack-github-action@v1.22.0 + continue-on-error: true + if: ${{ job.status != 'success' }} + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEPLOYMENT_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + with: + payload: | + { + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "Testnet Deployment - Failed" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Job Status>" + } + }, + { + "type": "divider" + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Commit SHA: <${{ github.server_url }}/${{ github.repository }}/tree/${{ github.sha }}|${{ github.sha }}>\nPre-release: <${{ github.server_url }}/${{ github.repository }}/tree/${{ github.ref_name }}|${{ github.ref_name}}>" + } + } + ] + } + + pandoras_box: + uses: 0xpolygon/polygon-edge/.github/workflows/pandoras_box.yml@develop + name: Pandora's Box + needs: deploy_testnet + secrets: + SLACK_PERFORMANCE_WEBHOOK_URL: ${{ secrets.SLACK_PERFORMANCE_WEBHOOK_URL }} + PANDORAS_TARGET: ${{ secrets.PANDORAS_TARGET }} + PANDORAS_MNEMONIC: ${{ secrets.PANDORAS_MNEMONIC }} + with: + runner: testnet + environment: testnet + transaction_batch: '20' + transaction_count: '2000' + + loadbot: + uses: 0xpolygon/polygon-edge/.github/workflows/loadbot.yml@develop + name: Loadbot + needs: deploy_testnet + secrets: + SLACK_PERFORMANCE_WEBHOOK_URL: ${{ secrets.SLACK_PERFORMANCE_WEBHOOK_URL }} + LOADBOT_JSONRPC_URL: ${{ secrets.LOADBOT_JSONRPC_URL }} + LOADBOT_SENDER_ADDRESS: ${{ secrets.LOADBOT_SENDER_ADDRESS }} + LOADBOT_GRPC_ADDRESS: ${{ secrets.LOADBOT_GRPC_ADDRESS }} + LOADBOT_0x1AB8C3df809b85012a009c0264eb92dB04eD6EFa: ${{ secrets.LOADBOT_0x1AB8C3df809b85012a009c0264eb92dB04eD6EFa }} + with: + runner: testnet + environment: testnet + chain_id: '750' + transaction_amount: '100' + transaction_count: '2000' + transactions_per_second: '100' diff --git a/.github/workflows/deploy_edgenet.yaml b/.github/workflows/deploy_edgenet.yaml index 53d5e242c5..64df993c4e 100644 --- a/.github/workflows/deploy_edgenet.yaml +++ b/.github/workflows/deploy_edgenet.yaml @@ -1,6 +1,7 @@ -on: +--- +on: # yamllint disable-line rule:truthy release: - types: + types: - published jobs: @@ -13,7 +14,7 @@ jobs: wget https://github.com/Trapesys/aws-commander/releases/download/v0.2.0/aws-commander_0.2.0_Linux_x86_64.tar.gz tar -xf aws-commander_0.2.0_Linux_x86_64.tar.gz sudo mv aws-commander /usr/local/bin - + - name: Checkout code uses: actions/checkout@v3 @@ -22,8 +23,8 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.EDGENET_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.EDGENET_AWS_SECRET_ACCESS_KEY }} run: > - /usr/local/bin/aws-commander - -instances i-039f7c0b3328a00f8,i-035b9f2d78cfb8ea9,i-00a6c7cb3a213f21f,i-03ac2f42ddcba6120 - -mode ansible - -playbook .github/workflows/ansible/update_edgenet.yaml + /usr/local/bin/aws-commander + -instances i-039f7c0b3328a00f8,i-035b9f2d78cfb8ea9,i-00a6c7cb3a213f21f,i-03ac2f42ddcba6120 + -mode ansible + -playbook .github/workflows/ansible/update_edgenet.yaml -aws-zone us-west-2 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000000..8c2070edf4 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,28 @@ +--- +name: Lint +on: # yamllint disable-line rule:truthy + push: + branches-ignore: + - 'develop' + - 'release/**' + tags-ignore: + - 'v*' + paths: + - '**.go' + workflow_call: {} + workflow_dispatch: {} + +jobs: + golangci_lint: + runs-on: ubuntu-latest + steps: + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: 1.17.x + + - name: Checkout code + uses: actions/checkout@v3 + + - name: Lint + uses: golangci/golangci-lint-action@v3 diff --git a/.github/workflows/loadbot.yml b/.github/workflows/loadbot.yml new file mode 100644 index 0000000000..0581e64534 --- /dev/null +++ b/.github/workflows/loadbot.yml @@ -0,0 +1,154 @@ +--- + +name: Loadbot +on: # yamllint disable-line rule:truthy + workflow_dispatch: + inputs: + runner: + description: 'The runner to execute on' + default: 'ubuntu-latest' + type: choice + options: + - ubuntu-latest + - devnet + - testnet + environment: + description: 'The environment to run against' + required: false + type: environment + chain_id: + default: '750' + description: The chain ID of the network to test + type: string + transaction_count: + default: '1000' + description: The number of transactions to send + type: string + transactions_per_second: + default: '100' + description: The amount of transactions per second to send + type: string + transaction_value: + default: '100' + description: The value per transactions to send + type: string + workflow_call: + inputs: + chain_id: + description: The chain ID of the network to test + type: string + transaction_count: + description: The number of transactions to send + type: string + transactions_per_second: + description: The amount of transactions per second to send + type: string + transaction_amount: + description: The amount per transactions to send + type: string + runner: + required: true + type: string + description: The runner label to use + environment: + description: 'The environment to run against' + type: string + secrets: + SLACK_PERFORMANCE_WEBHOOK_URL: + required: true + LOADBOT_SENDER_ADDRESS: + required: true + LOADBOT_GRPC_ADDRESS: + required: true + LOADBOT_JSONRPC_URL: + required: true + LOADBOT_0x1AB8C3df809b85012a009c0264eb92dB04eD6EFa: + required: true +jobs: + run_loadbot: + name: Run Loadbot + concurrency: ${{ github.event.inputs.environment }}_performance + environment: ${{ github.event.inputs.environment }} + runs-on: ${{ github.event.inputs.runner }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Setup Go environment + uses: actions/setup-go@v3.3.0 + with: + go-version: 1.17.x + - name: Install JQ + run: | + mkdir -p $HOME/.local/bin + curl -sLo $HOME/.local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 && chmod +x $HOME/.local/bin/jq + echo "$HOME/.local/bin" >> $GITHUB_PATH + - id: loadbot + name: Run Loadbot + continue-on-error: true + run: | + go run . loadbot --jsonrpc ${{ secrets.LOADBOT_JSONRPC_URL }} --grpc-address ${{ secrets.LOADBOT_GRPC_ADDRESS }} --sender ${{ secrets.LOADBOT_SENDER_ADDRESS }} --count ${{ github.event.inputs.transaction_count}} --value ${{ github.event.inputs.transaction_value }} --tps ${{ github.event.inputs.transactions_per_second }} --chain-id ${{ github.event.inputs.chain_id }} --detailed --json > loadbotOutput.json + cat loadbotOutput.json + echo "::set-output name=tps::$(cat loadbotOutput.json | jq -r '.approx_tps')" + echo "::set-output name=totalTx::$(cat loadbotOutput.json | jq -r '.count_data.total')" + echo "::set-output name=failedTx::$(cat loadbotOutput.json | jq -r '.count_data.failed')" + env: + LOADBOT_0x1AB8C3df809b85012a009c0264eb92dB04eD6EFa: ${{ secrets.LOADBOT_0x1AB8C3df809b85012a009c0264eb92dB04eD6EFa }} + - name: Notify Slack + uses: slackapi/slack-github-action@v1.22.0 + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_PERFORMANCE_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + with: + payload: | + { + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "Loadbot Results" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Approximate TPS: `${{ steps.loadbot.outputs.tps }}`" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Total Transactions: `${{ steps.loadbot.outputs.totalTx }}`" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Failed Transactions: `${{ steps.loadbot.outputs.failedTx }}`" + } + }, + { + "type": "divider" + }, + { + "type": "context", + "elements": [ + { + "type": "mrkdwn", + "text": "Environment: `${{ github.event.inputs.environment }}`" + }, + { + "type": "mrkdwn", + "text": "JSON-RPC Endpoint: ${{ secrets.LOADBOT_JSONRPC_URL }}" + }, + { + "type": "mrkdwn", + "text": "Workflow: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Results>" + } + ] + } + ] + } diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index b34eb43aad..0000000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,83 +0,0 @@ -on: - push: - branches: - - main - - develop - pull_request: - -jobs: - lint: - runs-on: ubuntu-latest - steps: - - name: Install Go - uses: actions/setup-go@v3 - with: - go-version: 1.17.x - - - name: Checkout code - uses: actions/checkout@v3 - - - name: Lint - uses: golangci/golangci-lint-action@v3 - - test: - runs-on: ubuntu-latest - steps: - - name: Install Go - uses: actions/setup-go@v3 - with: - go-version: 1.17.x - - - name: Checkout code - uses: actions/checkout@v3 - with: - submodules: recursive - - # We need to build the binary with the race flag enabled - # because it will get picked up and run during e2e tests - # and the e2e tests should error out if any kind of race is found - - name: Go build with race - run: CGO_ENABLED=1 GOOS=linux go build -race -a -o artifacts/polygon-edge . - - - name: Add artifacts directory to the path - run: echo "$(pwd)/artifacts" >> $GITHUB_PATH - - - name: Go test - run: go test -coverprofile coverage.out -timeout 28m ./... - - - name: Upload coverage file to Codecov - uses: codecov/codecov-action@v3 - with: - files: coverage.out - - - name: Go build without race - run: CGO_ENABLED=0 GOOS=linux go build -a -o artifacts/polygon-edge . - - - name: Extract branch name - if: github.event_name != 'pull_request' - shell: bash - run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" - id: extract_branch - - - name: Prepare artifact for Deployer - if: github.event_name != 'pull_request' - uses: actions/upload-artifact@v2 - with: - name: polygon-edge_${{ github.sha }}_${{ steps.extract_branch.outputs.branch }} - path: artifacts/ - - notify-deployer: - needs: test - runs-on: ubuntu-latest - if: github.event_name != 'pull_request' - steps: - - name: Extract branch name - shell: bash - run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" - id: extract_branch - - name: Notify Deployer - env: - ARTIFACT_NAME: polygon-edge_${{ github.sha }}_${{ steps.extract_branch.outputs.branch }} - COMMITER: ${{ github.actor }} - run: | - curl -X POST ${{ secrets.DEPLOYER_API }} -H 'Accept: application/vnd.github.everest-preview+json' -u ${{ secrets.DEPLOYER_ACCESS_TOKEN }} --data '{"event_type": "new_artefact", "client_payload": { "commiter":"'"$COMMITER"'","commit":"'"$GITHUB_SHA"'","branch":"'"$GITHUB_REF"'","repository": "'"$GITHUB_REPOSITORY"'","artefact_name":"'"$ARTIFACT_NAME"'" }}' diff --git a/.github/workflows/pandoras_box.yml b/.github/workflows/pandoras_box.yml new file mode 100644 index 0000000000..ecb38f1a95 --- /dev/null +++ b/.github/workflows/pandoras_box.yml @@ -0,0 +1,157 @@ +--- +name: Pandora's Box +on: # yamllint disable-line rule:truthy + workflow_dispatch: + inputs: + runner: + description: 'The runner to execute on' + default: 'ubuntu-latest' + type: choice + options: + - ubuntu-latest + - devnet + - testnet + environment: + description: 'The environment to run against' + required: false + type: environment + transaction_count: + default: '2000' + description: The number of transactions to send + type: string + transaction_batch: + default: '20' + description: The transaction batch size + type: string + workflow_call: + inputs: + transaction_count: + required: true + description: The number of transactions to send + type: string + transaction_batch: + required: true + description: The transaction batch size + type: string + environment: + description: 'The environment to run against' + type: string + runner: + required: true + type: string + description: The runner label to use + secrets: + SLACK_PERFORMANCE_WEBHOOK_URL: + required: true + PANDORAS_TARGET: + required: true + PANDORAS_MNEMONIC: + required: true + +jobs: + open_pandoras_box: + name: Open Pandora's Box + concurrency: ${{ github.event.inputs.environment }}_performance + environment: ${{ github.event.inputs.environment }} + runs-on: ${{ github.event.inputs.runner }} + steps: + - name: Checkout Pandora's Box Repository + uses: actions/checkout@v3 + with: + repository: trapesys/pandoras-box + - uses: actions/setup-node@v3 + with: + node-version: 16 + cache: 'npm' + - name: Prepare Pandora's Box + run: | + npm install -g yarn + yarn install && yarn build && yarn link + echo "$HOME/.yarn/bin" >> $GITHUB_PATH + - name: Notify Slack + if: false + uses: slackapi/slack-github-action@v1.22.0 + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_PERFORMANCE_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + with: + payload: | + { + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "GitHub Action - Pandora's Box - Starting" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Status>" + } + } + ] + } + - name: Install JQ + run: | + mkdir -p $HOME/.local/bin + curl -sLo $HOME/.local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 && chmod +x $HOME/.local/bin/jq + echo "$HOME/.local/bin" >> $GITHUB_PATH + - id: pandora + name: Open Pandora's Box + run: | + pandoras-box -url ${{ secrets.PANDORAS_TARGET }} -m "${{ secrets.PANDORAS_MNEMONIC }}" -b ${{ github.event.inputs.transaction_batch }} -t ${{ github.event.inputs.transaction_count }} -o pandorasConsequences.json + echo "::set-output name=tps::$(cat pandorasConsequences.json | jq -r '.averageTPS')" + - name: Archive Pandora's Consequences + continue-on-error: true + uses: actions/upload-artifact@v3 + with: + name: pandoras-consequences-report + path: pandorasConsequences.json + - name: Notify Slack + uses: slackapi/slack-github-action@v1.22.0 + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_PERFORMANCE_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + with: + payload: | + { + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "Pandora's Box Results" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Approximate TPS: `${{ steps.pandora.outputs.tps }}`" + } + }, + { + "type": "divider" + }, + { + "type": "context", + "elements": [ + { + "type": "mrkdwn", + "text": "Environment: `${{ github.event.inputs.environment }}`" + }, + { + "type": "mrkdwn", + "text": "JSON-RPC Endpoint: ${{ secrets.PANDORAS_TARGET }}" + }, + { + "type": "mrkdwn", + "text": "Workflow: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Results>" + } + ] + } + ] + } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 33bc2663f6..33d4c09942 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,6 @@ +--- name: Release - -on: +on: # yamllint disable-line rule:truthy push: branches-ignore: - '**' diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 0000000000..be9dd0fa21 --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,26 @@ +--- +name: Security Scan +on: # yamllint disable-line rule:truthy + workflow_call: {} + workflow_dispatch: {} + schedule: + - cron: '0 0 * * 0' + +jobs: + gosec: + name: Gosec and Publish + runs-on: ubuntu-latest + continue-on-error: true + env: + GO111MODULE: on # yamllint disable-line rule:truthy + steps: + - name: Checkout Source + uses: actions/checkout@v2 + - name: Run Gosec Security Scanner + uses: securego/gosec@master + with: + args: '-no-fail -fmt sarif -out results.sarif ./...' + - name: Upload SARIF file + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: results.sarif diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000000..ec98f03bdf --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,36 @@ +--- +name: Test +on: # yamllint disable-line rule:truthy + workflow_dispatch: + workflow_call: + +jobs: + go_test: + name: Polygon Edge + runs-on: ubuntu-latest + steps: + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: 1.17.x + + - name: Checkout Code + uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Install Polygon Edge + run: go install -race -a . + env: + GO111MODULE: on # yamllint disable-line rule:truthy + CGO_ENABLED: '1' + GOARC: amd64 + GOOS: linux + + - name: Run Go Test + run: go test -coverprofile coverage.out -timeout 28m ./... + + - name: Upload coverage file to Codecov + uses: codecov/codecov-action@v3 + with: + files: coverage.out From d3f9c33ccaaac75a013cbe3809c55518b1a61455 Mon Sep 17 00:00:00 2001 From: epikichi Date: Tue, 27 Sep 2022 12:30:34 -0400 Subject: [PATCH 2/6] updating performance test workflows to have higher Tx amounts. Removing legacy deployer ref --- .github/workflows/ci.yml | 16 ---------------- .github/workflows/deploy.testnet.yml | 6 +++--- .github/workflows/loadbot.yml | 2 +- .github/workflows/pandoras_box.yml | 4 ++-- 4 files changed, 6 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e76c1f1c26..d26aa34955 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,19 +30,3 @@ jobs: uses: securego/gosec@master with: args: ./... - - notify-deployer: - needs: test - runs-on: ubuntu-latest - if: github.event_name != 'pull_request' - steps: - - name: Extract branch name - shell: bash - run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" - id: extract_branch - - name: Notify Deployer - env: - ARTIFACT_NAME: polygon-edge_${{ github.sha }}_${{ steps.extract_branch.outputs.branch }} - COMMITER: ${{ github.actor }} - run: | - curl -X POST ${{ secrets.DEPLOYER_API }} -H 'Accept: application/vnd.github.everest-preview+json' -u ${{ secrets.DEPLOYER_ACCESS_TOKEN }} --data '{"event_type": "new_artefact", "client_payload": { "commiter":"'"$COMMITER"'","commit":"'"$GITHUB_SHA"'","branch":"'"$GITHUB_REF"'","repository": "'"$GITHUB_REPOSITORY"'","artefact_name":"'"$ARTIFACT_NAME"'" }}' diff --git a/.github/workflows/deploy.testnet.yml b/.github/workflows/deploy.testnet.yml index c5fe9d3530..aefdf7242f 100644 --- a/.github/workflows/deploy.testnet.yml +++ b/.github/workflows/deploy.testnet.yml @@ -164,8 +164,8 @@ jobs: with: runner: testnet environment: testnet - transaction_batch: '20' - transaction_count: '2000' + transaction_batch: '200' + transaction_count: '10000' loadbot: uses: 0xpolygon/polygon-edge/.github/workflows/loadbot.yml@develop @@ -182,5 +182,5 @@ jobs: environment: testnet chain_id: '750' transaction_amount: '100' - transaction_count: '2000' + transaction_count: '10000' transactions_per_second: '100' diff --git a/.github/workflows/loadbot.yml b/.github/workflows/loadbot.yml index 0581e64534..d2b8416ece 100644 --- a/.github/workflows/loadbot.yml +++ b/.github/workflows/loadbot.yml @@ -21,7 +21,7 @@ on: # yamllint disable-line rule:truthy description: The chain ID of the network to test type: string transaction_count: - default: '1000' + default: '10000' description: The number of transactions to send type: string transactions_per_second: diff --git a/.github/workflows/pandoras_box.yml b/.github/workflows/pandoras_box.yml index ecb38f1a95..157114200c 100644 --- a/.github/workflows/pandoras_box.yml +++ b/.github/workflows/pandoras_box.yml @@ -16,11 +16,11 @@ on: # yamllint disable-line rule:truthy required: false type: environment transaction_count: - default: '2000' + default: '10000' description: The number of transactions to send type: string transaction_batch: - default: '20' + default: '100' description: The transaction batch size type: string workflow_call: From ddb87de7a7c61ca04926d96a494387f52021f971 Mon Sep 17 00:00:00 2001 From: epikichi Date: Tue, 4 Oct 2022 15:20:22 -0400 Subject: [PATCH 3/6] fileformat consistency --- .github/workflows/deploy.devnet.yml | 328 +++++++++++------------ .github/workflows/deploy.testnet.yml | 372 +++++++++++++-------------- .github/workflows/loadbot.yml | 308 +++++++++++----------- .github/workflows/pandoras_box.yml | 314 +++++++++++----------- 4 files changed, 661 insertions(+), 661 deletions(-) diff --git a/.github/workflows/deploy.devnet.yml b/.github/workflows/deploy.devnet.yml index da8172b6fb..47d07434ef 100644 --- a/.github/workflows/deploy.devnet.yml +++ b/.github/workflows/deploy.devnet.yml @@ -1,164 +1,164 @@ ---- -name: DevNet Workflow - -on: # yamllint disable-line rule:truthy - workflow_dispatch: - push: - branches: - - develop - -permissions: - id-token: write - contents: read - security-events: write - -jobs: - gosec: - name: Gosec and Publish - uses: 0xpolygon/polygon-edge/.github/workflows/security.yml@develop - - build: - name: Build - uses: 0xpolygon/polygon-edge/.github/workflows/build.yml@develop - - test: - name: Test - uses: 0xpolygon/polygon-edge/.github/workflows/test.yml@develop - needs: build - - deploy_devnet: - name: Deploy DevNet - needs: build - concurrency: deploy_devnet - environment: - name: devnet - url: https://rpc.us-east-1.testing.psdk.io/ - runs-on: ubuntu-latest - steps: - - name: Download Polygon Edge Artifact - uses: actions/download-artifact@v3 - with: - name: polygon-edge - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - aws-region: ${{ secrets.AWS_REGION }} - - - name: Notify Slack - uses: slackapi/slack-github-action@v1.22.0 - if: false - continue-on-error: true - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEPLOYMENT_WEBHOOK_URL }} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK - with: - payload: | - { - "blocks": [ - { - "type": "header", - "text": { - "type": "plain_text", - "text": "Devnet Deployment - Started" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "Devnet Deployment Status: ${{ job.status }}\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Job Status>" - } - } - ] - } - - - name: Publish Binary package to S3 - run: | - aws s3 cp ./polygon-edge.tar.gz s3://${{ secrets.POLYGON_EDGE_ARTIFACT_BUCKET }} --metadata "{\"commit_sha\":\"${GITHUB_SHA}\"}" - - - name: Deploy DevNet - continue-on-error: true - env: - VALIDATOR_ASGS: ${{ secrets.VALIDATOR_ASGS }} - run: | - # Parallel deploy all nodes, relying on ALB Health Checks - echo "Deploying new Devnet" - echo "--------------------" - - # Should finish relatively fast. Can add a check to guard against pre-mature cycle - echo "Cleaning up the data directories and stopping Polygon-Edge Devnet service..." - aws ssm send-command --document-name polygon-edge-validators-maintenance-clean-data --targets Key=tag:aws:autoscaling:groupName,Values=$VALIDATOR_ASGS >> /dev/null - - for vasg in ${VALIDATOR_ASGS//,/ } - do - instance_id=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-names $vasg --query 'AutoScalingGroups[0].Instances[0].InstanceId' --output text) - echo "Draining $instance_id in $vasg" - irid=$(aws autoscaling start-instance-refresh --auto-scaling-group-name $vasg --query InstanceRefreshId --output text) - done - - echo "Waiting for instance to clear health check..." - pending=true - - until [ $pending != true ] - do - pending=false - sleep 10 - for vasg in ${VALIDATOR_ASGS//,/ } - do - refresh_status=$(aws autoscaling describe-instance-refreshes --auto-scaling-group-name $vasg --query 'InstanceRefreshes[0].Status' --output text) - echo "ASG: $vasg" - echo "Status: $refresh_status" - - if [ "$refresh_status" == "Successful" ] - then - VALIDATOR_ASGS=${VALIDATOR_ASGS//$vasg/} - instance_id=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-names $vasg --query 'AutoScalingGroups[0].Instances[0].InstanceId' --output text) - - echo "-------------------------------------------------" - echo "New validator: $instance_id successfully deployed" - echo "-------------------------------------------------" - fi - pending=true - done - echo "" - done - echo "Devnet Deployment Complete" - - - name: Notify Slack - Failures - uses: slackapi/slack-github-action@v1.22.0 - if: ${{ job.status != 'success' }} - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEPLOYMENT_WEBHOOK_URL }} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK - with: - payload: | - { - "blocks": [ - { - "type": "header", - "text": { - "type": "plain_text", - "text": "Devnet Deployment - Failed" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Job Status>" - } - }, - { - "type": "divider" - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "Commit SHA: <${{ github.server_url }}/${{ github.repository }}/tree/${{ github.sha }}|${{ github.sha }}>\nBranch: <${{ github.server_url }}/${{ github.repository }}/tree/${{ github.ref_name }}|${{ github.ref_name}}>" - } - } - ] - } +--- +name: DevNet Workflow + +on: # yamllint disable-line rule:truthy + workflow_dispatch: + push: + branches: + - develop + +permissions: + id-token: write + contents: read + security-events: write + +jobs: + gosec: + name: Gosec and Publish + uses: 0xpolygon/polygon-edge/.github/workflows/security.yml@develop + + build: + name: Build + uses: 0xpolygon/polygon-edge/.github/workflows/build.yml@develop + + test: + name: Test + uses: 0xpolygon/polygon-edge/.github/workflows/test.yml@develop + needs: build + + deploy_devnet: + name: Deploy DevNet + needs: build + concurrency: deploy_devnet + environment: + name: devnet + url: https://rpc.us-east-1.testing.psdk.io/ + runs-on: ubuntu-latest + steps: + - name: Download Polygon Edge Artifact + uses: actions/download-artifact@v3 + with: + name: polygon-edge + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Notify Slack + uses: slackapi/slack-github-action@v1.22.0 + if: false + continue-on-error: true + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEPLOYMENT_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + with: + payload: | + { + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "Devnet Deployment - Started" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Devnet Deployment Status: ${{ job.status }}\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Job Status>" + } + } + ] + } + + - name: Publish Binary package to S3 + run: | + aws s3 cp ./polygon-edge.tar.gz s3://${{ secrets.POLYGON_EDGE_ARTIFACT_BUCKET }} --metadata "{\"commit_sha\":\"${GITHUB_SHA}\"}" + + - name: Deploy DevNet + continue-on-error: true + env: + VALIDATOR_ASGS: ${{ secrets.VALIDATOR_ASGS }} + run: | + # Parallel deploy all nodes, relying on ALB Health Checks + echo "Deploying new Devnet" + echo "--------------------" + + # Should finish relatively fast. Can add a check to guard against pre-mature cycle + echo "Cleaning up the data directories and stopping Polygon-Edge Devnet service..." + aws ssm send-command --document-name polygon-edge-validators-maintenance-clean-data --targets Key=tag:aws:autoscaling:groupName,Values=$VALIDATOR_ASGS >> /dev/null + + for vasg in ${VALIDATOR_ASGS//,/ } + do + instance_id=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-names $vasg --query 'AutoScalingGroups[0].Instances[0].InstanceId' --output text) + echo "Draining $instance_id in $vasg" + irid=$(aws autoscaling start-instance-refresh --auto-scaling-group-name $vasg --query InstanceRefreshId --output text) + done + + echo "Waiting for instance to clear health check..." + pending=true + + until [ $pending != true ] + do + pending=false + sleep 10 + for vasg in ${VALIDATOR_ASGS//,/ } + do + refresh_status=$(aws autoscaling describe-instance-refreshes --auto-scaling-group-name $vasg --query 'InstanceRefreshes[0].Status' --output text) + echo "ASG: $vasg" + echo "Status: $refresh_status" + + if [ "$refresh_status" == "Successful" ] + then + VALIDATOR_ASGS=${VALIDATOR_ASGS//$vasg/} + instance_id=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-names $vasg --query 'AutoScalingGroups[0].Instances[0].InstanceId' --output text) + + echo "-------------------------------------------------" + echo "New validator: $instance_id successfully deployed" + echo "-------------------------------------------------" + fi + pending=true + done + echo "" + done + echo "Devnet Deployment Complete" + + - name: Notify Slack - Failures + uses: slackapi/slack-github-action@v1.22.0 + if: ${{ job.status != 'success' }} + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEPLOYMENT_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + with: + payload: | + { + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "Devnet Deployment - Failed" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Job Status>" + } + }, + { + "type": "divider" + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Commit SHA: <${{ github.server_url }}/${{ github.repository }}/tree/${{ github.sha }}|${{ github.sha }}>\nBranch: <${{ github.server_url }}/${{ github.repository }}/tree/${{ github.ref_name }}|${{ github.ref_name}}>" + } + } + ] + } diff --git a/.github/workflows/deploy.testnet.yml b/.github/workflows/deploy.testnet.yml index aefdf7242f..b26bf6a32e 100644 --- a/.github/workflows/deploy.testnet.yml +++ b/.github/workflows/deploy.testnet.yml @@ -1,186 +1,186 @@ ---- -name: TestNet Workflow -on: # yamllint disable-line rule:truthy - workflow_dispatch: - push: - branches: - - 'release/**' - -permissions: - id-token: write - contents: read - security-events: write - -jobs: - gosec: - name: Gosec and Publish - uses: 0xpolygon/polygon-edge/.github/workflows/security.yml@develop - - build: - uses: 0xpolygon/polygon-edge/.github/workflows/build.yml@develop - name: Build - - test: - uses: 0xpolygon/polygon-edge/.github/workflows/test.yml@develop - name: Test - needs: build - - deploy_testnet: - name: Update TestNet - needs: build - concurrency: deploy_testnet - environment: - name: testnet - url: https://rpc.us-east-1.tn.testing.psdk.io/ - runs-on: ubuntu-latest - steps: - - name: Download Polygon Edge Artifact - uses: actions/download-artifact@v3 - with: - name: polygon-edge - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - aws-region: ${{ secrets.AWS_REGION }} - - - name: Notify Slack - uses: slackapi/slack-github-action@v1.22.0 - continue-on-error: true - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEPLOYMENT_WEBHOOK_URL }} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK - with: - payload: | - { - "blocks": [ - { - "type": "header", - "text": { - "type": "plain_text", - "text": "Testnet Deployment - Started" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "Testnet Deployment Status: ${{ job.status }}\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Job Status>" - } - } - ] - } - - - name: Publish Binary package to S3 - run: | - aws s3 cp ./polygon-edge.tar.gz s3://${{ secrets.POLYGON_EDGE_ARTIFACT_BUCKET }} --metadata "{\"commit_sha\":\"${GITHUB_SHA}\"}" - - name: Deploy to Testnet - continue-on-error: true - env: - VALIDATOR_ASGS: ${{ secrets.VALIDATOR_ASGS }} - run: | - echo "Deploying new TestNet..." - for vasg in ${VALIDATOR_ASGS//,/ } - do - instance_id=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-names $vasg --query 'AutoScalingGroups[0].Instances[0].InstanceId' --output text) - - echo "Draining $instance_id in $vasg" - irid=$(aws autoscaling start-instance-refresh --auto-scaling-group-name $vasg --query InstanceRefreshId --output text) - - echo "Waiting for instance to clear health check..." - refresh_status=Pending - until [ $refresh_status != "InProgress" ] && [ $refresh_status != "Pending" ] - do - sleep 10 - refresh_status=$(aws autoscaling describe-instance-refreshes --instance-refresh-ids $irid --auto-scaling-group-name $vasg --query 'InstanceRefreshes[0].Status' --output text) - - echo "ASG: $vasg" - echo "Status: $refresh_status" - done - - if [ "$refresh_status" != "Successful" ] - then - echo "instance refresh failed." - echo "autoscaling group: $vasg" - echo "instance refresh id: $irid" - exit 1 - fi - - instance_id=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-names $vasg --query 'AutoScalingGroups[0].Instances[0].InstanceId' --output text) - echo "-------------------------------------------------" - echo "New validator: $instance_id successfully deployed" - echo "-------------------------------------------------" - - done - echo "Testnet Deployment Complete" - - - name: Notify Slack - uses: slackapi/slack-github-action@v1.22.0 - continue-on-error: true - if: ${{ job.status != 'success' }} - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEPLOYMENT_WEBHOOK_URL }} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK - with: - payload: | - { - "blocks": [ - { - "type": "header", - "text": { - "type": "plain_text", - "text": "Testnet Deployment - Failed" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Job Status>" - } - }, - { - "type": "divider" - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "Commit SHA: <${{ github.server_url }}/${{ github.repository }}/tree/${{ github.sha }}|${{ github.sha }}>\nPre-release: <${{ github.server_url }}/${{ github.repository }}/tree/${{ github.ref_name }}|${{ github.ref_name}}>" - } - } - ] - } - - pandoras_box: - uses: 0xpolygon/polygon-edge/.github/workflows/pandoras_box.yml@develop - name: Pandora's Box - needs: deploy_testnet - secrets: - SLACK_PERFORMANCE_WEBHOOK_URL: ${{ secrets.SLACK_PERFORMANCE_WEBHOOK_URL }} - PANDORAS_TARGET: ${{ secrets.PANDORAS_TARGET }} - PANDORAS_MNEMONIC: ${{ secrets.PANDORAS_MNEMONIC }} - with: - runner: testnet - environment: testnet - transaction_batch: '200' - transaction_count: '10000' - - loadbot: - uses: 0xpolygon/polygon-edge/.github/workflows/loadbot.yml@develop - name: Loadbot - needs: deploy_testnet - secrets: - SLACK_PERFORMANCE_WEBHOOK_URL: ${{ secrets.SLACK_PERFORMANCE_WEBHOOK_URL }} - LOADBOT_JSONRPC_URL: ${{ secrets.LOADBOT_JSONRPC_URL }} - LOADBOT_SENDER_ADDRESS: ${{ secrets.LOADBOT_SENDER_ADDRESS }} - LOADBOT_GRPC_ADDRESS: ${{ secrets.LOADBOT_GRPC_ADDRESS }} - LOADBOT_0x1AB8C3df809b85012a009c0264eb92dB04eD6EFa: ${{ secrets.LOADBOT_0x1AB8C3df809b85012a009c0264eb92dB04eD6EFa }} - with: - runner: testnet - environment: testnet - chain_id: '750' - transaction_amount: '100' - transaction_count: '10000' - transactions_per_second: '100' +--- +name: TestNet Workflow +on: # yamllint disable-line rule:truthy + workflow_dispatch: + push: + branches: + - 'release/**' + +permissions: + id-token: write + contents: read + security-events: write + +jobs: + gosec: + name: Gosec and Publish + uses: 0xpolygon/polygon-edge/.github/workflows/security.yml@develop + + build: + uses: 0xpolygon/polygon-edge/.github/workflows/build.yml@develop + name: Build + + test: + uses: 0xpolygon/polygon-edge/.github/workflows/test.yml@develop + name: Test + needs: build + + deploy_testnet: + name: Update TestNet + needs: build + concurrency: deploy_testnet + environment: + name: testnet + url: https://rpc.us-east-1.tn.testing.psdk.io/ + runs-on: ubuntu-latest + steps: + - name: Download Polygon Edge Artifact + uses: actions/download-artifact@v3 + with: + name: polygon-edge + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Notify Slack + uses: slackapi/slack-github-action@v1.22.0 + continue-on-error: true + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEPLOYMENT_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + with: + payload: | + { + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "Testnet Deployment - Started" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Testnet Deployment Status: ${{ job.status }}\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Job Status>" + } + } + ] + } + + - name: Publish Binary package to S3 + run: | + aws s3 cp ./polygon-edge.tar.gz s3://${{ secrets.POLYGON_EDGE_ARTIFACT_BUCKET }} --metadata "{\"commit_sha\":\"${GITHUB_SHA}\"}" + - name: Deploy to Testnet + continue-on-error: true + env: + VALIDATOR_ASGS: ${{ secrets.VALIDATOR_ASGS }} + run: | + echo "Deploying new TestNet..." + for vasg in ${VALIDATOR_ASGS//,/ } + do + instance_id=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-names $vasg --query 'AutoScalingGroups[0].Instances[0].InstanceId' --output text) + + echo "Draining $instance_id in $vasg" + irid=$(aws autoscaling start-instance-refresh --auto-scaling-group-name $vasg --query InstanceRefreshId --output text) + + echo "Waiting for instance to clear health check..." + refresh_status=Pending + until [ $refresh_status != "InProgress" ] && [ $refresh_status != "Pending" ] + do + sleep 10 + refresh_status=$(aws autoscaling describe-instance-refreshes --instance-refresh-ids $irid --auto-scaling-group-name $vasg --query 'InstanceRefreshes[0].Status' --output text) + + echo "ASG: $vasg" + echo "Status: $refresh_status" + done + + if [ "$refresh_status" != "Successful" ] + then + echo "instance refresh failed." + echo "autoscaling group: $vasg" + echo "instance refresh id: $irid" + exit 1 + fi + + instance_id=$(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-names $vasg --query 'AutoScalingGroups[0].Instances[0].InstanceId' --output text) + echo "-------------------------------------------------" + echo "New validator: $instance_id successfully deployed" + echo "-------------------------------------------------" + + done + echo "Testnet Deployment Complete" + + - name: Notify Slack + uses: slackapi/slack-github-action@v1.22.0 + continue-on-error: true + if: ${{ job.status != 'success' }} + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEPLOYMENT_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + with: + payload: | + { + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "Testnet Deployment - Failed" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Job Status>" + } + }, + { + "type": "divider" + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Commit SHA: <${{ github.server_url }}/${{ github.repository }}/tree/${{ github.sha }}|${{ github.sha }}>\nPre-release: <${{ github.server_url }}/${{ github.repository }}/tree/${{ github.ref_name }}|${{ github.ref_name}}>" + } + } + ] + } + + pandoras_box: + uses: 0xpolygon/polygon-edge/.github/workflows/pandoras_box.yml@develop + name: Pandora's Box + needs: deploy_testnet + secrets: + SLACK_PERFORMANCE_WEBHOOK_URL: ${{ secrets.SLACK_PERFORMANCE_WEBHOOK_URL }} + PANDORAS_TARGET: ${{ secrets.PANDORAS_TARGET }} + PANDORAS_MNEMONIC: ${{ secrets.PANDORAS_MNEMONIC }} + with: + runner: testnet + environment: testnet + transaction_batch: '200' + transaction_count: '10000' + + loadbot: + uses: 0xpolygon/polygon-edge/.github/workflows/loadbot.yml@develop + name: Loadbot + needs: deploy_testnet + secrets: + SLACK_PERFORMANCE_WEBHOOK_URL: ${{ secrets.SLACK_PERFORMANCE_WEBHOOK_URL }} + LOADBOT_JSONRPC_URL: ${{ secrets.LOADBOT_JSONRPC_URL }} + LOADBOT_SENDER_ADDRESS: ${{ secrets.LOADBOT_SENDER_ADDRESS }} + LOADBOT_GRPC_ADDRESS: ${{ secrets.LOADBOT_GRPC_ADDRESS }} + LOADBOT_0x1AB8C3df809b85012a009c0264eb92dB04eD6EFa: ${{ secrets.LOADBOT_0x1AB8C3df809b85012a009c0264eb92dB04eD6EFa }} + with: + runner: testnet + environment: testnet + chain_id: '750' + transaction_amount: '100' + transaction_count: '10000' + transactions_per_second: '100' diff --git a/.github/workflows/loadbot.yml b/.github/workflows/loadbot.yml index d2b8416ece..49dbcd2af5 100644 --- a/.github/workflows/loadbot.yml +++ b/.github/workflows/loadbot.yml @@ -1,154 +1,154 @@ ---- - -name: Loadbot -on: # yamllint disable-line rule:truthy - workflow_dispatch: - inputs: - runner: - description: 'The runner to execute on' - default: 'ubuntu-latest' - type: choice - options: - - ubuntu-latest - - devnet - - testnet - environment: - description: 'The environment to run against' - required: false - type: environment - chain_id: - default: '750' - description: The chain ID of the network to test - type: string - transaction_count: - default: '10000' - description: The number of transactions to send - type: string - transactions_per_second: - default: '100' - description: The amount of transactions per second to send - type: string - transaction_value: - default: '100' - description: The value per transactions to send - type: string - workflow_call: - inputs: - chain_id: - description: The chain ID of the network to test - type: string - transaction_count: - description: The number of transactions to send - type: string - transactions_per_second: - description: The amount of transactions per second to send - type: string - transaction_amount: - description: The amount per transactions to send - type: string - runner: - required: true - type: string - description: The runner label to use - environment: - description: 'The environment to run against' - type: string - secrets: - SLACK_PERFORMANCE_WEBHOOK_URL: - required: true - LOADBOT_SENDER_ADDRESS: - required: true - LOADBOT_GRPC_ADDRESS: - required: true - LOADBOT_JSONRPC_URL: - required: true - LOADBOT_0x1AB8C3df809b85012a009c0264eb92dB04eD6EFa: - required: true -jobs: - run_loadbot: - name: Run Loadbot - concurrency: ${{ github.event.inputs.environment }}_performance - environment: ${{ github.event.inputs.environment }} - runs-on: ${{ github.event.inputs.runner }} - steps: - - name: Checkout code - uses: actions/checkout@v3 - - name: Setup Go environment - uses: actions/setup-go@v3.3.0 - with: - go-version: 1.17.x - - name: Install JQ - run: | - mkdir -p $HOME/.local/bin - curl -sLo $HOME/.local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 && chmod +x $HOME/.local/bin/jq - echo "$HOME/.local/bin" >> $GITHUB_PATH - - id: loadbot - name: Run Loadbot - continue-on-error: true - run: | - go run . loadbot --jsonrpc ${{ secrets.LOADBOT_JSONRPC_URL }} --grpc-address ${{ secrets.LOADBOT_GRPC_ADDRESS }} --sender ${{ secrets.LOADBOT_SENDER_ADDRESS }} --count ${{ github.event.inputs.transaction_count}} --value ${{ github.event.inputs.transaction_value }} --tps ${{ github.event.inputs.transactions_per_second }} --chain-id ${{ github.event.inputs.chain_id }} --detailed --json > loadbotOutput.json - cat loadbotOutput.json - echo "::set-output name=tps::$(cat loadbotOutput.json | jq -r '.approx_tps')" - echo "::set-output name=totalTx::$(cat loadbotOutput.json | jq -r '.count_data.total')" - echo "::set-output name=failedTx::$(cat loadbotOutput.json | jq -r '.count_data.failed')" - env: - LOADBOT_0x1AB8C3df809b85012a009c0264eb92dB04eD6EFa: ${{ secrets.LOADBOT_0x1AB8C3df809b85012a009c0264eb92dB04eD6EFa }} - - name: Notify Slack - uses: slackapi/slack-github-action@v1.22.0 - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_PERFORMANCE_WEBHOOK_URL }} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK - with: - payload: | - { - "blocks": [ - { - "type": "header", - "text": { - "type": "plain_text", - "text": "Loadbot Results" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "Approximate TPS: `${{ steps.loadbot.outputs.tps }}`" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "Total Transactions: `${{ steps.loadbot.outputs.totalTx }}`" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "Failed Transactions: `${{ steps.loadbot.outputs.failedTx }}`" - } - }, - { - "type": "divider" - }, - { - "type": "context", - "elements": [ - { - "type": "mrkdwn", - "text": "Environment: `${{ github.event.inputs.environment }}`" - }, - { - "type": "mrkdwn", - "text": "JSON-RPC Endpoint: ${{ secrets.LOADBOT_JSONRPC_URL }}" - }, - { - "type": "mrkdwn", - "text": "Workflow: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Results>" - } - ] - } - ] - } +--- + +name: Loadbot +on: # yamllint disable-line rule:truthy + workflow_dispatch: + inputs: + runner: + description: 'The runner to execute on' + default: 'ubuntu-latest' + type: choice + options: + - ubuntu-latest + - devnet + - testnet + environment: + description: 'The environment to run against' + required: false + type: environment + chain_id: + default: '750' + description: The chain ID of the network to test + type: string + transaction_count: + default: '10000' + description: The number of transactions to send + type: string + transactions_per_second: + default: '100' + description: The amount of transactions per second to send + type: string + transaction_value: + default: '100' + description: The value per transactions to send + type: string + workflow_call: + inputs: + chain_id: + description: The chain ID of the network to test + type: string + transaction_count: + description: The number of transactions to send + type: string + transactions_per_second: + description: The amount of transactions per second to send + type: string + transaction_amount: + description: The amount per transactions to send + type: string + runner: + required: true + type: string + description: The runner label to use + environment: + description: 'The environment to run against' + type: string + secrets: + SLACK_PERFORMANCE_WEBHOOK_URL: + required: true + LOADBOT_SENDER_ADDRESS: + required: true + LOADBOT_GRPC_ADDRESS: + required: true + LOADBOT_JSONRPC_URL: + required: true + LOADBOT_0x1AB8C3df809b85012a009c0264eb92dB04eD6EFa: + required: true +jobs: + run_loadbot: + name: Run Loadbot + concurrency: ${{ github.event.inputs.environment }}_performance + environment: ${{ github.event.inputs.environment }} + runs-on: ${{ github.event.inputs.runner }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Setup Go environment + uses: actions/setup-go@v3.3.0 + with: + go-version: 1.17.x + - name: Install JQ + run: | + mkdir -p $HOME/.local/bin + curl -sLo $HOME/.local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 && chmod +x $HOME/.local/bin/jq + echo "$HOME/.local/bin" >> $GITHUB_PATH + - id: loadbot + name: Run Loadbot + continue-on-error: true + run: | + go run . loadbot --jsonrpc ${{ secrets.LOADBOT_JSONRPC_URL }} --grpc-address ${{ secrets.LOADBOT_GRPC_ADDRESS }} --sender ${{ secrets.LOADBOT_SENDER_ADDRESS }} --count ${{ github.event.inputs.transaction_count}} --value ${{ github.event.inputs.transaction_value }} --tps ${{ github.event.inputs.transactions_per_second }} --chain-id ${{ github.event.inputs.chain_id }} --detailed --json > loadbotOutput.json + cat loadbotOutput.json + echo "::set-output name=tps::$(cat loadbotOutput.json | jq -r '.approx_tps')" + echo "::set-output name=totalTx::$(cat loadbotOutput.json | jq -r '.count_data.total')" + echo "::set-output name=failedTx::$(cat loadbotOutput.json | jq -r '.count_data.failed')" + env: + LOADBOT_0x1AB8C3df809b85012a009c0264eb92dB04eD6EFa: ${{ secrets.LOADBOT_0x1AB8C3df809b85012a009c0264eb92dB04eD6EFa }} + - name: Notify Slack + uses: slackapi/slack-github-action@v1.22.0 + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_PERFORMANCE_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + with: + payload: | + { + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "Loadbot Results" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Approximate TPS: `${{ steps.loadbot.outputs.tps }}`" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Total Transactions: `${{ steps.loadbot.outputs.totalTx }}`" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Failed Transactions: `${{ steps.loadbot.outputs.failedTx }}`" + } + }, + { + "type": "divider" + }, + { + "type": "context", + "elements": [ + { + "type": "mrkdwn", + "text": "Environment: `${{ github.event.inputs.environment }}`" + }, + { + "type": "mrkdwn", + "text": "JSON-RPC Endpoint: ${{ secrets.LOADBOT_JSONRPC_URL }}" + }, + { + "type": "mrkdwn", + "text": "Workflow: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Results>" + } + ] + } + ] + } diff --git a/.github/workflows/pandoras_box.yml b/.github/workflows/pandoras_box.yml index 157114200c..91ffab620b 100644 --- a/.github/workflows/pandoras_box.yml +++ b/.github/workflows/pandoras_box.yml @@ -1,157 +1,157 @@ ---- -name: Pandora's Box -on: # yamllint disable-line rule:truthy - workflow_dispatch: - inputs: - runner: - description: 'The runner to execute on' - default: 'ubuntu-latest' - type: choice - options: - - ubuntu-latest - - devnet - - testnet - environment: - description: 'The environment to run against' - required: false - type: environment - transaction_count: - default: '10000' - description: The number of transactions to send - type: string - transaction_batch: - default: '100' - description: The transaction batch size - type: string - workflow_call: - inputs: - transaction_count: - required: true - description: The number of transactions to send - type: string - transaction_batch: - required: true - description: The transaction batch size - type: string - environment: - description: 'The environment to run against' - type: string - runner: - required: true - type: string - description: The runner label to use - secrets: - SLACK_PERFORMANCE_WEBHOOK_URL: - required: true - PANDORAS_TARGET: - required: true - PANDORAS_MNEMONIC: - required: true - -jobs: - open_pandoras_box: - name: Open Pandora's Box - concurrency: ${{ github.event.inputs.environment }}_performance - environment: ${{ github.event.inputs.environment }} - runs-on: ${{ github.event.inputs.runner }} - steps: - - name: Checkout Pandora's Box Repository - uses: actions/checkout@v3 - with: - repository: trapesys/pandoras-box - - uses: actions/setup-node@v3 - with: - node-version: 16 - cache: 'npm' - - name: Prepare Pandora's Box - run: | - npm install -g yarn - yarn install && yarn build && yarn link - echo "$HOME/.yarn/bin" >> $GITHUB_PATH - - name: Notify Slack - if: false - uses: slackapi/slack-github-action@v1.22.0 - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_PERFORMANCE_WEBHOOK_URL }} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK - with: - payload: | - { - "blocks": [ - { - "type": "header", - "text": { - "type": "plain_text", - "text": "GitHub Action - Pandora's Box - Starting" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Status>" - } - } - ] - } - - name: Install JQ - run: | - mkdir -p $HOME/.local/bin - curl -sLo $HOME/.local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 && chmod +x $HOME/.local/bin/jq - echo "$HOME/.local/bin" >> $GITHUB_PATH - - id: pandora - name: Open Pandora's Box - run: | - pandoras-box -url ${{ secrets.PANDORAS_TARGET }} -m "${{ secrets.PANDORAS_MNEMONIC }}" -b ${{ github.event.inputs.transaction_batch }} -t ${{ github.event.inputs.transaction_count }} -o pandorasConsequences.json - echo "::set-output name=tps::$(cat pandorasConsequences.json | jq -r '.averageTPS')" - - name: Archive Pandora's Consequences - continue-on-error: true - uses: actions/upload-artifact@v3 - with: - name: pandoras-consequences-report - path: pandorasConsequences.json - - name: Notify Slack - uses: slackapi/slack-github-action@v1.22.0 - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_PERFORMANCE_WEBHOOK_URL }} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK - with: - payload: | - { - "blocks": [ - { - "type": "header", - "text": { - "type": "plain_text", - "text": "Pandora's Box Results" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "Approximate TPS: `${{ steps.pandora.outputs.tps }}`" - } - }, - { - "type": "divider" - }, - { - "type": "context", - "elements": [ - { - "type": "mrkdwn", - "text": "Environment: `${{ github.event.inputs.environment }}`" - }, - { - "type": "mrkdwn", - "text": "JSON-RPC Endpoint: ${{ secrets.PANDORAS_TARGET }}" - }, - { - "type": "mrkdwn", - "text": "Workflow: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Results>" - } - ] - } - ] - } +--- +name: Pandora's Box +on: # yamllint disable-line rule:truthy + workflow_dispatch: + inputs: + runner: + description: 'The runner to execute on' + default: 'ubuntu-latest' + type: choice + options: + - ubuntu-latest + - devnet + - testnet + environment: + description: 'The environment to run against' + required: false + type: environment + transaction_count: + default: '10000' + description: The number of transactions to send + type: string + transaction_batch: + default: '100' + description: The transaction batch size + type: string + workflow_call: + inputs: + transaction_count: + required: true + description: The number of transactions to send + type: string + transaction_batch: + required: true + description: The transaction batch size + type: string + environment: + description: 'The environment to run against' + type: string + runner: + required: true + type: string + description: The runner label to use + secrets: + SLACK_PERFORMANCE_WEBHOOK_URL: + required: true + PANDORAS_TARGET: + required: true + PANDORAS_MNEMONIC: + required: true + +jobs: + open_pandoras_box: + name: Open Pandora's Box + concurrency: ${{ github.event.inputs.environment }}_performance + environment: ${{ github.event.inputs.environment }} + runs-on: ${{ github.event.inputs.runner }} + steps: + - name: Checkout Pandora's Box Repository + uses: actions/checkout@v3 + with: + repository: trapesys/pandoras-box + - uses: actions/setup-node@v3 + with: + node-version: 16 + cache: 'npm' + - name: Prepare Pandora's Box + run: | + npm install -g yarn + yarn install && yarn build && yarn link + echo "$HOME/.yarn/bin" >> $GITHUB_PATH + - name: Notify Slack + if: false + uses: slackapi/slack-github-action@v1.22.0 + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_PERFORMANCE_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + with: + payload: | + { + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "GitHub Action - Pandora's Box - Starting" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Status>" + } + } + ] + } + - name: Install JQ + run: | + mkdir -p $HOME/.local/bin + curl -sLo $HOME/.local/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 && chmod +x $HOME/.local/bin/jq + echo "$HOME/.local/bin" >> $GITHUB_PATH + - id: pandora + name: Open Pandora's Box + run: | + pandoras-box -url ${{ secrets.PANDORAS_TARGET }} -m "${{ secrets.PANDORAS_MNEMONIC }}" -b ${{ github.event.inputs.transaction_batch }} -t ${{ github.event.inputs.transaction_count }} -o pandorasConsequences.json + echo "::set-output name=tps::$(cat pandorasConsequences.json | jq -r '.averageTPS')" + - name: Archive Pandora's Consequences + continue-on-error: true + uses: actions/upload-artifact@v3 + with: + name: pandoras-consequences-report + path: pandorasConsequences.json + - name: Notify Slack + uses: slackapi/slack-github-action@v1.22.0 + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_PERFORMANCE_WEBHOOK_URL }} + SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK + with: + payload: | + { + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "Pandora's Box Results" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Approximate TPS: `${{ steps.pandora.outputs.tps }}`" + } + }, + { + "type": "divider" + }, + { + "type": "context", + "elements": [ + { + "type": "mrkdwn", + "text": "Environment: `${{ github.event.inputs.environment }}`" + }, + { + "type": "mrkdwn", + "text": "JSON-RPC Endpoint: ${{ secrets.PANDORAS_TARGET }}" + }, + { + "type": "mrkdwn", + "text": "Workflow: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Results>" + } + ] + } + ] + } From 30de509c23441317ff92cd7c7db9f290ee8949ea Mon Sep 17 00:00:00 2001 From: epikichi Date: Tue, 4 Oct 2022 16:04:56 -0400 Subject: [PATCH 4/6] removing unneded step in test workflow --- .github/workflows/test.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8f138a6705..7dd93a8016 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,14 +19,6 @@ jobs: with: submodules: recursive - - name: Install Polygon Edge - run: go install -race -a . - env: - GO111MODULE: on # yamllint disable-line rule:truthy - CGO_ENABLED: '1' - GOARC: amd64 - GOOS: linux - - name: Run Go Test run: go test -coverprofile coverage.out -timeout 20m `go list ./... | grep -v e2e` From be9a8e64d9c759217ba46ac4fce586000e2a55b2 Mon Sep 17 00:00:00 2001 From: epikichi Date: Thu, 6 Oct 2022 05:54:57 -0400 Subject: [PATCH 5/6] updating with 1.18 changes --- .github/workflows/build.yml | 2 +- .github/workflows/e2e.yaml | 3 ++- .github/workflows/lint.yml | 2 +- .github/workflows/loadbot.yml | 2 +- .github/workflows/test.yml | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 077e00affb..07151a4486 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,7 +14,7 @@ jobs: - name: Setup Go environment uses: actions/setup-go@v3.3.0 with: - go-version: 1.17.x + go-version: 1.18.x - name: Build Polygon Edge run: go build -ldflags="-s -w -linkmode external -extldflags "-static" -X \"github.com/0xPolygon/polygon-edge/versioning.Version=${GITHUB_REF_NAME}\" -X \"github.com/0xPolygon/polygon-edge/versioning.Commit=${GITHUB_SHA}\"" && tar -czvf polygon-edge.tar.gz polygon-edge diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 3b3b459b81..c17a7eaaca 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1,5 +1,6 @@ +--- name: E2E tests -on: +on: # yamllint disable-line rule:truthy push: branches: - main diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 8c2070edf4..9a1c351f25 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -19,7 +19,7 @@ jobs: - name: Install Go uses: actions/setup-go@v3 with: - go-version: 1.17.x + go-version: 1.18.x - name: Checkout code uses: actions/checkout@v3 diff --git a/.github/workflows/loadbot.yml b/.github/workflows/loadbot.yml index 49dbcd2af5..30ebabefcf 100644 --- a/.github/workflows/loadbot.yml +++ b/.github/workflows/loadbot.yml @@ -76,7 +76,7 @@ jobs: - name: Setup Go environment uses: actions/setup-go@v3.3.0 with: - go-version: 1.17.x + go-version: 1.18.x - name: Install JQ run: | mkdir -p $HOME/.local/bin diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7dd93a8016..7f00e43063 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: 1.17.x + go-version: 1.18.x - name: Checkout Code uses: actions/checkout@v3 From ecfa89c9991ba1dca5dc4f920226574319134c80 Mon Sep 17 00:00:00 2001 From: epikichi Date: Thu, 6 Oct 2022 17:20:40 -0400 Subject: [PATCH 6/6] using more appropriate inputs context. added missing variable --- .github/workflows/deploy.testnet.yml | 2 +- .github/workflows/loadbot.yml | 15 ++++++++------- .github/workflows/pandoras_box.yml | 11 ++++++----- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.github/workflows/deploy.testnet.yml b/.github/workflows/deploy.testnet.yml index b26bf6a32e..5ac046dc76 100644 --- a/.github/workflows/deploy.testnet.yml +++ b/.github/workflows/deploy.testnet.yml @@ -181,6 +181,6 @@ jobs: runner: testnet environment: testnet chain_id: '750' - transaction_amount: '100' + transaction_value: '100' transaction_count: '10000' transactions_per_second: '100' diff --git a/.github/workflows/loadbot.yml b/.github/workflows/loadbot.yml index 30ebabefcf..1941049c66 100644 --- a/.github/workflows/loadbot.yml +++ b/.github/workflows/loadbot.yml @@ -43,14 +43,15 @@ on: # yamllint disable-line rule:truthy transactions_per_second: description: The amount of transactions per second to send type: string - transaction_amount: - description: The amount per transactions to send + transaction_value: + description: The value per transactions to send type: string runner: required: true type: string description: The runner label to use environment: + required: true description: 'The environment to run against' type: string secrets: @@ -67,9 +68,9 @@ on: # yamllint disable-line rule:truthy jobs: run_loadbot: name: Run Loadbot - concurrency: ${{ github.event.inputs.environment }}_performance - environment: ${{ github.event.inputs.environment }} - runs-on: ${{ github.event.inputs.runner }} + concurrency: ${{ inputs.environment }}_performance + environment: ${{ inputs.environment }} + runs-on: ${{ inputs.runner }} steps: - name: Checkout code uses: actions/checkout@v3 @@ -86,7 +87,7 @@ jobs: name: Run Loadbot continue-on-error: true run: | - go run . loadbot --jsonrpc ${{ secrets.LOADBOT_JSONRPC_URL }} --grpc-address ${{ secrets.LOADBOT_GRPC_ADDRESS }} --sender ${{ secrets.LOADBOT_SENDER_ADDRESS }} --count ${{ github.event.inputs.transaction_count}} --value ${{ github.event.inputs.transaction_value }} --tps ${{ github.event.inputs.transactions_per_second }} --chain-id ${{ github.event.inputs.chain_id }} --detailed --json > loadbotOutput.json + go run . loadbot --jsonrpc ${{ secrets.LOADBOT_JSONRPC_URL }} --grpc-address ${{ secrets.LOADBOT_GRPC_ADDRESS }} --sender ${{ secrets.LOADBOT_SENDER_ADDRESS }} --count ${{ inputs.transaction_count}} --value ${{ inputs.transaction_value }} --tps ${{ inputs.transactions_per_second }} --chain-id ${{ inputs.chain_id }} --detailed --json > loadbotOutput.json cat loadbotOutput.json echo "::set-output name=tps::$(cat loadbotOutput.json | jq -r '.approx_tps')" echo "::set-output name=totalTx::$(cat loadbotOutput.json | jq -r '.count_data.total')" @@ -138,7 +139,7 @@ jobs: "elements": [ { "type": "mrkdwn", - "text": "Environment: `${{ github.event.inputs.environment }}`" + "text": "Environment: `${{ inputs.environment }}`" }, { "type": "mrkdwn", diff --git a/.github/workflows/pandoras_box.yml b/.github/workflows/pandoras_box.yml index 91ffab620b..2a91497981 100644 --- a/.github/workflows/pandoras_box.yml +++ b/.github/workflows/pandoras_box.yml @@ -36,6 +36,7 @@ on: # yamllint disable-line rule:truthy environment: description: 'The environment to run against' type: string + required: true runner: required: true type: string @@ -51,9 +52,9 @@ on: # yamllint disable-line rule:truthy jobs: open_pandoras_box: name: Open Pandora's Box - concurrency: ${{ github.event.inputs.environment }}_performance - environment: ${{ github.event.inputs.environment }} - runs-on: ${{ github.event.inputs.runner }} + concurrency: ${{ inputs.environment }}_performance + environment: ${{ inputs.environment }} + runs-on: ${{ inputs.runner }} steps: - name: Checkout Pandora's Box Repository uses: actions/checkout@v3 @@ -102,7 +103,7 @@ jobs: - id: pandora name: Open Pandora's Box run: | - pandoras-box -url ${{ secrets.PANDORAS_TARGET }} -m "${{ secrets.PANDORAS_MNEMONIC }}" -b ${{ github.event.inputs.transaction_batch }} -t ${{ github.event.inputs.transaction_count }} -o pandorasConsequences.json + pandoras-box -url ${{ secrets.PANDORAS_TARGET }} -m "${{ secrets.PANDORAS_MNEMONIC }}" -b ${{ inputs.transaction_batch }} -t ${{ inputs.transaction_count }} -o pandorasConsequences.json echo "::set-output name=tps::$(cat pandorasConsequences.json | jq -r '.averageTPS')" - name: Archive Pandora's Consequences continue-on-error: true @@ -141,7 +142,7 @@ jobs: "elements": [ { "type": "mrkdwn", - "text": "Environment: `${{ github.event.inputs.environment }}`" + "text": "Environment: `${{ inputs.environment }}`" }, { "type": "mrkdwn",