From b7c700f17d8dd5c96f062c8be4ea1e15f153c234 Mon Sep 17 00:00:00 2001 From: Anton Belodedenko <2033996+ab77@users.noreply.github.com> Date: Mon, 10 Jun 2024 14:23:03 -0700 Subject: [PATCH] test compose workflow change-type: minor --- .github/workflows/flowzone.yml | 5 - .github/workflows/tests.yml | 701 ++++++++++++++++++++++++--------- Makefile | 32 +- docker-compose.yml | 16 +- docs/getting-started.md | 2 +- 5 files changed, 562 insertions(+), 194 deletions(-) diff --git a/.github/workflows/flowzone.yml b/.github/workflows/flowzone.yml index b46d44a099..1f4aeb2c31 100644 --- a/.github/workflows/flowzone.yml +++ b/.github/workflows/flowzone.yml @@ -50,8 +50,3 @@ jobs: github.event_name == 'pull_request_target' )) && github.event.action != 'closed' secrets: inherit - with: - environment: balena-cloud.com - fleet: balena/open-balena - # https://dash.cloudflare.com/001b3ed2352612aaa068aca1b0022736/balena-devices.com/dns - dns_tld: balena-devices.com diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e6cad34937..37bfd8e893 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -3,68 +3,73 @@ name: openBalena tests on: workflow_call: - inputs: - environment: - description: "balenaCloud environment" - required: true - type: string - fleet: - description: "balenaCloud fleet" - required: true - type: string - dns_tld: - description: "domain name to use for issuing SSL certificates" - required: true - type: string # https://docs.github.com/en/actions/security-guides/automatic-token-authentication # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions permissions: - actions: read - checks: read contents: read - deployments: read id-token: write # AWS GitHub OIDC required: write - issues: read - discussions: read packages: read - pages: read - pull-requests: read - repository-projects: read - security-events: read - statuses: read env: # Stack ID # arn:aws:cloudformation:us-east-1:491725000532:stack/balena-tests-s3-certs/814dea60-404d-11ed-b06f-0a7d458f8ba5 AWS_S3_CERTS_BUCKET: balena-tests-certs # (kvm) nested virtualisation not supported on AWS/EC2 instance types|classes other than X.metal - AWS_EC2_INSTANCE_TYPE: c6a.2xlarge + AWS_EC2_INSTANCE_TYPES: "c6a.2xlarge c6i.2xlarge c5n.2xlarge c5.2xlarge c5a.2xlarge m5.2xlarge m5n.2xlarge m5a.2xlarge" AWS_EC2_LAUNCH_TEMPLATE: lt-02e10a4f66261319d - AWS_EC2_LT_VERSION: 2 + AWS_EC2_LT_VERSION: "6" AWS_IAM_USERNAME: balena-tests-iam-User-1GXO3XP12N6LL + AWS_LOGS_RETENTION: "30" AWS_VPC_SECURITY_GROUP_IDS: sg-057937f4d89d9d51c AWS_VPC_SUBNET_IDS: 'subnet-02d18a08ea4058574 subnet-0a026eae1df907a09' # otherwise it tries to send data to an endpoint provided by a private project # https://github.com/balena-io/analytics-backend # .. which is not part of openBalena - BALENARC_NO_ANALYTICS: '1' # https://github.com/balena-io/balena-cli/blob/master/lib/events.ts#L62-L70 - DEBUG: '0' # https://github.com/balena-io/balena-cli/issues/2447 - RETRY: 3 - SUBDOMAIN: auto + BALENARC_NO_ANALYTICS: "1" # https://github.com/balena-io/balena-cli/blob/master/lib/events.ts#L62-L70 + DEBUG: "0" # https://github.com/balena-io/balena-cli/issues/2447 + RETRY: "3" jobs: test: runs-on: ["self-hosted", "X64", "distro:jammy"] # tests require socat v1.7.4 timeout-minutes: 60 strategy: + matrix: + target: + - compose-private-pki + - balena-public-pki + include: + # tests compose flow using self-signed PKI + - target: compose-private-pki + # Canonical, Ubuntu, 24.04 LTS, amd64 noble image build on 2024-04-23 + ami: ami-04b70fa74e45c3917 + subdomain: auto + dns_tld: balena-devices.com + verbose: ${{ vars.VERBOSE || 'false' }} + + # .. balena flow using Let's Encrypt (ACME) PKI + # https://dash.cloudflare.com/001b3ed2352612aaa068aca1b0022736/balena-devices.com/dns + - target: balena-public-pki + # balenaOS-2.113.12-generic-amd64 + ami: ami-03a3995797dee84fa + subdomain: auto + dns_tld: balena-devices.com + environment: balena-cloud.com + fleet: balena/open-balena + verbose: ${{ vars.VERBOSE || 'false' }} + + # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs#handling-failures fail-fast: true + environment: + name: ${{ matrix.target }} steps: - uses: actions/checkout@b80ff79f1755d06ba70441c368a6fe801f5f3a62 - with: - # FIXME: remove once balenaBlocks/balenaVirt is a thing - submodules: true + + # https://github.com/unfor19/install-aws-cli-action + - name: Setup awscli + uses: unfor19/install-aws-cli-action@v1 - uses: aws-actions/configure-aws-credentials@bd0758102444af2a09b9e47a2c93d0f091c1252d with: @@ -73,15 +78,36 @@ jobs: # balena-io/environments-bases: aws/balenacloud/ephemeral-tests/balena-tests-iam.yml role-to-assume: ${{ vars.AWS_IAM_ROLE }} + # https://docs.aws.amazon.com/systems-manager/latest/userguide/session-manager-working-with-install-plugin.html#install-plugin-debian + - name: install session-manager-plugin + if: | + github.event_name == 'pull_request' && + github.event.action != 'closed' && + matrix.target == 'compose-private-pki' + run: | + runner_arch="$(echo "${RUNNER_ARCH}" | tr '[:upper:]' '[:lower:]' | sed 's/x64/64bit/g')" + + session-manager-plugin || (curl -sSfo session-manager-plugin.deb https://s3.amazonaws.com/session-manager-downloads/plugin/latest/ubuntu_${runner_arch}/session-manager-plugin.deb \ + && sudo dpkg -i session-manager-plugin.deb \ + && rm -f session-manager-plugin.deb) + + # https://github.com/balena-io-examples/setup-balena-action + - name: Setup balena CLI + uses: balena-io-examples/setup-balena-action@main + # https://github.com/pdcastro/ssh-uuid#why # https://github.com/pdcastro/ssh-uuid#linux-debian-ubuntu-others - name: install additional dependencies + if: | + github.event_name == 'pull_request' && + github.event.action != 'closed' && + matrix.target == 'balena-public-pki' shell: bash run: | set -ue echo '::notice::install additional dependencies' - [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x + [[ '${{ matrix.verbose || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x source src/balena-tests/functions @@ -100,13 +126,119 @@ jobs: grep -q "${RUNNER_TEMP}/ssh-uuid" "${GITHUB_PATH}" \ || echo "${RUNNER_TEMP}/ssh-uuid" >> "${GITHUB_PATH}" - - name: (pre)register test device + - name: generate SSH private key + id: generate-key-pair + if: | + github.event_name == 'pull_request' && + github.event.action != 'closed' + run: | + set -ue + + verbose='+x' + if [[ '${{ matrix.verbose }}' =~ on|On|Yes|yes|true|True ]]; then + verbose='-x' + fi + set ${verbose} + + key_name="${{ matrix.target }}-${GITHUB_RUN_ID}-${GITHUB_RUN_NUMBER}-${GITHUB_RUN_ATTEMPT}" + echo "key_name=${key_name}" >> $GITHUB_OUTPUT + + set +x + private_key_material="$(aws ec2 create-key-pair \ + --key-name "${key_name}" | jq -r .KeyMaterial)" + + public_key="$(aws ec2 describe-key-pairs --include-public-key \ + --key-name "${key_name}" | jq -re .KeyPairs[].PublicKey)" + + # https://stackoverflow.com/a/70384422/1559300 + # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#masking-a-value-in-log + while read -r line; do + echo "::add-mask::${line}" + done <<< "${private_key_material}" + + ssh_private_key="$(cat << EOF + $(echo "${private_key_material}") + EOF + )" + echo "ssh_private_key<> $GITHUB_OUTPUT + set ${verbose} + + echo "${ssh_private_key}" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + echo "${ssh_public_key}" >> $GITHUB_OUTPUT + + env: + AWS_DEFAULT_REGION: ${{ vars.AWS_REGION || 'us-east-1' }} + + # https://github.com/webfactory/ssh-agent + - uses: webfactory/ssh-agent@dc588b651fe13675774614f8e6a936a468676387 # v0.9.0 + if: | + github.event_name == 'pull_request' && + github.event.action != 'closed' + with: + ssh-private-key: ${{ steps.generate-key-pair.outputs.ssh_private_key }} + + - name: provision balenaCloud SSH key + id: provision-ssh-key + # wait for cloud-config + # https://github.com/balena-os/cloud-config + timeout-minutes: 5 + if: | + github.event_name == 'pull_request' && + github.event.action != 'closed' && + matrix.target == 'balena-public-pki' + run: | + set -ue + + [[ '${{ matrix.verbose || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x + + source src/balena-tests/functions + + with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}' + + if ! [[ -e "${HOME}/.ssh/id_rsa" ]]; then + echo '${{ steps.generate-key-pair.outputs.ssh_private_key }}' > "${HOME}/.ssh/id_rsa" + echo '${{ steps.generate-key-pair.outputs.ssh_public_key }}' > "${HOME}/.ssh/id_rsa.pub" + fi + + echo "::notice::check $(balena keys | wc -l) keys" + + match='' + for key in $(balena keys | grep -v ID | awk '{print $1}'); do + fp=$(balena key ${key} | tail -n 1 | ssh-keygen -E md5 -lf /dev/stdin | awk '{print $2}') + if [[ $fp =~ $(ssh-keygen -E md5 -lf "${HOME}/.ssh/id_rsa" | awk '{print $2}') ]]; then + match="${key}" + break + fi + done + + if [[ -z $match ]]; then + balena key add "${GITHUB_SHA}" "${HOME}/.ssh/id_rsa.pub" + else + balena keys + fi + + while ! [[ "$(ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \ + cat /mnt/boot/config.json | jq -r .uuid)" =~ ${{ steps.register-test-device.outputs.balena_device_uuid }} ]]; do + + echo "::warning::Still working..." + sleep "$(( (RANDOM % 5) + 5 ))s" + done + + echo "key_id=${GITHUB_SHA}" >> "${GITHUB_OUTPUT}" + + - name: (pre)register balenaOS test device id: register-test-device - if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}} + if: | + github.event_name == 'pull_request' && + github.event.action != 'closed' && + matrix.target == 'balena-public-pki' run: | set -ue - [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x + [[ '${{ matrix.verbose || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x source src/balena-tests/functions @@ -115,7 +247,7 @@ jobs: balena_device_uuid="$(openssl rand -hex 16)" # https://www.balena.io/docs/learn/more/masterclasses/advanced-cli/#52-preregistering-a-device - with_backoff balena device register '${{ inputs.fleet }}' --uuid "${balena_device_uuid}" + with_backoff balena device register '${{ matrix.fleet }}' --uuid "${balena_device_uuid}" device_id="$(balena device "${balena_device_uuid}" | grep ^ID: | cut -c20-)" @@ -145,12 +277,15 @@ jobs: echo "balena_device_id=${device_id}" >> "${GITHUB_OUTPUT}" # https://github.com/balena-io/balena-cli/issues/1543 - - name: pin device to draft release - if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}} + - name: pin balenaOS test device to draft release + if: | + github.event_name == 'pull_request' && + github.event.action != 'closed' && + matrix.target == 'balena-public-pki' run: | set -uae - [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x + [[ '${{ matrix.verbose || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x source src/balena-tests/functions @@ -158,7 +293,7 @@ jobs: pr_id='${{ github.event.pull_request.id }}' head_sha='${{ github.event.pull_request.head.sha || github.event.head_commit.id }}' - release_id="$(with_backoff balena releases '${{ inputs.fleet }}' --json \ + release_id="$(with_backoff balena releases '${{ matrix.fleet }}' --json \ | jq -r --arg pr_id "${pr_id}" --arg head_sha "${head_sha}" '.[] | select(.release_tag[].tag_key=="balena-ci-commit-sha") | select(.release_tag[].value==$head_sha) @@ -171,12 +306,15 @@ jobs: with_backoff balena device ${{ steps.register-test-device.outputs.balena_device_uuid }} - - name: configure test device environment - if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}} + - name: configure balenaOS test device environment + if: | + github.event_name == 'pull_request' && + github.event.action != 'closed' && + matrix.target == 'balena-public-pki' run: | set -ue - [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x + [[ '${{ matrix.verbose || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x source src/balena-tests/functions @@ -188,7 +326,7 @@ jobs: with_backoff balena env add BALENARC_NO_ANALYTICS '1' \ --device '${{ steps.register-test-device.outputs.balena_device_uuid }}' - with_backoff balena env add DNS_TLD '${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \ + with_backoff balena env add DNS_TLD '${{ matrix.target }}' \ --device '${{ steps.register-test-device.outputs.balena_device_uuid }}' with_backoff balena env add DB_HOST db \ @@ -210,39 +348,39 @@ jobs: # 10.0.2.100 # - with_backoff balena env add API_HOST 'api.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \ + with_backoff balena env add API_HOST 'api.${{ matrix.target }}' \ --device '${{ steps.register-test-device.outputs.balena_device_uuid }}' # not used but required for config.json to be valid - with_backoff balena env add DELTA_HOST 'delta.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \ + with_backoff balena env add DELTA_HOST 'delta.${{ matrix.target }}' \ --device '${{ steps.register-test-device.outputs.balena_device_uuid }}' - with_backoff balena env add REGISTRY2_HOST 'registry2.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \ + with_backoff balena env add REGISTRY2_HOST 'registry2.${{ matrix.target }}' \ --device '${{ steps.register-test-device.outputs.balena_device_uuid }}' - with_backoff balena env add VPN_HOST 'cloudlink.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \ + with_backoff balena env add VPN_HOST 'cloudlink.${{ matrix.target }}' \ --device '${{ steps.register-test-device.outputs.balena_device_uuid }}' - with_backoff balena env add HOST 'api.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \ + with_backoff balena env add HOST 'api.${{ matrix.target }}' \ --service api \ --device '${{ steps.register-test-device.outputs.balena_device_uuid }}' - with_backoff balena env add TOKEN_AUTH_CERT_ISSUER 'api.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \ + with_backoff balena env add TOKEN_AUTH_CERT_ISSUER 'api.${{ matrix.target }}' \ --service api \ --device '${{ steps.register-test-device.outputs.balena_device_uuid }}' - with_backoff balena env add REGISTRY2_TOKEN_AUTH_ISSUER 'api.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \ + with_backoff balena env add REGISTRY2_TOKEN_AUTH_ISSUER 'api.${{ matrix.target }}' \ --service registry \ --device '${{ steps.register-test-device.outputs.balena_device_uuid }}' - with_backoff balena env add REGISTRY2_TOKEN_AUTH_REALM 'https://api.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}/auth/v1/token' \ + with_backoff balena env add REGISTRY2_TOKEN_AUTH_REALM 'https://api.${{ matrix.target }}/auth/v1/token' \ --service registry \ --device '${{ steps.register-test-device.outputs.balena_device_uuid }}' - with_backoff balena env add REGISTRY2_S3_REGION_ENDPOINT 's3.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \ + with_backoff balena env add REGISTRY2_S3_REGION_ENDPOINT 's3.${{ matrix.target }}' \ --device '${{ steps.register-test-device.outputs.balena_device_uuid }}' - with_backoff balena env add WEBRESOURCES_S3_HOST 's3.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \ + with_backoff balena env add WEBRESOURCES_S3_HOST 's3.${{ matrix.target }}' \ --device '${{ steps.register-test-device.outputs.balena_device_uuid }}' # https://github.com/balena-io/cert-manager/blob/master/entry.sh#L255-L278 @@ -256,7 +394,7 @@ jobs: with_backoff balena env add COMMON_REGION '${{ env.AWS_REGION }}' \ --device '${{ steps.register-test-device.outputs.balena_device_uuid }}' - with_backoff balena env add SUPERUSER_EMAIL 'admin@${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \ + with_backoff balena env add SUPERUSER_EMAIL 'admin@${{ matrix.target }}' \ --device '${{ steps.register-test-device.outputs.balena_device_uuid }}' with_backoff balena env add ORG_UNIT openBalena \ @@ -272,12 +410,15 @@ jobs: --service sut \ --device '${{ steps.register-test-device.outputs.balena_device_uuid }}' - - name: configure test device secrets - if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}} + - name: configure balenaOS test device secrets + if: | + github.event_name == 'pull_request' && + github.event.action != 'closed' && + matrix.target == 'balena-public-pki' run: | set -ue - [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x + [[ '${{ matrix.verbose || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x source src/balena-tests/functions @@ -302,28 +443,35 @@ jobs: --service cert-manager \ --device '${{ steps.register-test-device.outputs.balena_device_uuid }}' - - name: provision ephemeral test device - id: provision-test-device - if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}} + - name: provision balenaOS ephemeral SUT + id: balena-sut + if: | + github.event_name == 'pull_request' && + github.event.action != 'closed' && + matrix.target == 'balena-public-pki' run: | set -ue - [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x + [[ '${{ matrix.verbose || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x for subnet_id in ${{ env.AWS_VPC_SUBNET_IDS }}; do # spot, on-demand for market_type in ${{ vars.MARKET_TYPES || 'spot' }}; do - # https://docs.aws.amazon.com/cli/latest/reference/ec2/run-instances.html - response="$(aws ec2 run-instances \ - --launch-template 'LaunchTemplateId=${{ env.AWS_EC2_LAUNCH_TEMPLATE }},Version=${{ env.AWS_EC2_LT_VERSION }}' \ - --instance-type '${{ env.AWS_EC2_INSTANCE_TYPE }}' \ - $([[ $market_type =~ spot ]] && echo '--instance-market-options MarketType=spot') \ - --security-group-ids '${{ env.AWS_VPC_SECURITY_GROUP_IDS }}' \ - --subnet-id "${subnet_id}" \ - --associate-public-ip-address \ - --user-data file://config.json \ - --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=balena-tests},{Key=MarketType,Value=${market_type}},{Key=Owner,Value=${{ env.AWS_IAM_USERNAME }}},{Key=GITHUB_SHA,Value=${GITHUB_SHA}-tests}]" || true)" - + for instance_type in ${AWS_EC2_INSTANCE_TYPES}; do + # https://docs.aws.amazon.com/cli/latest/reference/ec2/run-instances.html + response="$(aws ec2 run-instances \ + --image-id ${{ matrix.ami }} \ + --launch-template 'LaunchTemplateId=${{ env.AWS_EC2_LAUNCH_TEMPLATE }},Version=${{ env.AWS_EC2_LT_VERSION }}' \ + --instance-type "${instance_type}" \ + $([[ $market_type =~ spot ]] && echo '--instance-market-options MarketType=spot') \ + --security-group-ids '${{ env.AWS_VPC_SECURITY_GROUP_IDS }}' \ + --subnet-id "${subnet_id}" \ + --associate-public-ip-address \ + --user-data file://config.json \ + --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=open-balena-tests},{Key=matrix.target,Value=${{ matrix.target }}},{Key=MarketType,Value=${market_type}},{Key=Owner,Value=${{ env.AWS_IAM_USERNAME }}},{Key=GITHUB_SHA,Value=${GITHUB_SHA}-tests},{Key=GITHUB_RUN_ID,Value=${GITHUB_RUN_ID}-tests},{Key=GITHUB_RUN_NUMBER,Value=${GITHUB_RUN_NUMBER}-tests},{Key=GITHUB_RUN_ATTEMPT,Value=${GITHUB_RUN_ATTEMPT}-tests}]" || true)" + + [[ -n $response ]] && break + done [[ -n $response ]] && break done [[ -n $response ]] && break @@ -340,72 +488,18 @@ jobs: echo "instance_id=${instance_id}" >> "${GITHUB_OUTPUT}" env: - AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ env.AWS_REGION }} - - - name: provision SSH key - id: provision-ssh-key - # wait for cloud-config - # https://github.com/balena-os/cloud-config - timeout-minutes: 5 - if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}} - run: | - set -ue - - [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x - - source src/balena-tests/functions - - with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}' + AWS_DEFAULT_REGION: ${{ vars.AWS_REGION || 'us-east-1' }} - if ! [[ -e "${HOME}/.ssh/id_rsa" ]]; then - ssh-keygen -N '' \ - -C "$(balena whoami | grep EMAIL | cut -c11-)" \ - -f "${HOME}/.ssh/id_rsa" - fi - - echo "::notice::check $(balena keys | wc -l) keys" - - match='' - for key in $(balena keys | grep -v ID | awk '{print $1}'); do - fp=$(balena key ${key} | tail -n 1 | ssh-keygen -E md5 -lf /dev/stdin | awk '{print $2}') - if [[ $fp =~ $(ssh-keygen -E md5 -lf "${HOME}/.ssh/id_rsa" | awk '{print $2}') ]]; then - match="${key}" - break - fi - done - - if [[ -z $match ]]; then - balena key add "${GITHUB_SHA}" "${HOME}/.ssh/id_rsa.pub" - else - balena keys - fi - - pgrep ssh-agent || ssh-agent -a "${SSH_AUTH_SOCK}" - - ssh-add "${HOME}/.ssh/id_rsa" - - while ! [[ "$(ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \ - cat /mnt/boot/config.json | jq -r .uuid)" =~ ${{ steps.register-test-device.outputs.balena_device_uuid }} ]]; do - - echo "::warning::Still working..." - sleep "$(( (RANDOM % 5) + 5 ))s" - done - - echo "key_id=${GITHUB_SHA}" >> "${GITHUB_OUTPUT}" - - env: - SSH_AUTH_SOCK: /tmp/ssh_agent.sock - - - name: wait for application + - name: wait for balenaCloud application timeout-minutes: 10 - if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}} + if: | + github.event_name == 'pull_request' && + github.event.action != 'closed' && + matrix.target == 'balena-public-pki' run: | set -ue - [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x + [[ '${{ matrix.verbose || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x source src/balena-tests/functions @@ -414,7 +508,7 @@ jobs: balena whoami && ssh-add -l while [[ "$(curl -X POST --silent --retry ${{ env.RETRY }} --fail \ - 'https://api.${{ inputs.environment }}/supervisor/v1/device' \ + 'https://api.${{ matrix.environment }}/supervisor/v1/device' \ --header 'authorization: Bearer ${{ secrets.BALENA_API_KEY }}' \ --header 'Content-Type:application/json' \ --data '{"uuid": "${{ steps.register-test-device.outputs.balena_device_uuid }}", "method": "GET"}' \ @@ -444,20 +538,20 @@ jobs: sleep "$(( (RANDOM % 30) + 30 ))s" done - env: - SSH_AUTH_SOCK: /tmp/ssh_agent.sock - # (TBC) https://www.balena.io/docs/reference/supervisor/docker-compose/ # due to lack of long form depends_on support in compositions, restart to ensure all # components are running with the latest configuration; preferred over restart via # Supervisor API restart due to potential HTTP [timeouts](https://github.com/balena-os/balena-supervisor/issues/1157) - - name: restart components + - name: restart balenaEngine composition timeout-minutes: 10 - if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}} + if: | + github.event_name == 'pull_request' && + github.event.action != 'closed' && + matrix.target == 'balena-public-pki' run: | set -ue - [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x + [[ '${{ matrix.verbose || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x source src/balena-tests/functions @@ -483,11 +577,11 @@ jobs: sleep "$(( (RANDOM % 30) + 30 ))s" done - env: - SSH_AUTH_SOCK: /tmp/ssh_agent.sock - - - name: SUT&DUT - if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}} + - name: SUT&DUT (balena) + if: | + github.event_name == 'pull_request' && + github.event.action != 'closed' && + matrix.target == 'balena-public-pki' timeout-minutes: 20 # https://giters.com/gfx/example-github-actions-with-tty # https://github.com/actions/runner/issues/241#issuecomment-924327172 @@ -495,7 +589,7 @@ jobs: run: | set -ue - [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x + [[ '${{ matrix.verbose || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x source src/balena-tests/functions @@ -513,7 +607,7 @@ jobs: status='' while [[ "$status" =~ Running ]]; do status="$(curl --silent --retry ${{ env.RETRY }} --fail \ - 'https://api.${{ inputs.environment }}/supervisor/v2/applications/state' \ + 'https://api.${{ matrix.environment }}/supervisor/v2/applications/state' \ --header 'authorization: Bearer ${{ secrets.BALENA_API_KEY }}' \ --header 'Content-Type:application/json' \ --data '{"uuid": "${{ steps.register-test-device.outputs.balena_device_uuid }}", "method": "GET"}' \ @@ -528,7 +622,7 @@ jobs: while ! [[ "$status" =~ exited ]]; do echo "::warning::Still working..." status="$(curl --silent --retry ${{ env.RETRY }} --fail \ - 'https://api.${{ inputs.environment }}/supervisor/v2/applications/state' \ + 'https://api.${{ matrix.environment }}/supervisor/v2/applications/state' \ --header 'authorization: Bearer ${{ secrets.BALENA_API_KEY }}' \ --header 'Content-Type:application/json' \ --data '{"uuid": "${{ steps.register-test-device.outputs.balena_device_uuid }}", "method": "GET"}' \ @@ -547,16 +641,267 @@ jobs: [[ $expected_exit_code -eq $actual_exit_code ]] || false env: - SSH_AUTH_SOCK: /tmp/ssh_agent.sock ATTEMPTS: 2 - - name: remove SSH key - if: always() + - name: provision Ubuntu ephemeral SUT + id: ubuntu-sut + if: | + github.event_name == 'pull_request' && + github.event.action != 'closed' && + matrix.target == 'compose-private-pki' + run: | + set -ue + + [[ '${{ matrix.verbose }}' =~ on|On|Yes|yes|true|True ]] && set -x + + function cleanup() { + rm -f user-data.yml + } + trap 'cleanup' EXIT + + aws sts get-caller-identity + + # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/user-data.html + # https://cloudinit.readthedocs.io/en/latest/reference/modules.html#update-etc-hosts + cat << EOF > user-data.yml + #cloud-config + output : { all : '| tee -a /var/log/cloud-init-output.log' } + manage_etc_hosts: api.${{ matrix.subdomain }}.${{ matrix.dns_tld }} + + packages: + - git + - jq + - wget + + write_files: + - path: /root/.env + permissions: 0600 + content: | + DNS_TLD=${{ matrix.subdomain }}.${{ matrix.dns_tld }} + PRODUCTION_MODE=false + VERBOSE=${{ matrix.verbose }} + + - path: /root/functions + permissions: 0755 + content: | + # https://coderwall.com/p/--eiqg/exponential-backoff-in-bash + function with_backoff() { + local max_attempts=${ATTEMPTS-5} + local timeout=${TIMEOUT-1} + local attempt=0 + local exitCode=0 + + set +e + while [[ $attempt < $max_attempts ]] + do + "$@" + exitCode=$? + + if [[ $exitCode == 0 ]] + then + break + fi + + echo "Failure! Retrying in $timeout.." 1>&2 + sleep "$timeout" + attempt=$(( attempt + 1 )) + timeout=$(( timeout * 2 )) + done + + if [[ $exitCode != 0 ]] + then + echo "You've failed me for the last time! ($*)" 1>&2 + fi + set -e + return $exitCode + } + + # docs/getting-started.md + - path: /root/getting-started.sh + permissions: 0755 + content: | + #!/usr/bin/env bash + + set -ax + + [[ '${{ matrix.verbose }}' =~ on|On|Yes|yes|true|True ]] && set -x + + source /root/functions + + apt-get update + which openssl || apt-get install -y make openssl + which git || apt-get install -y make git + which jq || apt-get install -y make jq + which make || apt-get install make + + which yq || with_backoff wget -q https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq + chmod +x /usr/bin/yq + yq --version + + which docker || curl -fsSL https://get.docker.com | sh - + usermod -aG docker ubuntu + systemctl enable docker && systemctl start docker + chown ubuntu:docker /var/run/docker.sock + + id -u balena || useradd -s /bin/bash -m -G docker,sudo balena + echo 'balena ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/balena + + while ! docker ps; do sleep $(((RANDOM%3)+1)); done + with_backoff docker login \ + --username='${{ secrets.DOCKERHUB_USER }}' \ + --password='${{ secrets.DOCKERHUB_TOKEN }}' + + with_backoff docker login ghcr.io \ + --username=token \ + --password=${{ secrets.GITHUB_TOKEN }} + + if [ ! -f /sys/fs/cgroup/cgroup.controllers ]; then + echo "cgroups v2 is disabled" + else + echo "cgroups v2 is enabled" + source /etc/default/grub + sed -i '/GRUB_CMDLINE_LINUX/d' /etc/default/grub + echo GRUB_CMDLINE_LINUX=$(printf '\"%s systemd.unified_cgroup_hierarchy=0\"\n' "${GRUB_CMDLINE_LINUX}") > /etc/default/grub + update-grub + reboot + fi + + # cloud-init runs as root + # (e.g.) https://cloudinit.readthedocs.io/en/latest/reference/merging.html#example-cloud-config + runcmd: + - '/root/getting-started.sh' # FIXME: this may run before the script is written + EOF + + for subnet_id in ${{ env.AWS_VPC_SUBNET_IDS }}; do + # spot, on-demand + for market_type in ${{ vars.MARKET_TYPES }}; do + for instance_type in ${AWS_EC2_INSTANCE_TYPES}; do + # https://docs.aws.amazon.com/cli/latest/reference/ec2/run-instances.html + response="$(aws ec2 run-instances \ + --image-id "${{ matrix.ami }}" \ + --launch-template "LaunchTemplateId=${AWS_EC2_LAUNCH_TEMPLATE},Version=${AWS_EC2_LT_VERSION}" \ + --instance-type "${instance_type}" \ + $([[ $market_type =~ spot ]] && echo '--instance-market-options MarketType=spot') \ + --security-group-ids "${AWS_VPC_SECURITY_GROUP_IDS}" \ + --subnet-id "${subnet_id}" \ + --key-name '${{ steps.generate-key-pair.outputs.key_name }}' \ + --associate-public-ip-address \ + --user-data file://user-data.yml \ + --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=open-balena-tests},{Key=matrix.target,Value=${{ matrix.target }}},{Key=MarketType,Value=${market_type}},{Key=Owner,Value=${{ env.AWS_IAM_USERNAME }}},{Key=GITHUB_SHA,Value=${GITHUB_SHA}-tests},{Key=GITHUB_RUN_ID,Value=${GITHUB_RUN_ID}-tests},{Key=GITHUB_RUN_NUMBER,Value=${GITHUB_RUN_NUMBER}-tests},{Key=GITHUB_RUN_ATTEMPT,Value=${GITHUB_RUN_ATTEMPT}-tests}]" || true)" + + [[ -n $response ]] && break + done + [[ -n $response ]] && break + done + [[ -n $response ]] && break + done + + [[ -z $response ]] && exit 1 + + instance_id="$(echo "${response}" | jq -r '.Instances[].InstanceId')" + echo "instance_id=${instance_id}" >> $GITHUB_OUTPUT + + aws ec2 wait instance-running --instance-ids "${instance_id}" + aws ec2 wait instance-status-ok --instance-ids "${instance_id}" + + env: + AWS_DEFAULT_REGION: ${{ vars.AWS_REGION || 'us-east-1' }} + COMMIT: ${{ github.event.pull_request.head.sha || github.event.head_commit.id || github.event.pull_request.head.ref }} + + - name: SUT&DUT (Ubuntu/compose) + if: | + github.event_name == 'pull_request' && + github.event.action != 'closed' && + matrix.target == 'compose-private-pki' + run: | + set -ue + + [[ '${{ matrix.verbose }}' =~ on|On|Yes|yes|true|True ]] && set -x + + function log_output() { + rm -f "{HOME}/.ssh/config" + + aws ssm list-command-invocations \ + --details \ + --output text \ + --command-id "${id}" || true + + aws logs describe-log-streams \ + --log-group-name open-balena-tests \ + --log-stream-name-prefix "${id}" || true + + aws logs put-retention-policy \ + --log-group-name open-balena-tests \ + --retention-in-days "${{ env.AWS_LOGS_RETENTION }}" || true + } + trap 'log_output' EXIT + + # docs/getting-started.md + CMDS="set -ax \ + && cloud-init status --wait --long && cat "${HOME}/.ssh/config" + host i-* + StrictHostKeyChecking no + UserKnownHostsFile /dev/null + ProxyCommand sh -c "aws ssm start-session --target %h --document-name AWS-StartSSHSession --parameters 'portNumber=%p'" + EOF + + # AWS-RunShellScript runs as root + id="$(aws ssm send-command \ + --instance-ids ${{ steps.launch-ec2-instance.outputs.instance_id }} \ + --document-name AWS-RunShellScript \ + --comment "open-balena-tests@${{ matrix.target }}" \ + --parameters commands=["${CMDS}"] \ + --cloud-watch-output-config '{"CloudWatchLogGroupName":"open-balena-tests","CloudWatchOutputEnabled":true}' | jq -r .Command.CommandId)" + + [[ -n $id ]] || false + + while [[ $(aws logs describe-log-streams \ + --log-group-name open-balena-tests \ + --log-stream-name-prefix "${id}" | jq -r '.logStreams|length') -le 0 ]]; do + echo '::info::waiting for logs...' + done + + # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines + CYAN='\033[0;36m'; NC='\033[0m'; echo -e "::group::${CYAN}${{ env.SLUG }}${NC}" + while [[ "$(aws ssm list-command-invocations --command-id "${id}" \ + | jq -r '.CommandInvocations[].Status')" =~ InProgress ]]; do + docker ps -q | xargs docker logs --timestamps --follow --details \ + || echo '::info::waiting for logs...' + sleep $(((RANDOM%1) + 1))s + done + echo "::endgroup::" + + if ! [[ "$(aws ssm list-command-invocations --command-id "${id}" \ + | jq -r '.CommandInvocations[].Status')" =~ Success ]]; then + false + fi + + env: + AWS_DEFAULT_REGION: ${{ vars.AWS_REGION || 'us-east-1' }} + DOCKER_HOST: ssh://ec2-user@${{ steps.launch-ec2-instance.outputs.instance_id }}:22 + COMMIT: ${{ github.event.pull_request.head.sha || github.event.head_commit.id || github.event.pull_request.head.ref }} + + - name: remove balenaCloud SSH key + if: always() && matrix.target == 'balena-public-pki' continue-on-error: true run: | set -ue - [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x + [[ '${{ matrix.verbose || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x source src/balena-tests/functions @@ -565,17 +910,16 @@ jobs: with_backoff balena keys | grep ${{ steps.provision-ssh-key.outputs.key_id }} \ | awk '{print $1}' | xargs balena key rm --yes - pgrep ssh-agent && (pgrep ssh-agent | xargs kill) - - rm -f /tmp/ssh_agent.sock - - - name: destroy balena test device - if: always() + - name: delete balenaOS test device + if: | + github.event_name == 'pull_request' && + github.event.action != 'closed' && + matrix.target == 'balena-public-pki' continue-on-error: true run: | set -ue - [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x + [[ '${{ matrix.verbose || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x source src/balena-tests/functions @@ -584,23 +928,26 @@ jobs: with_backoff balena device rm ${{ steps.register-test-device.outputs.balena_device_uuid }} --yes env: - AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ env.AWS_REGION }} + AWS_DEFAULT_REGION: ${{ vars.AWS_REGION || 'us-east-1' }} # always destroy test EC2 instances even if the workflow is cancelled - - name: destroy AWS test device + - name: destroy AWS test device(s) if: always() run: | set -ue - [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x + [[ '${{ matrix.verbose || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x source src/balena-tests/functions - if [[ -n '${{ steps.provision-test-device.outputs.instance_id }}' ]]; then + if [[ -n '${{ steps.balena-sut.outputs.instance_id }}' ]]; then with_backoff aws ec2 terminate-instances \ - --instance-ids ${{ steps.provision-test-device.outputs.instance_id }} + --instance-ids ${{ steps.balena-sut.outputs.instance_id }} + fi + + if [[ -n '${{ steps.ubuntu-sut.outputs.instance_id }}' ]]; then + with_backoff aws ec2 terminate-instances \ + --instance-ids ${{ steps.ubuntu-sut.outputs.instance_id }} fi with_backoff aws ec2 describe-instances --filters Name=tag:GITHUB_SHA,Values=${GITHUB_SHA}-tests \ @@ -609,7 +956,7 @@ jobs: stale_instances=$(mktemp) aws ec2 describe-instances --filters \ - Name=tag:Name,Values=balena-tests \ + Name=tag:Name,Values=open-balena-tests \ Name=instance-state-name,Values=running \ | jq -re '.Reservations[].Instances[].InstanceId + " " + .Reservations[].Instances[].LaunchTime' > ${stale_instances} || true @@ -628,26 +975,24 @@ jobs: fi env: - AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ env.AWS_REGION }} + AWS_DEFAULT_REGION: ${{ vars.AWS_REGION || 'us-east-1' }} # remove orphaned ACME DNS-01 validation records # https://letsencrypt.org/docs/challenge-types/#dns-01-challenge # FIXME: clean up older _acme-challenge.auto TXT records - - name: cleanup-dns-records - if: always() + - name: cleanup Cloudflare DNS + if: always() && matrix.target == 'balena-public-pki' continue-on-error: true run: | set -ue - [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x + [[ '${{ matrix.verbose || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x if [[ -n '${{ steps.register-test-device.outputs.balena_device_uuid }}' ]]; then - match="${{ steps.register-test-device.outputs.balena_device_uuid }}.${{ env.SUBDOMAIN }}" + match="${{ steps.register-test-device.outputs.balena_device_uuid }}.${{ matrix.subdomain }}" zone_id="$(curl --silent --retry ${{ env.RETRY }} \ - "https://api.cloudflare.com/client/v4/zones?name=${{ inputs.dns_tld }}" \ + "https://api.cloudflare.com/client/v4/zones?name=${{ matrix.dns_tld }}" \ -H 'Authorization: Bearer ${{ secrets.CLOUDFLARE_API_TOKEN }}' | jq -r '.result[].id')" for record in "$(curl --silent --retry ${{ env.RETRY }} \ @@ -673,4 +1018,4 @@ jobs: fi env: - DRY_RUN: true + DRY_RUN: false diff --git a/Makefile b/Makefile index ecbb3e077e..1832a9983f 100644 --- a/Makefile +++ b/Makefile @@ -3,15 +3,17 @@ SHELL := bash # export all variables to child processes by default export -# include the .env file +# include the .env file if it exists -include .env +BALENARC_NO_ANALYTICS ?= 1 DNS_TLD ?= $(error DNS_TLD not set) -TMPKI := $(shell mktemp) -STAGING_PKI ?= /usr/local/share/ca-certificates -PRODUCTION_MODE ?= true ORG_UNIT ?= openBalena +PRODUCTION_MODE ?= true +STAGING_PKI ?= /usr/local/share/ca-certificates SUPERUSER_EMAIL ?= admin@$(DNS_TLD) +TMPKI := $(shell mktemp) +VERBOSE ?= false .NOTPARALLEL: $(DOCKERCOMPOSE) @@ -39,16 +41,30 @@ ifneq ($(GANDI_API_TOKEN),) endif endif @rm -f .env - @echo "DNS_TLD=$(DNS_TLD)" >> .env + @echo "BALENARC_NO_ANALYTICS=$(BALENARC_NO_ANALYTICS)" >> .env + @echo "DNS_TLD=$(DNS_TLD)" > .env @echo "ORG_UNIT=$(ORG_UNIT)" >> .env - @echo "SUPERUSER_EMAIL=$(SUPERUSER_EMAIL)" >> .env @echo "PRODUCTION_MODE=$(PRODUCTION_MODE)" >> .env - @echo "GANDI_API_TOKEN=$(GANDI_API_TOKEN)" >> .env - @echo "CLOUDFLARE_API_TOKEN=$(CLOUDFLARE_API_TOKEN)" >> .env + @echo "SUPERUSER_EMAIL=$(SUPERUSER_EMAIL)" >> .env + @echo "VERBOSE=$(VERBOSE)" >> .env +ifneq ($(ACME_EMAIL),) @echo "ACME_EMAIL=$(ACME_EMAIL)" >> .env +endif +ifneq ($(CLOUDFLARE_API_TOKEN),) + @echo "CLOUDFLARE_API_TOKEN=$(CLOUDFLARE_API_TOKEN)" >> .env +endif +ifneq ($(GANDI_API_TOKEN),) + @echo "GANDI_API_TOKEN=$(GANDI_API_TOKEN)" >> .env +endif +ifneq ($(HAPROXY_CRT),) @echo "HAPROXY_CRT=$(HAPROXY_CRT)" >> .env +endif +ifneq ($(HAPROXY_KEY),) @echo "HAPROXY_KEY=$(HAPROXY_KEY)" >> .env +endif +ifneq ($(ROOT_CA),) @echo "ROOT_CA=$(ROOT_CA)" >> .env +endif @$(MAKE) showenv .PHONY: up diff --git a/docker-compose.yml b/docker-compose.yml index df4c6d685c..075258c890 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -344,11 +344,23 @@ services: *with-network-privileges, ] image: docker:dind + entrypoint: + - /bin/sh + - -c + command: + - | + set -x + + cp /certs/root-ca.pem /certs/server-ca.pem /usr/local/share/ca-certificates/ \ + && update-ca-certificates + + exec /usr/local/bin/dockerd-entrypoint.sh volumes: - - builder-data:/var/lib/docker + - /sys:/sys - builder-certs-ca:/docker-pki/ca - builder-certs-client:/docker-pki/client - - /sys:/sys + - builder-data:/var/lib/docker + - certs-data:/certs environment: DOCKER_TLS_CERTDIR: /docker-pki healthcheck: diff --git a/docs/getting-started.md b/docs/getting-started.md index 308f1fbbeb..d39db7d80e 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -82,7 +82,7 @@ name and configure records. ```bash sudo useradd -s /bin/bash -m -G docker,sudo balena - echo 'balena ALL=(ALL) NOPASSWD: ALL' | tee >/etc/sudoers.d/balena + echo 'balena ALL=(ALL) NOPASSWD: ALL' | sudo tee /etc/sudoers.d/balena ``` 6. Switch user: