nx-cloud-main.yml

name: Nx Cloud Main

on:
  workflow_call:
    secrets:
      NPM_TOKEN:
        required: false
      NX_CLOUD_ACCESS_TOKEN:
        required: false
      NX_CLOUD_AUTH_TOKEN:
        required: false
    inputs:
      number-of-agents:
        required: false
        type: number
      environment-variables:
        required: false
        type: string
      init-commands:
        required: false
        type: string
      final-commands:
        required: false
        type: string
      parallel-commands:
        required: false
        type: string
      parallel-commands-on-agents:
        required: false
        type: string
      node-version:
        required: false
        type: string
      yarn-version:
        required: false
        type: string
      npm-version:
        required: false
        type: string
      pnpm-version:
        required: false
        type: string
      install-commands:
        required: false
        type: string
      main-branch-name:
        required: false
        type: string
        default: main
      runs-on:
        required: false
        type: string
        default: ubuntu-latest
      artifacts-path:
        required: false
        type: string
        default: ""
      artifacts-name:
        required: false
        type: string
        default: "nx-main-artifacts"
      artifacts-retention-days:
        required: false
        type: number
        default: 90
      working-directory:
        required: false
        type: string
      timeout:
        required: false
        type: number

env:
  NX_CLOUD_DISTRIBUTED_EXECUTION: true
  NX_CLOUD_DISTRIBUTED_EXECUTION_AGENT_COUNT: ${{ inputs.number-of-agents }}
  NX_BRANCH: ${{ github.event.number || github.ref_name }}
  NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }}
  NX_CLOUD_AUTH_TOKEN: ${{ secrets.NX_CLOUD_AUTH_TOKEN }}
  NPM_TOKEN: ${{ secrets.NPM_TOKEN }}

jobs:
  main:
    runs-on: ${{ inputs.runs-on }}
    # The name of the job which will invoke this one is expected to be "Nx Cloud - Main Job", and whatever we call this will be appended
    # to that one after a forward slash, so we keep this one intentionally short to produce "Nx Cloud - Main Job / Run" in the Github UI
    name: Run
    # GitHub's default timeout is 360 minutes
    timeout-minutes: ${{ inputs.timeout || 360 }}
    defaults:
      run:
        working-directory: ${{ inputs.working-directory || github.workspace }}
        # Specify shell to help normalize across different operating systems
        shell: bash
    steps:
      - uses: actions/checkout@v4
        name: Checkout [Pull Request]
        if: ${{ github.event_name == 'pull_request' }}
        with:
          # By default, PRs will be checked-out based on the Merge Commit, but we want the actual branch HEAD.
          ref: ${{ github.event.pull_request.head.sha }}
          # We need to fetch all branches and commits so that Nx affected has a base to compare against.
          fetch-depth: 0

      - uses: actions/checkout@v4
        name: Checkout [Default Branch]
        if: ${{ github.event_name != 'pull_request' }}
        with:
          # We need to fetch all branches and commits so that Nx affected has a base to compare against.
          fetch-depth: 0

      - name: Derive appropriate SHAs for base and head for `nx affected` commands
        uses: nrwl/nx-set-shas@v4
        with:
          main-branch-name: ${{ inputs.main-branch-name }}

      - name: Detect package manager
        id: package_manager
        shell: bash
        run: |
          echo "name=$([[ -f ./yarn.lock ]] && echo "yarn" || ([[ -f ./pnpm-lock.yaml ]] && echo "pnpm") || echo "npm")" >> $GITHUB_OUTPUT

      # Set node/npm/yarn versions using volta, with optional overrides provided by the consumer
      - uses: volta-cli/action@v4
        with:
          node-version: "${{ inputs.node-version }}"
          npm-version: "${{ inputs.npm-version }}"
          yarn-version: "${{ inputs.yarn-version }}"
          package-json-path: "${{ inputs.working-directory || github.workspace }}/package.json"

      # Install pnpm with exact version provided by consumer or fallback to latest
      - name: Install PNPM
        if: steps.package_manager.outputs.name == 'pnpm'
        uses: pnpm/action-setup@v3
        with:
          version: ${{ inputs.pnpm-version }}
          package_json_file: "${{ inputs.working-directory && format('{0}/package.json', inputs.working-directory) || 'package.json' }}"

      - name: Print node/npm/yarn versions
        id: versions
        run: |
          node_ver=$( node --version )
          yarn_ver=$( yarn --version || true )
          pnpm_ver=$( pnpm --version || true )

          echo "Node: ${node_ver:1}"
          echo "NPM: $( npm --version )"
          if [[ $yarn_ver != '' ]]; then echo "Yarn: $yarn_ver"; fi
          if [[ $pnpm_ver != '' ]]; then echo "PNPM: $pnpm_ver"; fi

          echo "node_version=${node_ver:1}" >> $GITHUB_OUTPUT
          echo "yarn_version=${yarn_ver}" >> $GITHUB_OUTPUT

      - name: Use the node_modules cache if available [npm]
        if: steps.package_manager.outputs.name == 'npm'
        uses: actions/cache@v4
        with:
          path: ~/.npm
          key: ${{ runner.os }}-node-${{ steps.versions.outputs.node_version }}-${{ hashFiles('**/package-lock.json') }}
          restore-keys: |
            ${{ runner.os }}-node-${{ steps.versions.outputs.node_version }}-

      - name: Get pnpm cache directory path
        if: steps.package_manager.outputs.name == 'pnpm'
        id: pnpm-cache-dir-path
        run: echo "dir=$(pnpm store path)" >> $GITHUB_OUTPUT

      - name: Use the node_modules cache if available [pnpm]
        if: steps.package_manager.outputs.name == 'pnpm'
        uses: actions/cache@v4
        with:
          path: ${{ steps.pnpm-cache-dir-path.outputs.dir }}
          key: ${{ runner.os }}-node-${{ steps.versions.outputs.node_version }}-${{ hashFiles('**/pnpm-lock.yaml') }}
          restore-keys: |
            ${{ runner.os }}-node-${{ steps.versions.outputs.node_version }}-

      - name: Get yarn cache directory path [yarn 1.x]
        if: steps.package_manager.outputs.name == 'yarn' && startsWith(steps.versions.outputs.yarn_version, '1.')
        id: yarn-cache-dir-path
        run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT

      - name: Use the node_modules cache if available [yarn 1.x]
        if: steps.package_manager.outputs.name == 'yarn' && startsWith(steps.versions.outputs.yarn_version, '1.')
        uses: actions/cache@v4
        with:
          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
          key: ${{ runner.os }}-node-${{ steps.versions.outputs.node_version }}-yarn-${{ hashFiles('**/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-node-${{ steps.versions.outputs.node_version }}-yarn-

      - name: Get yarn cache directory path [yarn berry]
        if: steps.package_manager.outputs.name == 'yarn' && !startsWith(steps.versions.outputs.yarn_version, '1.')
        id: yarn-berry-cache-dir-path
        run: echo "dir=$(yarn config get cacheFolder)" >> $GITHUB_OUTPUT

      - name: Use the node_modules cache if available [yarn berry]
        if: steps.package_manager.outputs.name == 'yarn' && !startsWith(steps.versions.outputs.yarn_version, '1.')
        uses: actions/cache@v4
        with:
          path: ${{ steps.yarn-berry-cache-dir-path.outputs.dir }}
          key: ${{ runner.os }}-node-${{ steps.versions.outputs.node_version }}-yarn-berry-${{ hashFiles('**/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-node-${{ steps.versions.outputs.node_version }}-yarn-berry-

      - name: Process environment-variables
        if: ${{ inputs.environment-variables != '' }}
        uses: actions/github-script@v7
        env:
          ENV_VARS: ${{ inputs.environment-variables }}
        with:
          script: |
            const { appendFileSync } = require('fs');

            // trim spaces and escape quotes
            const cleanStr = str => str
              .trim()
              .replaceAll(/`/g, "\`");

            // parse variable to correct type
            const parseStr = str =>
              str === 'true' || str === 'TRUE'
                ? true
                : str === 'false' || str === 'FALSE'
                  ? false
                  : isNaN(str)
                    ? str
                    : parseFloat(str);

            const varsStr = process.env.ENV_VARS || '';
            const vars = varsStr
              .split('\n')
              .map(variable => variable.trim())
              .filter(variable => variable.indexOf('=') > 0)
              .map(variable => ({
                name: cleanStr(variable.split('=')[0]),
                value: cleanStr(variable.slice(variable.indexOf('=') + 1))
              }));

            for (const v of vars) {
              console.log(`Appending environment variable \`${v.name}\` with value \`${v.value}\` to ${process.env.GITHUB_ENV}`);
              appendFileSync(process.env.GITHUB_ENV, `${v.name}=${parseStr(v.value)}\n`);
            }

      - name: Run any configured install-commands
        if: ${{ inputs.install-commands != '' }}
        run: |
          ${{ inputs.install-commands }}

      - name: Install dependencies
        if: ${{ inputs.install-commands == '' }}
        run: |
          if [ "${{ steps.package_manager.outputs.name == 'yarn' }}" == "true" ]; then
            if [ "${{ startsWith(steps.versions.outputs.yarn_version, '1.') }}" == "true" ]; then
              echo "Running yarn install --frozen-lockfile"
              yarn install --frozen-lockfile
            else
              echo "Running yarn install --immutable"
              yarn install --immutable
            fi
          elif [ "${{ steps.package_manager.outputs.name == 'pnpm' }}" == "true" ]; then
            echo "Running pnpm install --frozen-lockfile"
            pnpm install --frozen-lockfile
          else
            echo "Running npm ci"
            npm ci
          fi

      # An unfortunate side-effect of the way reusable workflows work is that by the time they are pulled into the "caller"
      # repo, they are effectively completely embedded in that context. This means that we cannot reference any files which
      # are local to this repo which defines the workflow, and we therefore need to work around this by embedding the contents
      # of the shell utilities for executing commands into the workflow directly.
      - name: Create command utils
        uses: actions/github-script@v7
        with:
          script: |
            const { writeFileSync } = require('fs');
            const runCommandsInParallelScript = `
              # Extract the provided commands from the stringified JSON array.
              IFS=$'\n' read -d '' -a userCommands < <((jq -c -r '.[]') <<<"$1")

              # Invoke the provided commands in parallel and collect their exit codes.
              pids=()
              for userCommand in "\${userCommands[@]}"; do
                eval "$userCommand" & pids+=($!)
              done

              # If any one of the invoked commands exited with a non-zero exit code, exit the whole thing with code 1.
              for pid in \${pids[*]}; do
                if ! wait $pid; then
                  exit 1
                fi
              done

              # All the invoked commands must have exited with code zero.
              exit 0
            `;
            writeFileSync('./.github/workflows/run-commands-in-parallel.sh', runCommandsInParallelScript);

      - name: Prepare command utils
        # We need to escape the workspace path to be consistent cross-platform: https://github.com/actions/runner/issues/1066
        run: chmod +x ${GITHUB_WORKSPACE//\\//}/.github/workflows/run-commands-in-parallel.sh

      - name: Initialize the Nx Cloud distributed CI run
        run: npx nx-cloud start-ci-run

      # The good thing about the multi-line string input for sequential commands is that we can simply forward it on as is to the bash shell and it will behave
      # how we want it to in terms of quote escaping, variable assignment etc
      - name: Run any configured init-commands sequentially
        if: ${{ inputs.init-commands != '' }}
        run: |
          ${{ inputs.init-commands }}

      - name: Process parallel commands configuration
        uses: actions/github-script@v7
        id: parallel_commands_config
        env:
          PARALLEL_COMMANDS: ${{ inputs.parallel-commands }}
          PARALLEL_COMMANDS_ON_AGENTS: ${{ inputs.parallel-commands-on-agents }}
        with:
          # For the ones configured for main, explicitly set NX_CLOUD_DISTRIBUTED_EXECUTION to false, taking into account commands chained with &&
          # within the strings. In order to properly escape single quotes we need to do some manual replacing and escaping so that the commands
          # are forwarded onto the run-commands-in-parallel.sh script appropriately.
          script: |
            const parallelCommandsOnMainStr = process.env.PARALLEL_COMMANDS || '';
            const parallelCommandsOnAgentsStr = process.env.PARALLEL_COMMANDS_ON_AGENTS || '';

            const parallelCommandsOnMain = parallelCommandsOnMainStr
              .split('\n')
              .map(command => command.trim())
              .filter(command => command.length > 0)
              .map(s => s.replace(/'/g, '%27'));
            const parallelCommandsOnAgents = parallelCommandsOnAgentsStr
              .split('\n')
              .map(command => command.trim())
              .filter(command => command.length > 0)
              .map(s => s.replace(/'/g, '%27'));

            const formattedArrayOfCommands = [
              ...parallelCommandsOnMain.map(s => s
                .split(' && ')
                .map(s => `NX_CLOUD_DISTRIBUTED_EXECUTION=false ${s}`)
                .join(' && ')
              ),
              ...parallelCommandsOnAgents,
            ];

            const stringifiedEncodedArrayOfCommands = JSON.stringify(formattedArrayOfCommands)
              .replace(/%27/g, "'\\''");

            return stringifiedEncodedArrayOfCommands
          result-encoding: string

      - name: Run any configured parallel commands on main and agent jobs
        # We need to escape the workspace path to be consistent cross-platform: https://github.com/actions/runner/issues/1066
        run: ${GITHUB_WORKSPACE//\\//}/.github/workflows/run-commands-in-parallel.sh '${{ steps.parallel_commands_config.outputs.result }}'

      - name: Stop all running agents for this CI run
        # It's important that we always run this step, otherwise in the case of any failures in preceding non-Nx steps, the agents will keep running and waste billable minutes
        if: ${{ always() }}
        run: npx nx-cloud stop-all-agents

      # The good thing about the multi-line string input for sequential commands is that we can simply forward it on as is to the bash shell and it will behave
      # how we want it to in terms of quote escaping, variable assignment etc
      - name: Run any configured final-commands sequentially
        if: ${{ inputs.final-commands != '' }}
        run: |
          ${{ inputs.final-commands }}

      - name: Uploading artifacts
        uses: actions/upload-artifact@v4
        if: ${{ inputs.artifacts-path != '' }}
        with:
          name: ${{ inputs.artifacts-name }}
          path: ${{ inputs.artifacts-path }}
          retention-days: ${{ inputs.artifacts-retention-days }}