From f2b12469f33f19e6dd3f7e0cfbc1a859ff87be5b Mon Sep 17 00:00:00 2001 From: Audric Ackermann Date: Mon, 23 Sep 2024 17:14:21 +1000 Subject: [PATCH] chore: improve CI workflow speed and usage --- .github/workflows/build-binaries.yml | 138 +++++++++++++++---- actions/build_no_publish/action.yml | 64 --------- actions/build_publish/action.yml | 29 ---- actions/setup_and_build/action.yml | 33 +++-- actions/upload_prod_artefacts/action.yml | 27 ++++ build/updateLocalConfig.js | 2 +- package.json | 23 +--- ts/test/session/unit/updater/updater_test.ts | 7 +- yarn.lock | 22 --- 9 files changed, 173 insertions(+), 172 deletions(-) delete mode 100644 actions/build_no_publish/action.yml delete mode 100644 actions/build_publish/action.yml create mode 100644 actions/upload_prod_artefacts/action.yml diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index 2e11d98cb3..ceccef8109 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -1,3 +1,7 @@ +# There is a fair bit of duplication here, but it is the best to save our github free minutes for now. +# We could save and restore cache to different jobs but that takes roughly 3 minutes to save, +# so better run them in parrallel instead. + name: Session Desktop on: @@ -18,55 +22,141 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +env: + # we want to publish on "push to master" only. When we don't want to publish, we want to upload artefacts + SHOULD_PUBLISH: ${{ github.event_name == 'push' && github.ref == 'master' }} + jobs: - build: - runs-on: ${{ matrix.os }} + build_linux: + runs-on: ubuntu-20.04 strategy: - fail-fast: false matrix: - # We want a mac arm64 build, and according to this https://github.com/actions/runner-images#available-images macos-14 is always arm64 - # macos-14 is disabled for now as we hit our free tier limit for macos builds - os: [windows-2022, ubuntu-20.04, macos-12] + # this needs to be a valid target of https://www.electron.build/linux#target + pkg_to_build: ['deb', 'rpm', 'freebsd', 'AppImage'] + env: - SIGNAL_ENV: production GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - run: git config --global core.autocrlf false - name: Checkout git repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup & Build uses: ./actions/setup_and_build + with: + cache_suffix: ${{ matrix.pkg_to_build }} - name: Lint Files - # no need to lint files on all platforms. Just do it once on the quicker one - if: runner.os == 'Linux' + # no need to lint files on all platforms run: yarn lint-full - name: Enforce yarn.lock has no duplicates + # no need to dedup yarn.lock on all platforms uses: ./actions/deduplicate_fail + + # we want to test on all platforms are some are testing the menus rendered (and are depent on the platform) + - name: Unit Test + run: yarn test + + - name: Build but do not publish ${{ matrix.pkg_to_build }} + # we want this to run always, except on "push" to "master" + if: ${{ env.SHOULD_PUBLISH == 'false' }} + run: | + sed -i 's/"target": \["deb"\]/"target": "${{ matrix.pkg_to_build }}"/g' package.json && yarn build-release + + - name: Upload artefacts ${{ matrix.pkg_to_build }} + # we want this to run always, except on "push" to "master" + if: ${{ env.SHOULD_PUBLISH == 'false' }} + uses: ./actions/upload_prod_artefacts + with: + upload_prefix: ${{ runner.os }}-${{ runner.arch }}-${{ matrix.pkg_to_build }} + + - name: Build & publish ${{ matrix.pkg_to_build }} + # we want this to run only when on "push" to "master" + if: ${{ env.SHOULD_PUBLISH == 'true' }} + run: | + sed -i 's/\"target\": \\[\"deb\"\\]/\"target\": \"${{ matrix.pkg_to_build }}\"/g' package.json; yarn build-release-publish + + build_windows: + runs-on: windows-2022 + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - run: git config --global core.autocrlf false + + - name: Checkout git repo + uses: actions/checkout@v4 + + - name: Setup & Build + uses: ./actions/setup_and_build + with: + cache_suffix: 'windows_x64' + + # we want to test on all platforms are some are testing the menus rendered (and are depent on the platform) - name: Unit Test run: yarn test - name: Build but do not publish # we want this to run always, except on "push" to "master" - if: github.event_name != 'push' || github.ref != 'master' - uses: ./actions/build_no_publish + if: ${{ env.SHOULD_PUBLISH == 'false' }} + run: yarn build-release + + - name: Upload artefacts + # we want this to run always, except on "push" to "master" + if: ${{ env.SHOULD_PUBLISH == 'false' }} + uses: ./actions/upload_prod_artefacts with: - MAC_CERTIFICATE: ${{ secrets.MAC_CERTIFICATE }} - MAC_CERTIFICATE_PASSWORD: ${{ secrets.MAC_CERTIFICATE_PASSWORD }} - SIGNING_APPLE_ID: ${{ secrets.SIGNING_APPLE_ID }} - SIGNING_APP_PASSWORD: ${{ secrets.SIGNING_APP_PASSWORD }} - SIGNING_TEAM_ID: ${{ secrets.SIGNING_TEAM_ID }} + upload_prefix: ${{ runner.os }}-${{ runner.arch }} - name: Build & publish # we want this to run only when on "push" to "master" - if: github.event_name == 'push' && github.ref == 'master' - uses: ./actions/build_publish + if: ${{ env.SHOULD_PUBLISH == 'true' }} + run: yarn build-release-publish # No other args needed for windows publish + + # We want a mac arm64 build, and according to this https://github.com/actions/runner-images#available-images macos-14 is always arm64 + # macos-14 is disabled for now as we hit our free tier limit for macos builds + build_macos_x64: + runs-on: macos-12 + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + MAC_CERTIFICATE: ${{ secrets.MAC_CERTIFICATE }} + MAC_CERTIFICATE_PASSWORD: ${{ secrets.MAC_CERTIFICATE_PASSWORD }} + SIGNING_APPLE_ID: ${{ secrets.SIGNING_APPLE_ID }} + SIGNING_APP_PASSWORD: ${{ secrets.SIGNING_APP_PASSWORD }} + SIGNING_TEAM_ID: ${{ secrets.SIGNING_TEAM_ID }} + steps: + - run: git config --global core.autocrlf false + + - name: Checkout git repo + uses: actions/checkout@v4 + + - name: Setup & Build + uses: ./actions/setup_and_build with: - MAC_CERTIFICATE: ${{ secrets.MAC_CERTIFICATE }} - MAC_CERTIFICATE_PASSWORD: ${{ secrets.MAC_CERTIFICATE_PASSWORD }} - SIGNING_APPLE_ID: ${{ secrets.SIGNING_APPLE_ID }} - SIGNING_APP_PASSWORD: ${{ secrets.SIGNING_APP_PASSWORD }} - SIGNING_TEAM_ID: ${{ secrets.SIGNING_TEAM_ID }} + cache_suffix: 'macos_x64' + + # we want to test on all platforms are some are testing the menus rendered (and are depent on the platform) + - name: Unit Test + run: yarn test + + - name: Build but do not publish + # we want this to run always, except on "push" to "master" + if: ${{ env.SHOULD_PUBLISH == 'false' }} + run: | + source ./build/setup-mac-certificate.sh + yarn build-release --config.mac.bundleVersion=${{ github.ref }} + + - name: Upload artefacts + # we want this to run always, except on "push" to "master" + if: ${{ env.SHOULD_PUBLISH == 'false' }} + uses: ./actions/upload_prod_artefacts + with: + upload_prefix: ${{ runner.os }}-${{ runner.arch }} + + - name: Build & publish + # we want this to run only when on "push" to "master" + if: ${{ env.SHOULD_PUBLISH == 'true' }} + run: | + source ./build/setup-mac-certificate.sh + yarn build-release-publish --config.mac.bundleVersion=${{ github.ref }} diff --git a/actions/build_no_publish/action.yml b/actions/build_no_publish/action.yml deleted file mode 100644 index fc2c23c832..0000000000 --- a/actions/build_no_publish/action.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: 'Build Only (no publish)' -description: 'Build only (no publish)' -inputs: - MAC_CERTIFICATE: - description: 'MAC_CERTIFICATE (mac build only)' - required: true - MAC_CERTIFICATE_PASSWORD: - description: 'MAC_CERTIFICATE_PASSWORD (mac build only)' - required: true - SIGNING_APPLE_ID: - description: 'SIGNING_APPLE_ID (mac build only)' - required: true - SIGNING_APP_PASSWORD: - description: 'SIGNING_APP_PASSWORD (mac build only)' - required: true - SIGNING_TEAM_ID: - description: 'SIGNING_TEAM_ID (mac build only)' - required: true - -runs: - using: 'composite' - steps: - - name: Build windows production binaries - shell: bash - if: runner.os == 'Windows' - run: $(yarn bin)/electron-builder --config.extraMetadata.environment=%SIGNAL_ENV% --publish=never --config.directories.output=release - - - name: Build mac production binaries - shell: bash - if: runner.os == 'macOS' - run: | - source ./build/setup-mac-certificate.sh - $(yarn bin)/electron-builder --config.extraMetadata.environment=$SIGNAL_ENV --config.mac.bundleVersion=${{ github.ref }} --publish=never --config.directories.output=release - env: - MAC_CERTIFICATE: ${{ inputs.MAC_CERTIFICATE }} - MAC_CERTIFICATE_PASSWORD: ${{ inputs.MAC_CERTIFICATE_PASSWORD }} - SIGNING_APPLE_ID: ${{ inputs.SIGNING_APPLE_ID }} - SIGNING_APP_PASSWORD: ${{ inputs.SIGNING_APP_PASSWORD }} - SIGNING_TEAM_ID: ${{ inputs.SIGNING_TEAM_ID }} - - - name: Build linux production binaries - shell: bash - if: runner.os == 'Linux' - run: | - sudo apt-get install -y rpm - yarn build-release - - - name: Remove unpacked files - run: | - ls -d -- */ | xargs -I{} echo "Removing {}" - ls -d -- */ | xargs -I{} rm -rf {} - shell: bash - working-directory: ./release/ - - - name: Remaining files - run: ls . - shell: bash - working-directory: ./release/ - - - name: Upload Production Artifacts - uses: actions/upload-artifact@v4 - with: - name: ${{ runner.os }}-${{ runner.arch }}-production - path: release diff --git a/actions/build_publish/action.yml b/actions/build_publish/action.yml deleted file mode 100644 index 607d1bedc4..0000000000 --- a/actions/build_publish/action.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: 'Build & Publish' -description: 'Build & Publish' -runs: - using: 'composite' - steps: - - name: Build & publish windows production binaries - shell: bash - if: runner.os == 'Windows' && github.ref == 'master' && github.event_name == 'push' - run: $(yarn bin)/electron-builder --config.extraMetadata.environment=%SIGNAL_ENV% --publish=always - - - name: Build & publish mac production binaries - shell: bash - if: runner.os == 'macOS' && github.ref == 'master' && github.event_name == 'push' - run: | - source ./build/setup-mac-certificate.sh - $(yarn bin)/electron-builder --config.extraMetadata.environment=$SIGNAL_ENV --config.mac.bundleVersion=${{ github.ref }} --publish=always - env: - MAC_CERTIFICATE: ${{ secrets.MAC_CERTIFICATE }} - MAC_CERTIFICATE_PASSWORD: ${{ secrets.MAC_CERTIFICATE_PASSWORD }} - SIGNING_APPLE_ID: ${{ secrets.SIGNING_APPLE_ID }} - SIGNING_APP_PASSWORD: ${{ secrets.SIGNING_APP_PASSWORD }} - SIGNING_TEAM_ID: ${{ secrets.SIGNING_TEAM_ID }} - - - name: Build & publish linux production binaries - shell: bash - if: runner.os == 'Linux' && github.ref == 'master' && github.event_name == 'push' - run: | - sudo apt-get install -y rpm - yarn build-release-publish diff --git a/actions/setup_and_build/action.yml b/actions/setup_and_build/action.yml index 21e63ef33f..5c51a51464 100644 --- a/actions/setup_and_build/action.yml +++ b/actions/setup_and_build/action.yml @@ -1,29 +1,25 @@ - name: 'Setup and build' description: 'Setup and build Session Desktop' +inputs: + cache_suffix: + description: 'the package we are currently building (used as key for the cached node_modules)' + required: true + runs: using: 'composite' steps: - name: Install node - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version-file: '.nvmrc' - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.11' - - name: Cache Desktop node_modules - id: cache-desktop-modules - uses: actions/cache@v3 - if: runner.os != 'Windows' - with: - path: node_modules - key: ${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('package.json', 'yarn.lock', 'patches/**') }} - # Not having this will break the windows build because the PATH won't be set by msbuild. - name: Add msbuild to PATH - uses: microsoft/setup-msbuild@v1.3.1 + uses: microsoft/setup-msbuild@v2 if: runner.os == 'Windows' - name: Setup node for windows @@ -32,11 +28,24 @@ runs: run: | yarn global add node-gyp@latest + - uses: actions/cache/restore@v4 + id: cache-desktop-modules + with: + path: node_modules + key: ${{ runner.os }}-${{ runner.arch }}-${{ inputs.cache_suffix }}-${{ hashFiles('package.json', 'yarn.lock', 'patches/**') }} + - name: Install dependencies shell: bash if: steps.cache-desktop-modules.outputs.cache-hit != 'true' run: yarn install --frozen-lockfile --network-timeout 600000 + - uses: actions/cache/save@v4 + id: cache-desktop-modules-save + if: runner.os != 'Windows' + with: + path: node_modules + key: ${{ runner.os }}-${{ runner.arch }}-${{ inputs.cache_suffix }}-${{ hashFiles('package.json', 'yarn.lock', 'patches/**') }} + - name: Generate and concat files shell: bash run: yarn build-everything diff --git a/actions/upload_prod_artefacts/action.yml b/actions/upload_prod_artefacts/action.yml new file mode 100644 index 0000000000..2e9542f925 --- /dev/null +++ b/actions/upload_prod_artefacts/action.yml @@ -0,0 +1,27 @@ +name: 'Upload production artefact (not publish)' +description: 'Upload production artefact (not publish)' +inputs: + upload_prefix: + description: 'upload name prefix' + required: true + +runs: + using: 'composite' + steps: + - name: Remove unpacked files + run: | + ls -d -- */ | xargs -I{} echo "Removing {}" + ls -d -- */ | xargs -I{} rm -rf {} + shell: bash + working-directory: ./release/ + + - name: Remaining files + run: ls . + shell: bash + working-directory: ./release/ + + - name: Upload Production Artifacts + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.upload_prefix }}-production + path: release diff --git a/build/updateLocalConfig.js b/build/updateLocalConfig.js index 7fbbe4df1e..3691a54cf0 100644 --- a/build/updateLocalConfig.js +++ b/build/updateLocalConfig.js @@ -3,7 +3,7 @@ var _ = require('lodash'); var execSync = require('child_process').execSync; const updateLocalConfig = () => { - var environment = process.env.SIGNAL_ENV || 'production'; + var environment = 'production'; var configPath = `config/local-${environment}.json`; var localConfig; diff --git a/package.json b/package.json index eb923bb2fa..47edaa86aa 100644 --- a/package.json +++ b/package.json @@ -49,19 +49,9 @@ "format-full": "prettier --list-different --write \"*.{css,js,json,scss,ts,tsx}\" \"./**/*.{css,js,json,scss,ts,tsx}\"", "start-prod-test": "cross-env NODE_ENV=production NODE_APP_INSTANCE=$MULTI electron .", "test": "mocha", - "build-release": "run-script-os", - "build-release-non-linux": "yarn build-everything && cross-env SIGNAL_ENV=production electron-builder --config.extraMetadata.environment=production --publish=never --config.directories.output=release", - "build-release:win32": "yarn build-release-non-linux", - "build-release:macos": "yarn build-release-non-linux", - "build-release:linux": "yarn sedtoDeb; yarn build-release-non-linux && yarn sedtoAppImage && yarn build-release-non-linux && yarn sedtoDeb", - "build-release-publish": "run-script-os", - "build-release-publish-non-linux": "yarn build-everything && cross-env SIGNAL_ENV=production $(yarn bin)/electron-builder --config.extraMetadata.environment=$SIGNAL_ENV --publish=always", - "build-release-publish:win32": "yarn build-release-publish-non-linux", - "build-release-publish:macos": "yarn build-release-publish-non-linux", - "build-release-publish:linux": "yarn sedtoDeb; yarn build-release-publish-non-linux && yarn sedtoAppImage && yarn build-release-publish-non-linux && yarn sedtoDeb", - "appImage": "yarn sedtoAppImage; yarn build-release-non-linux; yarn sedtoDeb", - "sedtoAppImage": "sed -i 's/\"target\": \\[\"deb\", \"rpm\", \"freebsd\"\\]/\"target\": \"AppImage\"/g' package.json", - "sedtoDeb": "sed -i 's/\"target\": \"AppImage\"/\"target\": \\[\"deb\", \"rpm\", \"freebsd\"\\]/g' package.json", + "build-release-base": "cross-env NODE_ENV=production electron-builder --config.extraMetadata.environment=production", + "build-release": "yarn build-release-base --publish=never --config.directories.output=release", + "build-release-publish": "yarn build-release-base --publish=always", "ready": "yarn dedup-yarn-lock --fail && yarn build-everything && yarn lint-full && yarn test", "postinstall": "yarn patch-package && yarn electron-builder install-app-deps", "update-git-info": "node ./build/updateLocalConfig.js", @@ -207,7 +197,6 @@ "postinstall-prepare": "^1.0.1", "prettier": "3.2.5", "protobufjs-cli": "^1.1.1", - "run-script-os": "^1.1.6", "sass": "^1.60.0", "sass-loader": "^13.2.2", "sinon": "9.0.2", @@ -272,7 +261,7 @@ "StartupWMClass": "Session" }, "asarUnpack": "node_modules/spellchecker/vendor/hunspell_dictionaries", - "target": ["deb", "rpm", "freebsd"], + "target": ["deb"], "icon": "build/icon-linux.icns" }, "asarUnpack": [ @@ -294,8 +283,8 @@ "files": [ "package.json", "config/default.json", - "config/${env.SIGNAL_ENV}.json", - "config/local-${env.SIGNAL_ENV}.json", + "config/production.json", + "config/local-production.json", "background.html", "about.html", "password.html", diff --git a/ts/test/session/unit/updater/updater_test.ts b/ts/test/session/unit/updater/updater_test.ts index 887faec20d..c6880182dd 100644 --- a/ts/test/session/unit/updater/updater_test.ts +++ b/ts/test/session/unit/updater/updater_test.ts @@ -6,14 +6,15 @@ import { isEmpty } from 'lodash'; describe('Updater', () => { it.skip('isUpdateAvailable', () => {}); - it('package.json target are correct', () => { + it('package.json target is correct', () => { const content = readFileSync( path.join(__dirname, '..', '..', '..', '..', '..', 'package.json') ); - if (!content || isEmpty(content) || !content.includes('"target": ["deb", "rpm", "freebsd"],')) { + // the CI for building release relies on this being set to build the different targets. + if (!content || isEmpty(content) || !content.includes('"target": ["deb"],')) { throw new Error( - 'Content empty or does not contain the target on a single line. They have to be for the linux appImage build to pass.' + 'Content empty or does not contain the target on a single line. They have to be for the linux CI builds to pass.' ); } }); diff --git a/yarn.lock b/yarn.lock index 28c77acd2a..ea585cedce 100644 --- a/yarn.lock +++ b/yarn.lock @@ -765,13 +765,6 @@ resolved "https://registry.yarnpkg.com/@types/filesize/-/filesize-3.6.0.tgz#5f1a25c7b4e3d5ee2bc63133d374d096b7008c8d" integrity sha512-rOWxCKMjt2DBuwddUnl5GOpf/jAkkqteB+XldncpVxVX+HPTmK2c5ACMOVEbp9gaH81IlhTdC3TwvRa5nopasw== -"@types/firstline@^2.0.2": - version "2.0.4" - resolved "https://registry.yarnpkg.com/@types/firstline/-/firstline-2.0.4.tgz#b8d3f8f7396d1589efea89db183c047a42efaf04" - integrity sha512-EYoMzk783ncj3soLGADXD/rklDMw1PAO5Hc3lRZa5G21vkfacwkdTlIdhTJ39omqDLezTSmxjDG1psd4A/mUHg== - dependencies: - "@types/node" "*" - "@types/fs-extra@5.0.5": version "5.0.5" resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-5.0.5.tgz#080d90a792f3fa2c5559eb44bd8ef840aae9104b" @@ -3674,11 +3667,6 @@ find-yarn-workspace-root@^2.0.0: dependencies: micromatch "^4.0.2" -firstline@1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/firstline/-/firstline-1.2.1.tgz#b88673c42009f8821fac2926e99720acee924fae" - integrity sha512-6eMQNJtDzyXSC1yeCBWspqA6LeV5la2XHGTXQq4O0xkglAutpyny/sB+zVdXTZ9nzcDW9ZGLxwXXkB+ZEtJuPw== - flat-cache@^3.0.4: version "3.2.0" resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-3.2.0.tgz#2c0c2d5040c99b1632771a9d105725c0115363ee" @@ -6381,11 +6369,6 @@ read-config-file@6.2.0: json5 "^2.2.0" lazy-val "^1.0.4" -read-last-lines-ts@^1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/read-last-lines-ts/-/read-last-lines-ts-1.2.1.tgz#99e46288c5373c06e16e90e666a46b595dad80a1" - integrity sha512-1VcCrAU38DILYiF4sbNY13zdrMGwrFqjGQnXJy28G1zLJItvnWtgCbqoAJlnZZSiEICMKdM4Ol7LYvVMEoKrAg== - read-pkg-up@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/read-pkg-up/-/read-pkg-up-7.0.1.tgz#f3a6135758459733ae2b95638056e1854e7ef507" @@ -6695,11 +6678,6 @@ run-parallel@^1.1.9: dependencies: queue-microtask "^1.2.2" -run-script-os@^1.1.6: - version "1.1.6" - resolved "https://registry.yarnpkg.com/run-script-os/-/run-script-os-1.1.6.tgz#8b0177fb1b54c99a670f95c7fdc54f18b9c72347" - integrity sha512-ql6P2LzhBTTDfzKts+Qo4H94VUKpxKDFz6QxxwaUZN0mwvi7L3lpOI7BqPCq7lgDh3XLl0dpeXwfcVIitlrYrw== - safe-array-concat@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/safe-array-concat/-/safe-array-concat-1.1.2.tgz#81d77ee0c4e8b863635227c721278dd524c20edb"