From b97c2ebd3e52db61cb283a7a56c96a4fdf8f58e4 Mon Sep 17 00:00:00 2001 From: Pablo Date: Mon, 8 Jul 2024 10:01:31 -0600 Subject: [PATCH 1/8] feat: add support for S3 caching --- actions/setup-nix/action.yml | 42 +++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/actions/setup-nix/action.yml b/actions/setup-nix/action.yml index b3a81eba..99d0d309 100644 --- a/actions/setup-nix/action.yml +++ b/actions/setup-nix/action.yml @@ -4,7 +4,7 @@ description: "Setup nix with caches" inputs: # custom cache inputs ---------------------------------- cache-url: - description: "Custom nix cache URL" + description: "Custom nix cache URL, for S3 use 's3://' " required: false default: "" cache-pubkey: @@ -50,6 +50,12 @@ inputs: "grafana org/tenant id - required if metrics-job-name is passed" required: false + # private signing key (optional) ---------------------- + private-signing-key: + description: "Private signing key for Nix cache" + required: false + default: "" + runs: using: composite steps: @@ -88,6 +94,38 @@ runs: # nix build is run via daemon on root, credentials linked for daemon access sudo ln -s ~/.aws $ROOT_PATH + - name: store private signing key + if: ${{ inputs.private-signing-key != '' }} + shell: bash + run: | + echo "${{ inputs.private-signing-key }}" > ~/.nix-signing-key + chmod 600 ~/.nix-signing-key + + - name: set NIX_SECRET_KEY_FILES + id: set-nix-secret-key-files + shell: bash + run: | + if [ '${{ inputs.private-signing-key }}' != '' ]; then + echo "NIX_SECRET_KEY_FILES=~/.nix-signing-key" >> $GITHUB_ENV + else + echo "NIX_SECRET_KEY_FILES=" >> $GITHUB_ENV + fi + + - name: save upload-to-cache.sh + shell: bash + if: ${{ inputs.cache-url != '' }} + run: | + cat << EOF > ~/upload-to-cache.sh + #!/bin/sh + set -eu + set -f # disable globbing + export IFS=' ' + echo "Uploading paths" \$OUT_PATHS + exec nix copy --to "${{ inputs.cache-url }}?scheme=https®ion=${{ inputs.aws-region }}" \$OUT_PATHS + EOF + echo "NIX_POST_BUILD_HOOK_FILE=~/upload-to-cache.sh" >> $GITHUB_ENV + chmod +x ~/upload-to-cache.sh + - uses: cachix/install-nix-action@ba0dd844c9180cbf77aa72a116d6fbc515d0e87b # v27 with: install_url: https://releases.nixos.org/nix/nix-2.18.1/install @@ -98,6 +136,8 @@ runs: sandbox = relaxed substituters = https://cache.nixos.org https://nix-community.cachix.org ${{ inputs.cache-url }} trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs= ${{ inputs.cache-pubkey }} + secret-key-files = ${{ env.NIX_SECRET_KEY_FILES }} + post-build-hook = ${{ env.NIX_POST_BUILD_HOOK_FILE }} - name: Collect metrics if: always() && inputs.metrics-job-name != '' From 461377bbc6d317e857f1d201ad8586622acc8a3b Mon Sep 17 00:00:00 2001 From: Pablo Date: Mon, 8 Jul 2024 12:13:56 -0600 Subject: [PATCH 2/8] feat: add aws credentials as input params --- actions/setup-nix/action.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/actions/setup-nix/action.yml b/actions/setup-nix/action.yml index 99d0d309..c5f327e9 100644 --- a/actions/setup-nix/action.yml +++ b/actions/setup-nix/action.yml @@ -21,6 +21,15 @@ inputs: description: "" required: false default: "us-west-1" + aws-access-key-id: + description: "The AWS_ACCESS_KEY_ID for S3 authentication. Will need to have read/write access to the bucket." + required: false + aws-secret-access-key: + description: "The AWS_SECRET_ACCESS_KEY for S3 authentication. Will need to have read/write access to the bucket." + required: false + aws-session-token: + description: "The AWS_SESSION_TOKEN for S3 authentication. Will need to have read/write access to the bucket." + required: false role-to-assume: description: "" required: false @@ -71,6 +80,10 @@ runs: - name: setup AWS credentials for nix if: inputs.enable-aws == 'true' shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ inputs.aws-session-token }} run: | # set up folder with AWS configs # nix reads AWS access from folder config - does not work with env vars From 435e724c671024b566eb7022d09a8a51c7c784f3 Mon Sep 17 00:00:00 2001 From: Pablo Date: Mon, 8 Jul 2024 12:49:50 -0600 Subject: [PATCH 3/8] feat: nix signing key name changeset fix: path feat: remove secret key file fix: paths fix: paths fix: env var syntax fix: nix path not found fix: add nix path fix: add nix path fix: separate build path fix: restart nix daemon to make config take effect fix: substituters fix: avoid removing secret as subsequent calls use it. fix: use oidc auth via role assumption feat: add support for read only mode, update docs. feat: move sh script out of the action feat: adjust paths feat: adjust to home runner feat: user gh workspace feat: user gh workspace feat: copy script to /etc/nix feat: use determinate systems installer feat: fix paths for script post build hook feat: fix paths for script post build hook feat: fix paths for script post build hook feat: fix paths for script post build hook feat: add debug logs fix: use path outside nix sandbox fix: use path outside nix sandbox fix: use path outside nix sandbox fix: use path outside nix sandbox fix: use path outside nix sandbox fix: use path outside nix sandbox fix: debug remove params fix: replace values on script rather than passing params as nix config does not support params fix:remove channels and nix env upgrade fix: debug logs fix: remove source check of nix daemon fix: change nix run path fix: change nix run path fix: change nix run path fix: debug nix config fix: debug nix config fix: remove debug logs fix: test setting env variables in action fix: cache url setting fix: permissions fix: copy script with env vars hardcoded as originally implemented fix: copy script with env vars hardcoded as originally implemented fix: revert script fix: separate into 2 actions fix: version of aws config fix: set aws env vars feat: require singing key when on push mode feat: update docs --- .changeset/dull-pugs-wink.md | 5 + actions/setup-nix-cache/CHANGELOG.md | 1 + actions/setup-nix-cache/README.md | 83 +++++++++ actions/setup-nix-cache/action.yml | 174 ++++++++++++++++++ actions/setup-nix-cache/package.json | 11 ++ actions/setup-nix-cache/project.json | 7 + .../scripts/upload-to-cache.sh | 26 +++ actions/setup-nix/README.md | 59 +++--- actions/setup-nix/action.yml | 170 +++-------------- 9 files changed, 364 insertions(+), 172 deletions(-) create mode 100644 .changeset/dull-pugs-wink.md create mode 100644 actions/setup-nix-cache/CHANGELOG.md create mode 100644 actions/setup-nix-cache/README.md create mode 100644 actions/setup-nix-cache/action.yml create mode 100644 actions/setup-nix-cache/package.json create mode 100644 actions/setup-nix-cache/project.json create mode 100644 actions/setup-nix-cache/scripts/upload-to-cache.sh diff --git a/.changeset/dull-pugs-wink.md b/.changeset/dull-pugs-wink.md new file mode 100644 index 00000000..1376c98b --- /dev/null +++ b/.changeset/dull-pugs-wink.md @@ -0,0 +1,5 @@ +--- +"setup-nix": minor +--- + +Adds support for reading and pushing Artifacts to the S3 cache. diff --git a/actions/setup-nix-cache/CHANGELOG.md b/actions/setup-nix-cache/CHANGELOG.md new file mode 100644 index 00000000..51a15117 --- /dev/null +++ b/actions/setup-nix-cache/CHANGELOG.md @@ -0,0 +1 @@ +# setup-nix-cache diff --git a/actions/setup-nix-cache/README.md b/actions/setup-nix-cache/README.md new file mode 100644 index 00000000..8f8df209 --- /dev/null +++ b/actions/setup-nix-cache/README.md @@ -0,0 +1,83 @@ +# Setup Nix S3 Cache + +`setup-nix-cache` configures your Nix env with the specified caches. Support +access to the internal S3 Nix cache in read-only mode and read/write mode. + +## Configuration + +## Inputs + +| Name | Description | Required | Default | +| ----------------------- | -------------------------------------------------------------------------------------------------- | -------- | ----------- | +| `cache-url` | Custom Nix cache URL, for example, `'s3://'` for an S3-backed cache. | Yes | N/A | +| `cache-pubkeys` | Comma-separated list of custom Nix cache public keys. | Yes | N/A | +| `cache-mode` | Specify either `'read'` for read-only access or `'push'` for uploading build results to the cache. | Yes | `read` | +| `aws-region` | AWS region for the S3 bucket (only applicable if `enable-aws` is true). | No | `us-west-1` | +| `role-to-assume` | AWS role to assume for cache access (optional). | No | `""` | +| `role-duration-seconds` | Duration in seconds for assuming the role (default: 1 hour). | No | `3600` | +| `private-signing-key` | Private key for signing nix artifacts to push. | No | `3600` | + +## Usage + +### Basic Example (Read-only Mode) + +```yaml +name: Setup Nix Cache Example + +on: + push: + branches: + - main + +jobs: + setup-nix-cache: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Install Nix + uses: smartcontractkit/.github/actions/setup-nix@7a7de5813c702b2e9d042903a1e9cffd2c0b40c5 + + - name: Setup Nix S3 cache + uses: smartcontractkit/.github/actions/setup-nix-cache@7a7de5813c702b2e9d042903a1e9cffd2c0b40c5 + with: + cache-url: "s3://bucket-name" + cache-pubkeys: ${{ secrets.***REMOVED*** }} + cache-mode: "read" + aws-region: "us-west-1" + role-to-assume: "arn role" + role-duration-seconds: 3600 +``` + +## Write Mode (Uploads Built Artifacts to the Cache) + +```yaml +name: Setup Nix Cache and Upload Example with All Parameters + +on: + push: + branches: + - main + +jobs: + setup-nix-cache: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Install Nix + uses: smartcontractkit/.github/actions/setup-nix@7a7de5813c702b2e9d042903a1e9cffd2c0b40c5 + + - name: Setup Nix S3 cache + uses: smartcontractkit/.github/actions/setup-nix-cache@7a7de5813c702b2e9d042903a1e9cffd2c0b40c5 + with: + cache-url: "s3://bucket-name" + cache-pubkeys: ${{ secrets.***REMOVED*** }} + cache-mode: "read" + aws-region: "us-west-1" + role-to-assume: "arn for role" + role-duration-seconds: 3600 + private-signing-key: ${{ secrets.SOME_NIX_PRIVATE_KEY_FOR_SIGNING }} +``` diff --git a/actions/setup-nix-cache/action.yml b/actions/setup-nix-cache/action.yml new file mode 100644 index 00000000..ac709ba6 --- /dev/null +++ b/actions/setup-nix-cache/action.yml @@ -0,0 +1,174 @@ +name: setup-nix-cache +description: "Setup Nix cache with S3 support and optional signing" + +inputs: + cache-url: + description: "Custom Nix cache URL, e.g., for S3 use 's3://'" + required: true + + cache-pubkeys: + description: "Comma-separated list of custom Nix cache public keys" + required: true + + cache-mode: + description: + "Specify 'read' for read-only access to the cache or 'push' for uploading + build results" + required: true + default: "read" + + aws-region: + description: + "AWS region for the S3 bucket (required for accessing S3-backed caches)" + required: true + + role-to-assume: + description: "AWS role to assume for cache access (required)" + required: true + + role-duration-seconds: + description: "Duration in seconds for assuming the role (default: 1 hour)" + required: false + default: 3600 + + private-signing-key: + description: + "Private signing key for Nix cache (only required if pushing to cache)" + required: false + default: "" + +runs: + using: composite + steps: + # Step to configure AWS credentials for Nix cache + - name: Configure AWS credentials for Nix cache + uses: aws-actions/configure-aws-credentials@050bf7aae915badb82daa8e68fd95c9070706953 # v4.0.2 + with: + role-to-assume: ${{ inputs.role-to-assume }} + role-duration-seconds: ${{ inputs.role-duration-seconds }} + aws-region: ${{ inputs.aws-region }} + + # Step to set AWS environment variables and symlink the credentials + - name: 🛠️ Setup AWS credentials for Nix + shell: bash + run: | + echo "🔑 Setting up AWS credentials for Nix..." + aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID + aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY + aws configure set aws_session_token $AWS_SESSION_TOKEN + aws configure set region ${{ inputs.aws-region }} + + ROOT_PATH="" + if [[ "${{ runner.os }}" == "Linux" ]]; then + ROOT_PATH="/root" + elif [[ "${{ runner.os }}" == "macOS" ]]; then + ROOT_PATH="/var/root" + else + echo "⚠️ ${ { runner.os }} not supported!" + exit 1 + fi + + echo "🔗 Symlinking AWS credentials for Nix daemon..." + sudo ln -s ~/.aws $ROOT_PATH + + # Step to preprocess public keys + - name: Preprocess Public Keys + shell: bash + run: | + PUBLIC_KEYS=$(echo "${{ inputs.cache-pubkeys }}" | tr ',' ' ') + echo "PUBLIC_KEYS=$PUBLIC_KEYS" >> $GITHUB_ENV + + # Setup Cache with Nix + - name: Setup Nix Cache + shell: bash + run: | + # Define the substitutions and public keys strings + SUBSTITUTERS_LINE="substituters = https://cache.nixos.org ${{ inputs.cache-url }}?scheme=https®ion=${{ inputs.aws-region }}" + PUBLIC_KEYS_LINE="trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= ${{ env.PUBLIC_KEYS }}" + + # Replace or append the 'substituters' line + if grep -q "^substituters" /etc/nix/nix.conf; then + echo "🔄 Replacing existing 'substituters' line in /etc/nix/nix.conf" + sudo sed -i "s|^substituters.*|$SUBSTITUTERS_LINE|" /etc/nix/nix.conf + else + echo "➕ Appending 'substituters' line to /etc/nix/nix.conf" + echo "$SUBSTITUTERS_LINE" | sudo tee -a /etc/nix/nix.conf + fi + + # Replace or append the 'trusted-public-keys' line + if grep -q "^trusted-public-keys" /etc/nix/nix.conf; then + echo "🔄 Replacing existing 'trusted-public-keys' line in /etc/nix/nix.conf" + sudo sed -i "s|^trusted-public-keys.*|$PUBLIC_KEYS_LINE|" /etc/nix/nix.conf + else + echo "➕ Appending 'trusted-public-keys' line to /etc/nix/nix.conf" + echo "$PUBLIC_KEYS_LINE" | sudo tee -a /etc/nix/nix.conf + fi + + # Store private signing key if provided + - name: Store Private Signing Key for Nix cache signing + if: ${{ inputs.private-signing-key != '' }} + shell: bash + run: | + # Store the private signing key in a secure location + echo "${{ inputs.private-signing-key }}" > ${{ github.workspace }}/nix-signing.private + chmod 600 ${{ github.workspace }}/nix-signing.private + + # Define the secret-key-files line + SECRET_KEY_FILES_LINE="secret-key-files = ${{ github.workspace }}/nix-signing.private" + + # Replace or append the 'secret-key-files' line in /etc/nix/nix.conf + if grep -q "^secret-key-files" /etc/nix/nix.conf; then + echo "🔄 Replacing existing 'secret-key-files' line in /etc/nix/nix.conf" + sudo sed -i "s|^secret-key-files.*|$SECRET_KEY_FILES_LINE|" /etc/nix/nix.conf + else + echo "➕ Appending 'secret-key-files' line to /etc/nix/nix.conf" + echo "$SECRET_KEY_FILES_LINE" | sudo tee -a /etc/nix/nix.conf + fi + + # Set up post-build hook for cache upload if pushing + - name: Configure Post-build Hook for Cache Upload + if: ${{ inputs.cache-mode == 'push' }} + shell: bash + run: | + # Check if the private signing key is provided + if [[ -z "${{ inputs.private-signing-key }}" ]]; then + echo "❌ Error: Private signing key is required in 'push' mode." + exit 1 + fi + + echo "🚀 Enabling push mode: configuring post-build hook for cache upload" + + # Create a modified version of the script with the actual inputs for CACHE_URL and AWS_REGION + MODIFIED_SCRIPT_PATH="/tmp/upload-to-cache-modified.sh" + cp ${GITHUB_ACTION_PATH}/scripts/upload-to-cache.sh $MODIFIED_SCRIPT_PATH + + # Use sed to replace the placeholders in the script + sed -i "s|export CACHE_URL=\$1|export CACHE_URL=${{ inputs.cache-url }}|" $MODIFIED_SCRIPT_PATH + sed -i "s|export AWS_REGION=\$2|export AWS_REGION=${{ inputs.aws-region }}|" $MODIFIED_SCRIPT_PATH + + # Ensure the modified script has the right permissions + chmod 755 $MODIFIED_SCRIPT_PATH + + # Copy the modified script to a persistent location + PERSISTENT_SCRIPT_PATH="/etc/nix/upload-to-cache.sh" + sudo cp $MODIFIED_SCRIPT_PATH $PERSISTENT_SCRIPT_PATH + + # Check if the modified script exists and has correct permissions + if [ -f "$PERSISTENT_SCRIPT_PATH" ]; then + echo "✅ Modified script exists at $PERSISTENT_SCRIPT_PATH with the following permissions:" + ls -la $PERSISTENT_SCRIPT_PATH + else + echo "❌ Modified script does not exist at $PERSISTENT_SCRIPT_PATH" + exit 1 + fi + + # Set up the post-build hook to use the modified script + echo "Setting post-build-hook in /etc/nix/nix.conf" + echo "post-build-hook = $PERSISTENT_SCRIPT_PATH" | sudo tee -a /etc/nix/nix.conf + + # Restart Nix daemon for both push and read modes to apply configuration changes + - name: Restart Nix Daemon + shell: bash + run: | + echo "🔄 Restarting Nix daemon to apply configuration changes..." + sudo pkill -HUP nix-daemon || true diff --git a/actions/setup-nix-cache/package.json b/actions/setup-nix-cache/package.json new file mode 100644 index 00000000..c21963b2 --- /dev/null +++ b/actions/setup-nix-cache/package.json @@ -0,0 +1,11 @@ +{ + "name": "setup-nix-cache", + "version": "0.1.0", + "description": "Setup Nix cache with S3 support and optional signing", + "private": true, + "scripts": {}, + "author": "@smartcontractkit", + "license": "MIT", + "dependencies": {}, + "repository": "https://github.com/smartcontractkit/.github" +} diff --git a/actions/setup-nix-cache/project.json b/actions/setup-nix-cache/project.json new file mode 100644 index 00000000..62c43f2a --- /dev/null +++ b/actions/setup-nix-cache/project.json @@ -0,0 +1,7 @@ +{ + "name": "setup-nix-cache", + "$schema": "../../node_modules/nx/schemas/project-schema.json", + "projectType": "application", + "sourceRoot": "actions/setup-nix-cache", + "targets": {} +} diff --git a/actions/setup-nix-cache/scripts/upload-to-cache.sh b/actions/setup-nix-cache/scripts/upload-to-cache.sh new file mode 100644 index 00000000..fbca9957 --- /dev/null +++ b/actions/setup-nix-cache/scripts/upload-to-cache.sh @@ -0,0 +1,26 @@ +#!/bin/sh +set -eu +set -f # disable globbing +export IFS=' ' + +# Set HOME environment variable if it's not set +if [ -z "${HOME:-}" ]; then + export HOME="/home/runner" # GitHub Actions default home directory for the runner +fi + +# Ensure Nix daemon is loaded +if [ -e '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh' ]; then + . '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh' +fi + +export CACHE_URL=$1 +export AWS_REGION=$2 + +# Add your upload logic here, using the environment variables +echo "Uploading to cache at $CACHE_URL in region $AWS_REGION..." +# Update PATH +export PATH=/home/runner/.nix-profile/bin/nix:$PATH + +# Log and upload the paths to the Nix cache +echo "Uploading paths $OUT_PATHS to $CACHE_URL with AWS region $AWS_REGION" +exec nix copy --to "$CACHE_URL?scheme=https®ion=$AWS_REGION" $OUT_PATHS \ No newline at end of file diff --git a/actions/setup-nix/README.md b/actions/setup-nix/README.md index 279611c0..ce8534d9 100644 --- a/actions/setup-nix/README.md +++ b/actions/setup-nix/README.md @@ -1,32 +1,37 @@ -# setup-nix +# Setup Nix Action -`setup-nix` installs a nix environment and configures it with the specified -caches +`setup-nix` installs a nix environment using the +https://github.com/DeterminateSystems/nix-installer-action -## Configuration +## Inputs + +### install-url (optional) + +- **Description**: Custom URL for the Nix installer. +- **Required**: No +- **Default**: (If not provided, the action will use the default + DeterminateSystems installer). +- **Usage**: If you need to install Nix using a different installer URL, provide + it through this input. + +### extra-conf (optional) + +- **Description**: Additional Nix configuration options. +- **Required**: No +- **Default**: "" +- **Usage**: Use this input to provide extra configuration options that will be + appended to /etc/nix/nix.conf. + +- ## Usage ```yaml -inputs: - # custom cache inputs ---------------------------------- - # these can point to any public or private cache - cache-url: https://, s3://, etc - cache-pubkey: corresponding cache key - - # AWS inputs ------------------------------------ - # enable to read/write for private caches hosted using s3 buckets - # note: does not push to cache but environment is setup for pushing - enable-aws: bool, true/false - aws-region: credential location - role-to-assume: credential - role-duration-seconds: credential TTL - - # cachix inputs -------------------------------- - # enable to use private caches hosted on cachix - # enable to push to caches hosted on cachix - enable-cachix: bool, true/false - cachix-name: cache name - cachix-token: token for cachix account - - # github inputs --------------------------------- - github-token: token to enable reading private repositories +jobs: + setup_nix: + runs-on: ubuntu-latest + steps: + - name: Install Nix + uses: smartcontractkit/.github/actions/setup-nix@7a7de5813c702b2e9d042903a1e9cffd2c0b40c5 # make sure to use the latest commit hash for version + with: + extra-conf: | + sandbox = relaxed ``` diff --git a/actions/setup-nix/action.yml b/actions/setup-nix/action.yml index c5f327e9..85ecc54b 100644 --- a/actions/setup-nix/action.yml +++ b/actions/setup-nix/action.yml @@ -1,165 +1,45 @@ -name: setup-nix -description: "Setup nix with caches" +name: install-nix +description: + "Install Nix using DeterminateSystems/nix-installer-action by default, with an + option for a custom install URL" inputs: - # custom cache inputs ---------------------------------- - cache-url: - description: "Custom nix cache URL, for S3 use 's3://' " - required: false - default: "" - cache-pubkey: - description: "Custom nix cache binary public key" - required: false - default: "" - - # AWS inputs ------------------------------------ - enable-aws: - description: "Enable AWS credentials for nix cache" - required: false - default: false - aws-region: - description: "" - required: false - default: "us-west-1" - aws-access-key-id: - description: "The AWS_ACCESS_KEY_ID for S3 authentication. Will need to have read/write access to the bucket." - required: false - aws-secret-access-key: - description: "The AWS_SECRET_ACCESS_KEY for S3 authentication. Will need to have read/write access to the bucket." - required: false - aws-session-token: - description: "The AWS_SESSION_TOKEN for S3 authentication. Will need to have read/write access to the bucket." - required: false - role-to-assume: - description: "" - required: false - default: "" - role-duration-seconds: - description: "" - required: false - default: 3600 - - # grafana inputs (optional) ---------------------- - metrics-job-name: - description: "grafana metrics job name" - required: false - metrics-id: - description: - "grafana metrics id, used for continuity of metrics during job name - changes - required if metrics-job-name is passed" - required: false - gc-host: - description: "grafana hostname - required if metrics-job-name is passed" - required: false - gc-basic-auth: - description: "grafana basic auth - required if metrics-job-name is passed" - required: false - gc-org-id: + install-url: description: - "grafana org/tenant id - required if metrics-job-name is passed" + "Custom URL for Nix installer (default uses DeterminateSystems installer)" required: false + default: "" - # private signing key (optional) ---------------------- - private-signing-key: - description: "Private signing key for Nix cache" + extra-conf: + description: "Additional Nix configuration options" required: false default: "" runs: using: composite steps: - - name: configure AWS credentials - if: inputs.enable-aws == 'true' - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 + - name: Install Nix + if: ${{ inputs.install-url == '' }} + uses: DeterminateSystems/nix-installer-action@da36cb69b1c3247ad7a1f931ebfd954a1105ef14 # v14 with: - role-to-assume: ${{ inputs.role-to-assume }} - role-duration-seconds: ${{ inputs.role-duration-seconds }} - aws-region: ${{ inputs.aws-region }} - mask-aws-account-id: true - - - name: setup AWS credentials for nix - if: inputs.enable-aws == 'true' - shell: bash - env: - AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ inputs.aws-session-token }} - run: | - # set up folder with AWS configs - # nix reads AWS access from folder config - does not work with env vars - aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID - aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY - aws configure set aws_session_token $AWS_SESSION_TOKEN - aws configure set region ${{ inputs.aws-region }} - - # determine root path - ROOT_PATH="" - if [[ "${{ runner.os }}" == "Linux" ]]; then - ROOT_PATH="/root" - elif [[ "${{ runner.os }}" == "macOS" ]]; then - ROOT_PATH="/var/root" - else - echo "${{ runner.os }} not supported" - exit 1 - fi - - # symlink credential folders for nix build access - # nix build is run via daemon on root, credentials linked for daemon access - sudo ln -s ~/.aws $ROOT_PATH + determinate: true + extra-conf: | + sandbox = relaxed + ${{ inputs.extra-conf }} - - name: store private signing key - if: ${{ inputs.private-signing-key != '' }} + - name: Install Nix with Custom URL + if: ${{ inputs.install-url != '' }} shell: bash run: | - echo "${{ inputs.private-signing-key }}" > ~/.nix-signing-key - chmod 600 ~/.nix-signing-key + echo "Installing Nix from custom URL: ${{ inputs.install-url }}" + curl -L ${{ inputs.install-url }} | sh - - name: set NIX_SECRET_KEY_FILES - id: set-nix-secret-key-files - shell: bash - run: | - if [ '${{ inputs.private-signing-key }}' != '' ]; then - echo "NIX_SECRET_KEY_FILES=~/.nix-signing-key" >> $GITHUB_ENV - else - echo "NIX_SECRET_KEY_FILES=" >> $GITHUB_ENV + # Apply extra configuration if provided + if [ -n "${{ inputs.extra-conf }}" ]; then + echo "${{ inputs.extra-conf }}" >> /etc/nix/nix.conf fi - - name: save upload-to-cache.sh + - name: Verify Nix Installation shell: bash - if: ${{ inputs.cache-url != '' }} run: | - cat << EOF > ~/upload-to-cache.sh - #!/bin/sh - set -eu - set -f # disable globbing - export IFS=' ' - echo "Uploading paths" \$OUT_PATHS - exec nix copy --to "${{ inputs.cache-url }}?scheme=https®ion=${{ inputs.aws-region }}" \$OUT_PATHS - EOF - echo "NIX_POST_BUILD_HOOK_FILE=~/upload-to-cache.sh" >> $GITHUB_ENV - chmod +x ~/upload-to-cache.sh - - - uses: cachix/install-nix-action@ba0dd844c9180cbf77aa72a116d6fbc515d0e87b # v27 - with: - install_url: https://releases.nixos.org/nix/nix-2.18.1/install - nix_path: nixpkgs=channel:nixos-unstable - # Enables sandbox for all drv except those with `__noChroot = true;` - # Add caches for public nixos cache + public community cache + specified private cache - extra_nix_config: | - sandbox = relaxed - substituters = https://cache.nixos.org https://nix-community.cachix.org ${{ inputs.cache-url }} - trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs= ${{ inputs.cache-pubkey }} - secret-key-files = ${{ env.NIX_SECRET_KEY_FILES }} - post-build-hook = ${{ env.NIX_POST_BUILD_HOOK_FILE }} - - - name: Collect metrics - if: always() && inputs.metrics-job-name != '' - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1 - with: - id: ${{ inputs.metrics-id || inputs.metrics-job-name }} - basic-auth: ${{ inputs.gc-basic-auth }} - hostname: ${{ inputs.gc-host }} - org-id: ${{ inputs.gc-org-id }} - this-job-name: ${{ inputs.metrics-job-name }} - continue-on-error: true + nix --version From 61eca00f1ed36578e813bcefb00333f5651ebeb1 Mon Sep 17 00:00:00 2001 From: Pablo Date: Mon, 7 Oct 2024 13:47:49 -0600 Subject: [PATCH 4/8] chore: update comments on limitation for aws credentials --- actions/setup-nix-cache/action.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/actions/setup-nix-cache/action.yml b/actions/setup-nix-cache/action.yml index ac709ba6..5038ec8a 100644 --- a/actions/setup-nix-cache/action.yml +++ b/actions/setup-nix-cache/action.yml @@ -49,6 +49,8 @@ runs: aws-region: ${{ inputs.aws-region }} # Step to set AWS environment variables and symlink the credentials + # We need to set these using aws configure due to a limitation on how Nix reads the credentials for AWS. + # See https://github.com/NixOS/nix/issues/2143 for more details. - name: 🛠️ Setup AWS credentials for Nix shell: bash run: | From faa135a76d4a5009b9d24c18be3c35d31b707000 Mon Sep 17 00:00:00 2001 From: Pablo Estrada <139084212+ecPablo@users.noreply.github.com> Date: Mon, 7 Oct 2024 13:48:37 -0600 Subject: [PATCH 5/8] Update actions/setup-nix-cache/action.yml Co-authored-by: chainchad <96362174+chainchad@users.noreply.github.com> --- actions/setup-nix-cache/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actions/setup-nix-cache/action.yml b/actions/setup-nix-cache/action.yml index 5038ec8a..e86ce41e 100644 --- a/actions/setup-nix-cache/action.yml +++ b/actions/setup-nix-cache/action.yml @@ -66,7 +66,7 @@ runs: elif [[ "${{ runner.os }}" == "macOS" ]]; then ROOT_PATH="/var/root" else - echo "⚠️ ${ { runner.os }} not supported!" + echo "⚠️ ${{ runner.os }} not supported!" exit 1 fi From 2d2c804f347eb9c37d2b8c058a60750cbfc14d02 Mon Sep 17 00:00:00 2001 From: Pablo Estrada <139084212+ecPablo@users.noreply.github.com> Date: Mon, 7 Oct 2024 13:49:45 -0600 Subject: [PATCH 6/8] Update actions/setup-nix-cache/action.yml Co-authored-by: chainchad <96362174+chainchad@users.noreply.github.com> --- actions/setup-nix-cache/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/actions/setup-nix-cache/action.yml b/actions/setup-nix-cache/action.yml index e86ce41e..1f64cacf 100644 --- a/actions/setup-nix-cache/action.yml +++ b/actions/setup-nix-cache/action.yml @@ -156,7 +156,7 @@ runs: sudo cp $MODIFIED_SCRIPT_PATH $PERSISTENT_SCRIPT_PATH # Check if the modified script exists and has correct permissions - if [ -f "$PERSISTENT_SCRIPT_PATH" ]; then + if [[ -f "$PERSISTENT_SCRIPT_PATH" ]]; then echo "✅ Modified script exists at $PERSISTENT_SCRIPT_PATH with the following permissions:" ls -la $PERSISTENT_SCRIPT_PATH else From 9259988383a8caa3b4acb8e1314e5a43b369cc34 Mon Sep 17 00:00:00 2001 From: Pablo Date: Mon, 7 Oct 2024 13:58:58 -0600 Subject: [PATCH 7/8] fix: remove logging outputs of the added config lines --- actions/setup-nix-cache/action.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/actions/setup-nix-cache/action.yml b/actions/setup-nix-cache/action.yml index 1f64cacf..6080b357 100644 --- a/actions/setup-nix-cache/action.yml +++ b/actions/setup-nix-cache/action.yml @@ -94,7 +94,7 @@ runs: sudo sed -i "s|^substituters.*|$SUBSTITUTERS_LINE|" /etc/nix/nix.conf else echo "➕ Appending 'substituters' line to /etc/nix/nix.conf" - echo "$SUBSTITUTERS_LINE" | sudo tee -a /etc/nix/nix.conf + echo "$SUBSTITUTERS_LINE" >> /etc/nix/nix.conf fi # Replace or append the 'trusted-public-keys' line @@ -103,7 +103,7 @@ runs: sudo sed -i "s|^trusted-public-keys.*|$PUBLIC_KEYS_LINE|" /etc/nix/nix.conf else echo "➕ Appending 'trusted-public-keys' line to /etc/nix/nix.conf" - echo "$PUBLIC_KEYS_LINE" | sudo tee -a /etc/nix/nix.conf + echo "$PUBLIC_KEYS_LINE" >> /etc/nix/nix.conf fi # Store private signing key if provided @@ -124,7 +124,7 @@ runs: sudo sed -i "s|^secret-key-files.*|$SECRET_KEY_FILES_LINE|" /etc/nix/nix.conf else echo "➕ Appending 'secret-key-files' line to /etc/nix/nix.conf" - echo "$SECRET_KEY_FILES_LINE" | sudo tee -a /etc/nix/nix.conf + echo "$SECRET_KEY_FILES_LINE" >> /etc/nix/nix.conf fi # Set up post-build hook for cache upload if pushing @@ -166,7 +166,7 @@ runs: # Set up the post-build hook to use the modified script echo "Setting post-build-hook in /etc/nix/nix.conf" - echo "post-build-hook = $PERSISTENT_SCRIPT_PATH" | sudo tee -a /etc/nix/nix.conf + echo "post-build-hook = $PERSISTENT_SCRIPT_PATH" >> /etc/nix/nix.conf # Restart Nix daemon for both push and read modes to apply configuration changes - name: Restart Nix Daemon From 37846789f980dfc00cf46ca55fd290a859df06bc Mon Sep 17 00:00:00 2001 From: Pablo Date: Tue, 8 Oct 2024 07:17:37 -0600 Subject: [PATCH 8/8] fix: use sudo to write to conf --- actions/setup-nix-cache/action.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/actions/setup-nix-cache/action.yml b/actions/setup-nix-cache/action.yml index 6080b357..22bfcd0b 100644 --- a/actions/setup-nix-cache/action.yml +++ b/actions/setup-nix-cache/action.yml @@ -94,7 +94,7 @@ runs: sudo sed -i "s|^substituters.*|$SUBSTITUTERS_LINE|" /etc/nix/nix.conf else echo "➕ Appending 'substituters' line to /etc/nix/nix.conf" - echo "$SUBSTITUTERS_LINE" >> /etc/nix/nix.conf + sudo bash -c 'echo "$SUBSTITUTERS_LINE" >> /etc/nix/nix.conf' fi # Replace or append the 'trusted-public-keys' line @@ -103,7 +103,7 @@ runs: sudo sed -i "s|^trusted-public-keys.*|$PUBLIC_KEYS_LINE|" /etc/nix/nix.conf else echo "➕ Appending 'trusted-public-keys' line to /etc/nix/nix.conf" - echo "$PUBLIC_KEYS_LINE" >> /etc/nix/nix.conf + sudo bash -c 'echo "$PUBLIC_KEYS_LINE" >> /etc/nix/nix.conf' fi # Store private signing key if provided @@ -124,7 +124,7 @@ runs: sudo sed -i "s|^secret-key-files.*|$SECRET_KEY_FILES_LINE|" /etc/nix/nix.conf else echo "➕ Appending 'secret-key-files' line to /etc/nix/nix.conf" - echo "$SECRET_KEY_FILES_LINE" >> /etc/nix/nix.conf + sudo bash -c 'echo "$SECRET_KEY_FILES_LINE" >> /etc/nix/nix.conf' fi # Set up post-build hook for cache upload if pushing @@ -166,7 +166,7 @@ runs: # Set up the post-build hook to use the modified script echo "Setting post-build-hook in /etc/nix/nix.conf" - echo "post-build-hook = $PERSISTENT_SCRIPT_PATH" >> /etc/nix/nix.conf + sudo bash -c 'echo "post-build-hook = $PERSISTENT_SCRIPT_PATH" >> /etc/nix/nix.conf' # Restart Nix daemon for both push and read modes to apply configuration changes - name: Restart Nix Daemon