From 2f66985365d45e35ddeb84c546e0247ca9ced903 Mon Sep 17 00:00:00 2001 From: ci-infra Date: Wed, 14 Feb 2024 23:19:47 +0100 Subject: [PATCH 01/16] Fix workflows, documentation, autoupdate, and simplification --- .../autoupdate-other-derivations.yml | 2 + .../autoupdate-terraform-providers.yaml | 5 +- .../workflows/build-non-tf-derivations.yaml | 2 + .github/workflows/build-tf-derivations.yaml | 2 + .github/workflows/nixpkgs.yml | 2 + .../workflows/publish-other-derivations.yaml | 2 + .../publish-terraform-providers.yaml | 2 + README.md | 80 ++++++++- providers.json => autoupdate/providers.json | 0 autoupdate/update-all-providers | 27 +++ autoupdate/update-provider | 166 ++++++++++++++++++ autoupdate/update.py | 3 +- default.nix | 8 +- nix/sources.json | 93 +++------- pkgs/rancher-cli.nix | 39 ---- pkgs/tflint-ruleset-aws.nix | 39 ++-- toolbox | 51 ------ 17 files changed, 323 insertions(+), 200 deletions(-) rename providers.json => autoupdate/providers.json (100%) create mode 100755 autoupdate/update-all-providers create mode 100755 autoupdate/update-provider delete mode 100644 pkgs/rancher-cli.nix delete mode 100755 toolbox diff --git a/.github/workflows/autoupdate-other-derivations.yml b/.github/workflows/autoupdate-other-derivations.yml index 31d5d147..ed06fd3a 100644 --- a/.github/workflows/autoupdate-other-derivations.yml +++ b/.github/workflows/autoupdate-other-derivations.yml @@ -18,6 +18,8 @@ jobs: - uses: actions/checkout@v4 - name: Install Nix uses: cachix/install-nix-action@v25 + with: + nix_path: nixpkgs=channel:nixos-unstable - name: Setup binary cache uses: cachix/cachix-action@v14 with: diff --git a/.github/workflows/autoupdate-terraform-providers.yaml b/.github/workflows/autoupdate-terraform-providers.yaml index ed6eae41..75f33739 100644 --- a/.github/workflows/autoupdate-terraform-providers.yaml +++ b/.github/workflows/autoupdate-terraform-providers.yaml @@ -18,6 +18,8 @@ jobs: - uses: actions/checkout@v4 - name: Install Nix uses: cachix/install-nix-action@v25 + with: + nix_path: nixpkgs=nixpkgs=channel:nixos-unstable - name: Setup binary cache uses: cachix/cachix-action@v14 with: @@ -25,7 +27,8 @@ jobs: skipPush: true - name: Check for updates run : | - nix-shell --command autoupdate/update.py + ./autoupdate/update.py + ./update-all-providers env: GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}' SOURCE_FILTER: 'terraform' diff --git a/.github/workflows/build-non-tf-derivations.yaml b/.github/workflows/build-non-tf-derivations.yaml index df7f93b8..1e337cd9 100644 --- a/.github/workflows/build-non-tf-derivations.yaml +++ b/.github/workflows/build-non-tf-derivations.yaml @@ -19,6 +19,8 @@ jobs: - uses: actions/checkout@v4 - name: Install Nix uses: cachix/install-nix-action@v25 + with: + nix_path: nixpkgs=channel:nixos-unstable - name: Setup binary cache uses: cachix/cachix-action@v14 with: diff --git a/.github/workflows/build-tf-derivations.yaml b/.github/workflows/build-tf-derivations.yaml index f42dc77c..ecb8dd9c 100644 --- a/.github/workflows/build-tf-derivations.yaml +++ b/.github/workflows/build-tf-derivations.yaml @@ -19,6 +19,8 @@ jobs: - uses: actions/checkout@v4 - name: Install Nix uses: cachix/install-nix-action@v25 + with: + nix_path: nixpkgs=channel:nixos-unstable - name: Setup binary cache uses: cachix/cachix-action@v14 with: diff --git a/.github/workflows/nixpkgs.yml b/.github/workflows/nixpkgs.yml index 04d038fe..269f632a 100644 --- a/.github/workflows/nixpkgs.yml +++ b/.github/workflows/nixpkgs.yml @@ -17,6 +17,8 @@ jobs: - uses: actions/checkout@v4 - name: Install Nix uses: cachix/install-nix-action@v25 + with: + nix_path: nixpkgs=channel:nixos-unstable - name: Setup binary cache uses: cachix/cachix-action@v14 with: diff --git a/.github/workflows/publish-other-derivations.yaml b/.github/workflows/publish-other-derivations.yaml index 6c9b320b..c38059fc 100644 --- a/.github/workflows/publish-other-derivations.yaml +++ b/.github/workflows/publish-other-derivations.yaml @@ -20,6 +20,8 @@ jobs: - uses: actions/checkout@v4 - name: Install Nix uses: cachix/install-nix-action@v25 + with: + nix_path: nixpkgs=channel:nixos-unstable - name: Setup binary cache uses: cachix/cachix-action@v14 with: diff --git a/.github/workflows/publish-terraform-providers.yaml b/.github/workflows/publish-terraform-providers.yaml index 30818f49..40fe8064 100644 --- a/.github/workflows/publish-terraform-providers.yaml +++ b/.github/workflows/publish-terraform-providers.yaml @@ -20,6 +20,8 @@ jobs: - uses: actions/checkout@v4 - name: Install Nix uses: cachix/install-nix-action@v25 + with: + nix_path: nixpkgs=channel:nixos-unstable - name: Setup binary cache uses: cachix/cachix-action@v14 with: diff --git a/README.md b/README.md index f77ad503..f6556d67 100644 --- a/README.md +++ b/README.md @@ -200,8 +200,49 @@ Examples: * `cue: 0.0.14 -> 0.0.15` ## Maintainers +### Updating +We manage sources within 2 files: +* nix/sources.json: handled by niv +* providers.json: terraform providers handled by [update-provider](./update-provider) and [update-all-providers](./update-all-providers) -### Managing sources +Quickly, a full toolbox update should be performed this way: + +```code +niv update nixpkgs # update nixpkgs +./autoupdate/update.py # update all sources managed by niv with autoupdate set to true +( + cd autoupdate + ./update-all-providers # update all terraform providers which are not in nixpkgs +) +``` + +Checking: + +```code +nix-build +nix-build -A terraform-providers +``` + +Pushing to cachix: + +```code +nix-build | cachix push toolbox +nix-build -A terraform-providers | cachix push toolbox +``` + +#### Managing sources with autoupdate +We have a small helper to autoupdate entries in nix/sources.json without the attribute autoupdate set to false. +The helper will try to build everything and in case a vendorSha256 is outputed will add it to nix/sources.json. + +niv ignore this attribute and won't try to delete it unless you drop the source (niv drop ...). + +```code +./autoupdate/update.py +``` + +Entries with autoupdate attribute set to true should be treated manually with niv if relevant (ie we did not delibarately pinned the package version). + +#### Managing sources with niv Sources of `nixpkgs` or custom packages are managed with [niv](https://github.com/nmattia/niv). You can install it this way: ```sh @@ -223,9 +264,17 @@ To update sources to a particular version: niv update concourse -v 7.6.0 ``` +[!NOTE] +nixpkgs must be updated with niv. In providers.json its autoupdate attribute is set to false: + +```code +niv update nixpkgs +``` + #### golang sources Currently nixpkgs moves to hash and vendorHash attributes populated with SRI hashes values. -Currenty niv does not support vendorHash attribute so we need to add it directly in goBuild.* helpers. +Currenty niv does not support vendorHash attribute so we need to add it directly in buildGo.* helpers. + ``` buildGoModule rec { ... @@ -234,6 +283,11 @@ buildGoModule rec { } ``` + +Moreover a golang project from the old style building (packages) to the new one (modules). In golang modules, the vendor directory can be there or not. nix can trust it if asked. This situation creates a lot of case and situations where updating toolbox will break golang builds. + +The [autoupdate script](./autoupdate/update.py) takes care to add vendorSha256 + ### Testing a new package locally After adding a new package in the toolbox you can build it with: @@ -264,6 +318,28 @@ export NIXPKGS_ALLOW_UNFREE=1 In the current repo you will find an envrc.EXAMPLE file to source. ### Managing terraform providers sources +#### Automated management + +Adding a new provider or update an existing one. Its code must be located on Github: + +```code +./update-provider / +``` + +Example: + +```code +./update-provider terraform-provider-concourse/concourse +``` + +If the build fails because of vendor dir try to set vendorHash to null in providers.json then re-build: + +```code +./update-provider terraform-provider-concourse/concourse --force +``` + +#### Detailed management + We manage few providers with the same mechanism used in nixpkgs. Our custom providers are managed through [a json file](./providers.json) A provider is defined by this block: diff --git a/providers.json b/autoupdate/providers.json similarity index 100% rename from providers.json rename to autoupdate/providers.json diff --git a/autoupdate/update-all-providers b/autoupdate/update-all-providers new file mode 100755 index 00000000..1e81246e --- /dev/null +++ b/autoupdate/update-all-providers @@ -0,0 +1,27 @@ +#!/usr/bin/env nix-shell +#! nix-shell -i bash -p jq +# shellcheck shell=bash + +# Took from nixpkgs unstable @ pkgs/applications/networking/cluster/terraform-providers/ + +# Update all providers which have specified provider source address +set -euo pipefail + +readarray -t providers < <( + jq -r 'to_entries + | map_values(.value + { alias: .key }) + | .[] + | select(."homepage"?) + | .alias' providers.json +) + +cat </] + +Update a single provider in the providers.json inventory file. + +For example to update 'terraform-providers.aws' run: + + ./update-provider aws + +If the provider is not in the list already, use the form '/' +to add the provider to the list: + + ./update-provider hetznercloud/hcloud + +Options: + + * --force: Force the update even if the version matches. + * --no-build: Don't build provider +DOC +} + +build=1 +force= +provider= +spdx=1 + +while [[ $# -gt 0 ]]; do + case "$1" in + -h | --help) + show_usage + exit + ;; + --force) + force=1 + shift + ;; + --no-build) + build=0 + shift + ;; + --no-spdx) + spdx=0 + shift + ;; + *) + if [[ -n ${provider} ]]; then + echo "ERROR: provider name was passed two times: '${provider}' and '$1'" + echo "Use --help for more info" + exit 1 + fi + provider=$1 + shift + ;; + esac +done + +if [[ -z ${provider} ]]; then + echo "ERROR: No providers specified!" + echo + show_usage + exit 1 +fi + +# Usage: read_attr +read_attr() { + jq -r ".\"${provider}\".\"$1\"" providers.json +} + +# Usage: update_attr +update_attr() { + if [[ $2 == "null" ]]; then + jq -S ".\"${provider}\".\"$1\" = null" providers.json | sponge providers.json + else + jq -S ".\"${provider}\".\"$1\" = \"$2\"" providers.json | sponge providers.json + fi +} + +repo_root=$(git rev-parse --show-toplevel) + +generate_hash() { + nurl --expr "(import ${repo_root} {}).terraform-providers.${provider}.$1" +} + +echo_provider() { + echo "== terraform-providers.${provider}: $* ==" +} + +pushd "$(dirname "$0")" >/dev/null + +if [[ ${provider} =~ ^[^/]+/[^/]+$ ]]; then + homepage="https://registry.terraform.io/providers/${provider}" + provider=$(basename "${provider}") + echo_provider "init" + update_attr homepage "${homepage}" + # create empty stings so nix-prefetch works + update_attr hash "" + update_attr vendorHash "" + update_attr spdx "" +fi + +homepage="$(read_attr homepage)" + +registry_response=$(curl -s "${homepage//providers/v1/providers}") + +old_rev="$(read_attr rev)" +rev="$(jq -r '.tag' <<<"${registry_response}")" +if [[ ${force} != 1 ]]; then + if [[ ${old_rev} == "${rev}" ]]; then + echo_provider "already at version ${rev}" + exit + fi + if [[ ${rev//v/} =~ [[:alpha:]] ]]; then + echo_provider "not updating to unstable version ${rev}" + exit + fi +fi +echo_provider "updating from ${old_rev} to ${rev}" +update_attr rev "${rev}" + +provider_source_url="$(jq -r '.source' <<<"${registry_response}")" + +org="$(echo "${provider_source_url}" | cut -d '/' -f 4)" +update_attr owner "${org}" +repo="$(echo "${provider_source_url}" | cut -d '/' -f 5)" +update_attr repo "${repo}" + +if [[ ${spdx} == 1 ]]; then + old_spdx="$(read_attr spdx)" + if [[ ${old_spdx} != null ]]; then + spdx="$(curl -L -s ${GITHUB_TOKEN:+-u ":${GITHUB_TOKEN}"} "https://api.github.com/repos/${org}/${repo}/license" | jq -r '.license.spdx_id')" + update_attr spdx "${spdx}" + fi +fi + +echo_provider "calculating hash" +hash=$(generate_hash src) +update_attr hash "${hash}" + +old_vendor_hash="$(read_attr vendorHash)" +if [[ ${old_vendor_hash} != null ]]; then + echo_provider "calculating vendorHash" + vendorHash=$(generate_hash goModules) + update_attr vendorHash "${vendorHash}" +fi + +# Check that the provider builds +if [[ ${build} == 1 ]]; then + echo_provider "building" + nix-build --no-out-link "${repo_root}" -A "terraform-providers.${provider}" +fi + +popd >/dev/null diff --git a/autoupdate/update.py b/autoupdate/update.py index 7ba26355..d05a4dae 100755 --- a/autoupdate/update.py +++ b/autoupdate/update.py @@ -1,4 +1,5 @@ -#!/usr/bin/env -S=2 python -u +#!/usr/bin/env nix-shell +#! nix-shell -i python3 -p python3 python3Packages.pyyaml python3Packages.graphqlclient import json import os import re diff --git a/default.nix b/default.nix index 21e901ea..52feccff 100644 --- a/default.nix +++ b/default.nix @@ -30,6 +30,8 @@ rec { saml2aws k9s terraform_1 terraform-docs tflint + cue + rancher ; terraform_1_0_0 = builtins.trace "terraform_1_0_0 is deprecated use terraform_1" terraform_1; @@ -66,8 +68,6 @@ rec { vault ;} // automated-providers // special-providers; - cue = callPackage ./pkgs/cue.nix { source = sources.cue; }; - fly = callPackage ./pkgs/fly.nix { inherit sources; }; git = pkgs.git; @@ -94,9 +94,7 @@ rec { velero = pkgs.callPackage ./pkgs/velero.nix { source = sources.velero; }; - rancher-cli = pkgs.callPackage ./pkgs/rancher-cli.nix { source = sources.rancher-cli; }; - - tflint-ruleset-aws = pkgs.callPackage ./pkgs/tflint-ruleset-aws.nix { source = sources.tflint-ruleset-aws; }; + tflint-ruleset-aws = pkgs.callPackage ./pkgs/tflint-ruleset-aws.nix {tflint-ruleset-aws = pkgs.tflint-plugins.tflint-ruleset-aws;}; print-client-zones-infos = callPackage ./pkgs/print-client-zones-infos {}; diff --git a/nix/sources.json b/nix/sources.json index 20a5b1ce..980bae70 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -52,20 +52,7 @@ "type": "tarball", "url": "https://github.com/Caascad/conformity-tooling/archive/75232d9e000bf6dfb929a95a6156f8e37dd6f44d.tar.gz", "url_template": "https://github.com///archive/.tar.gz", - "version": "0.0.2" - }, - "cue": { - "branch": "master", - "description": "Validate and define text-based and dynamic configuration", - "homepage": "https://cuelang.org", - "owner": "cue-lang", - "repo": "cue", - "rev": "8709d8aa900927e4eacd464e17e63b6fc0661937", - "sha256": "1nrfr7ryr2xbmmq3cihhkn98l3kxk0hbwscgji45cvgirmpxdjyn", - "type": "tarball", - "url": "https://github.com/cue-lang/cue/archive/v0.6.0.tar.gz", - "url_template": "https://github.com///archive/v.tar.gz", - "version": "0.6.0" + "version": "0.0.1" }, "fly-wrapper": { "branch": "master", @@ -86,12 +73,12 @@ "homepage": "", "owner": "databus23", "repo": "helm-diff", - "rev": "c54f0858a1050c2bd068956ed2d407bbc0f93e8c", - "sha256": "1q4638kawhpdxkks36zmjh3pwh7lvn1yx0g0n1pjpl5h5cg56yy7", + "rev": "c15a11093b3dddca987ad5602e29069a84d545e1", + "sha256": "0vwgpnjrgzw705b2f7ld6vzpyqvlpaa0zzwkp9xxmqh5n38y4fc4", "type": "tarball", - "url": "https://github.com/databus23/helm-diff/archive/v3.8.1.tar.gz", + "url": "https://github.com/databus23/helm-diff/archive/v3.9.4.tar.gz", "url_template": "https://github.com///archive/v.tar.gz", - "version": "3.8.1" + "version": "3.9.4" }, "kail": { "branch": "master", @@ -99,12 +86,12 @@ "homepage": "", "owner": "boz", "repo": "kail", - "rev": "688f29a7cb3cda4cdf3c4c01b023c3e463fa0d86", - "sha256": "1lipphs00q8dx5wqn8caps82x4llwdb1h4k70pjzy2b3hqvb9nf7", + "rev": "ec131e2d9c752664702f7711a1981ec8ec1f594a", + "sha256": "0wqkm4vbn4d0x4pii15lspl1il3zx4v3m1x7dh2lg0i18r83pi8v", "type": "tarball", - "url": "https://github.com/boz/kail/archive/v0.16.1.tar.gz", + "url": "https://github.com/boz/kail/archive/v0.17.4.tar.gz", "url_template": "https://github.com///archive/v.tar.gz", - "version": "0.16.1" + "version": "0.17.4" }, "ketall": { "branch": "master", @@ -132,19 +119,6 @@ "url_template": "https://github.com///archive/v.tar.gz", "version": "1.6.2" }, - "kubectl-node-shell": { - "branch": "master", - "description": "Exec into node via kubectl", - "homepage": "https://github.com/kvaps/kubectl-node-shell", - "owner": "kvaps", - "repo": "kubectl-node-shell", - "rev": "bfa4119425152e9883dc0e139830ff0c0fb35a34", - "sha256": "0wff3361kn1bm82n7nnxal4ssqjrrh2dyfq82ncipcr35154g57q", - "type": "tarball", - "url": "https://github.com/kvaps/kubectl-node-shell/archive/v1.8.0.tar.gz", - "url_template": "https://github.com///archive/v.tar.gz", - "version": "1.8.0" - }, "kubectl-topology": { "branch": "master", "description": "A 'kubectl' plugin that provides insight into the topology of a Kubernetes cluster.", @@ -191,10 +165,10 @@ "homepage": "https://github.com/nmattia/niv", "owner": "nmattia", "repo": "niv", - "rev": "9341b1027da2c2f95f8e808a3cb4b403e0a62c77", - "sha256": "1l996s518iv7bcfzzhxlsn35ahbslpbvhl3ds1zpnama7la23y9b", + "rev": "290965abaa02be33b601032d850c588a6bafb1a5", + "sha256": "1f75kd0s7ch882camvsp0shkbx0vs0hshn184il0fi6i7k4xs5hy", "type": "tarball", - "url": "https://github.com/nmattia/niv/archive/9341b1027da2c2f95f8e808a3cb4b403e0a62c77.tar.gz", + "url": "https://github.com/nmattia/niv/archive/290965abaa02be33b601032d850c588a6bafb1a5.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "nixpkgs": { @@ -204,10 +178,10 @@ "homepage": null, "owner": "NixOS", "repo": "nixpkgs", - "rev": "5ad9903c16126a7d949101687af0aa589b1d7d3d", - "sha256": "1i0nvgzzadbl29hzs5n4qbc0nnw69nh79b0kq3g7zi1926rczlqn", + "rev": "89653a03e0915e4a872788d10680e7eec92f8600", + "sha256": "0yarb5hqbawp7cd48p9pgf98gqdls9g1002bpzqdvhbcmbck17dj", "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/5ad9903c16126a7d949101687af0aa589b1d7d3d.tar.gz", + "url": "https://github.com/NixOS/nixpkgs/archive/89653a03e0915e4a872788d10680e7eec92f8600.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "poetry2nix": { @@ -216,26 +190,12 @@ "homepage": "", "owner": "nix-community", "repo": "poetry2nix", - "rev": "e0b44e9e2d3aa855d1dd77b06f067cd0e0c3860d", - "sha256": "0zz3qzp2b5i9gw4yfxfrq07iadcdadackph12h02w19bb3535rm6", + "rev": "4eb2ac54029af42a001c9901194e9ce19cbd8a40", + "sha256": "16fi71fpywiqsya1z99kkb14dansyrmkkrb2clzs3b5qqx673wf4", "type": "tarball", - "url": "https://github.com/nix-community/poetry2nix/archive/e0b44e9e2d3aa855d1dd77b06f067cd0e0c3860d.tar.gz", + "url": "https://github.com/nix-community/poetry2nix/archive/4eb2ac54029af42a001c9901194e9ce19cbd8a40.tar.gz", "url_template": "https://github.com///archive/.tar.gz", - "version": "1.42.1" - }, - "rancher-cli": { - "autoupdate": false, - "branch": "v2.6", - "description": "Rancher CLI", - "homepage": null, - "owner": "rancher", - "repo": "cli", - "rev": "8f35783891bde2e0528a8a13bb979a88bbfb179c", - "sha256": "04xzpc3c9s3rd1qm6vzra1ixjycsxjcfrslk228pm4cx2kbkw4iy", - "type": "tarball", - "url": "https://github.com/rancher/cli/archive/v2.4.11.tar.gz", - "url_template": "https://github.com///archive/v.tar.gz", - "version": "2.4.11" + "version": "2024.2.618482" }, "rswitch": { "branch": "master", @@ -248,20 +208,7 @@ "type": "tarball", "url": "https://github.com/Caascad/rswitch/archive/7a44cbc5643a3e5f50fd446964d69a5a46a24efd.tar.gz", "url_template": "https://github.com///archive/.tar.gz", - "version": "1.2.9" - }, - "tflint-ruleset-aws": { - "branch": "master", - "description": "TFLint ruleset for terraform-provider-aws", - "homepage": "", - "owner": "terraform-linters", - "repo": "tflint-ruleset-aws", - "rev": "a988165d0e08fefba7c3e2a57c2d910fc1701f5b", - "sha256": "16sg7833vg0k34dfkdkv8sji2y29sg0dkhv96skmrmqsgq7yd8s8", - "type": "tarball", - "url": "https://github.com/terraform-linters/tflint-ruleset-aws/archive/v0.26.0.tar.gz", - "url_template": "https://github.com///archive/v.tar.gz", - "version": "0.26.0" + "version": "1.2.8" }, "vault-token-helper": { "branch": "main", diff --git a/pkgs/rancher-cli.nix b/pkgs/rancher-cli.nix deleted file mode 100644 index 35ae0a81..00000000 --- a/pkgs/rancher-cli.nix +++ /dev/null @@ -1,39 +0,0 @@ -{ buildGoModule -, fetchFromGitHub -, lib -, source -}: - -buildGoModule rec { - pname = "rancher-cli"; - version = source.version; - - src = fetchFromGitHub { - owner = source.owner; - repo = source.repo; - rev = "v${version}"; - sha256 = source.sha256; - }; - - vendorHash = "sha256-/etX/SFoaze2ZVp6XLypgEfQ22R/tD1xNwuTTvvvPw8="; # niv does not provide it. obtained by replaction current hash with lib.fakeHash - - postInstall = '' - mv $out/bin/cli $out/bin/rancher - ''; - - CGO_ENABLED = 0; - - ldflags = [ - "-w" "-s" - "-extldflags '-static'" - "-X main.VERSION=${source.version}" - ]; - - meta = with lib; { - description = "Rancher CLI to interact with Rancher Server"; - homepage = "https://rancher.com/"; - license = licenses.asl20; - maintainers = [ "xmaillard" ]; - }; -} - diff --git a/pkgs/tflint-ruleset-aws.nix b/pkgs/tflint-ruleset-aws.nix index 78cd6cc1..eca05c44 100644 --- a/pkgs/tflint-ruleset-aws.nix +++ b/pkgs/tflint-ruleset-aws.nix @@ -1,32 +1,15 @@ -{ buildGoModule -, fetchFromGitHub +{ stdenv , lib -, source -, pkgs +, tflint-ruleset-aws }: - -buildGoModule rec { +stdenv.mkDerivation rec { pname = "tflint-ruleset-aws"; - version = source.version; - - src = fetchFromGitHub { - owner = source.owner; - repo = source.repo; - rev = "v${version}"; - sha256 = source.sha256; - }; - - # vendorHash = lib.fakeHash; - vendorHash = "sha256-JhAAyfDVRZS2QyvXNa61srlZKgsBFeKloeKbcXXpytk="; - - # Integration test with tflint not working - doCheck = false; - - meta = with lib; { - description = "TFLint ruleset for terraform-provider-aws"; - homepage = "https://github.com/terraform-linters/tflint-ruleset-aws"; - license = licenses.asl20; - maintainers = [ "xmaillard" ]; - }; + version = tflint-ruleset-aws.version; + unpackPhase = ":"; + + installPhase = '' + mkdir -p $out/bin + cp ${tflint-ruleset-aws}/github.com/terraform-linters/${pname}/${version}/* $out/bin + ''; + meta = tflint-ruleset-aws.meta; } - diff --git a/toolbox b/toolbox deleted file mode 100755 index dd4f6d73..00000000 --- a/toolbox +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash - -# needed for vault -export NIXPKGS_ALLOW_UNFREE=1 -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -DIR="$( cd -P "$( dirname "$SOURCE" )" >/dev/null 2>&1 && pwd )" - -_continue() { - read -p "Continue [y/n]: " -n 1 -r - echo - [[ $REPLY =~ ^[Yy]$ ]] || exit 1 -} - -log() { - local args="$*" - local prefix="\x1B[32m[toolbox]:\x1B[0m" - echo -e "$prefix $args" -} - -log-warning() { - local args="$*" - local prefix="\x1B[33m[toolbox]:\x1B[0m" - echo -e "$prefix $args" >&2 -} - -if ! nix-env -q toolbox >/dev/null 2>&1; then - -echo -log-warning "Toolbox needs to be migrated !" -log "The installation process of the toolbox has changed since you have installed it." -log "The install script needs to be run again." -echo -_continue -echo - -"$DIR"/install - -echo -log-warning "Migration complete !" -log "The toolbox script is now installed in your environment as any other tool." - -fi - -[ ${BASH_ALIASES[toolbox]+abc} ] || log-warning "You can remove the toolbox bash alias from your .bashrc" - -nix run toolbox.toolbox -c toolbox "$@" From 5a337ba6fefdd9c6e9aa5c1f5b661c35da0a3519 Mon Sep 17 00:00:00 2001 From: ci-infra Date: Thu, 15 Feb 2024 10:47:24 +0100 Subject: [PATCH 02/16] try --- .github/workflows/autoupdate-other-derivations.yml | 6 +++++- .github/workflows/autoupdate-terraform-providers.yaml | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/autoupdate-other-derivations.yml b/.github/workflows/autoupdate-other-derivations.yml index ed06fd3a..64554118 100644 --- a/.github/workflows/autoupdate-other-derivations.yml +++ b/.github/workflows/autoupdate-other-derivations.yml @@ -27,7 +27,11 @@ jobs: skipPush: true - name: Check for updates run : | - nix-shell --command autoupdate/update.py + ./autoupdate/update.py + ( + cd autoupdate + ./update-all-providers + ) env: GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}' SOURCE_FILTER: '(?!terraform.*)' diff --git a/.github/workflows/autoupdate-terraform-providers.yaml b/.github/workflows/autoupdate-terraform-providers.yaml index 75f33739..9f3c0c35 100644 --- a/.github/workflows/autoupdate-terraform-providers.yaml +++ b/.github/workflows/autoupdate-terraform-providers.yaml @@ -28,7 +28,10 @@ jobs: - name: Check for updates run : | ./autoupdate/update.py + ( + cd autoupdate ./update-all-providers + ) env: GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}' SOURCE_FILTER: 'terraform' From 4289d1eb0ecd1721e590b8ff8215ec70c4c9c9c8 Mon Sep 17 00:00:00 2001 From: ci-infra Date: Thu, 15 Feb 2024 10:50:40 +0100 Subject: [PATCH 03/16] try --- autoupdate/update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autoupdate/update.py b/autoupdate/update.py index d05a4dae..f544db09 100755 --- a/autoupdate/update.py +++ b/autoupdate/update.py @@ -1,5 +1,5 @@ #!/usr/bin/env nix-shell -#! nix-shell -i python3 -p python3 python3Packages.pyyaml python3Packages.graphqlclient +#! nix-shell -i python3 -p python3 python3Packages.pyyaml python3Packages.graphqlclient niv import json import os import re From 803fd1027ccca069a6733d0ea6d59e0af8679a9a Mon Sep 17 00:00:00 2001 From: ci-infra Date: Thu, 15 Feb 2024 10:54:32 +0100 Subject: [PATCH 04/16] try --- .github/workflows/autoupdate-other-derivations.yml | 2 +- .github/workflows/autoupdate-terraform-providers.yaml | 2 +- README.md | 2 +- autoupdate/update.py | 2 +- nix/sources.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/autoupdate-other-derivations.yml b/.github/workflows/autoupdate-other-derivations.yml index 64554118..21744e26 100644 --- a/.github/workflows/autoupdate-other-derivations.yml +++ b/.github/workflows/autoupdate-other-derivations.yml @@ -27,7 +27,7 @@ jobs: skipPush: true - name: Check for updates run : | - ./autoupdate/update.py + nix-shell --command ./autoupdate/update.py ( cd autoupdate ./update-all-providers diff --git a/.github/workflows/autoupdate-terraform-providers.yaml b/.github/workflows/autoupdate-terraform-providers.yaml index 9f3c0c35..a44dea57 100644 --- a/.github/workflows/autoupdate-terraform-providers.yaml +++ b/.github/workflows/autoupdate-terraform-providers.yaml @@ -27,7 +27,7 @@ jobs: skipPush: true - name: Check for updates run : | - ./autoupdate/update.py + nix-shell --command ./autoupdate/update.py ( cd autoupdate ./update-all-providers diff --git a/README.md b/README.md index f6556d67..52f6fe00 100644 --- a/README.md +++ b/README.md @@ -209,7 +209,7 @@ Quickly, a full toolbox update should be performed this way: ```code niv update nixpkgs # update nixpkgs -./autoupdate/update.py # update all sources managed by niv with autoupdate set to true +nix-shell --command ./autoupdate/update.py # update all sources managed by niv with autoupdate set to true ( cd autoupdate ./update-all-providers # update all terraform providers which are not in nixpkgs diff --git a/autoupdate/update.py b/autoupdate/update.py index f544db09..d05a4dae 100755 --- a/autoupdate/update.py +++ b/autoupdate/update.py @@ -1,5 +1,5 @@ #!/usr/bin/env nix-shell -#! nix-shell -i python3 -p python3 python3Packages.pyyaml python3Packages.graphqlclient niv +#! nix-shell -i python3 -p python3 python3Packages.pyyaml python3Packages.graphqlclient import json import os import re diff --git a/nix/sources.json b/nix/sources.json index 980bae70..7648401f 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -52,7 +52,7 @@ "type": "tarball", "url": "https://github.com/Caascad/conformity-tooling/archive/75232d9e000bf6dfb929a95a6156f8e37dd6f44d.tar.gz", "url_template": "https://github.com///archive/.tar.gz", - "version": "0.0.1" + "version": "0.0.2" }, "fly-wrapper": { "branch": "master", From b837b2609ce95f06d3ea706677be94fc1b373c50 Mon Sep 17 00:00:00 2001 From: ci-infra Date: Thu, 15 Feb 2024 11:05:31 +0100 Subject: [PATCH 05/16] try --- .github/workflows/autoupdate-other-derivations.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/autoupdate-other-derivations.yml b/.github/workflows/autoupdate-other-derivations.yml index 21744e26..7678846b 100644 --- a/.github/workflows/autoupdate-other-derivations.yml +++ b/.github/workflows/autoupdate-other-derivations.yml @@ -54,8 +54,8 @@ jobs: git add changes_wip.md git commit -m "add update changelog" git push origin $BRANCH_NAME - echo ::set-output name=updates::true - echo ::set-output name=branch::$BRANCH_NAME + echo updates=true >> ${GITHUB_OUTPUT} + echo branch=$BRANCH_NAME >> ${GITHUB_OUTPUT} else echo ::set-output name=updates::false fi From b9144b2d4eb5d483b96533de501cc6af9220676e Mon Sep 17 00:00:00 2001 From: ci-infra Date: Thu, 15 Feb 2024 11:08:50 +0100 Subject: [PATCH 06/16] try --- .github/workflows/autoupdate-other-derivations.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/autoupdate-other-derivations.yml b/.github/workflows/autoupdate-other-derivations.yml index 7678846b..6091ca3a 100644 --- a/.github/workflows/autoupdate-other-derivations.yml +++ b/.github/workflows/autoupdate-other-derivations.yml @@ -57,7 +57,7 @@ jobs: echo updates=true >> ${GITHUB_OUTPUT} echo branch=$BRANCH_NAME >> ${GITHUB_OUTPUT} else - echo ::set-output name=updates::false + echo updates=false >> ${GITHUB_OUTPUT} fi env: GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}' From b2c504f702f1d7414dc36d1473180bd4b6e47589 Mon Sep 17 00:00:00 2001 From: ci-infra Date: Thu, 15 Feb 2024 11:14:24 +0100 Subject: [PATCH 07/16] try --- nix/sources.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/sources.json b/nix/sources.json index 7648401f..980bae70 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -52,7 +52,7 @@ "type": "tarball", "url": "https://github.com/Caascad/conformity-tooling/archive/75232d9e000bf6dfb929a95a6156f8e37dd6f44d.tar.gz", "url_template": "https://github.com///archive/.tar.gz", - "version": "0.0.2" + "version": "0.0.1" }, "fly-wrapper": { "branch": "master", From 1f511eab84424fb138753bdfb894254a9e28196d Mon Sep 17 00:00:00 2001 From: ci-infra Date: Thu, 15 Feb 2024 11:17:45 +0100 Subject: [PATCH 08/16] try --- .github/workflows/autoupdate-other-derivations.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/autoupdate-other-derivations.yml b/.github/workflows/autoupdate-other-derivations.yml index 6091ca3a..92764eac 100644 --- a/.github/workflows/autoupdate-other-derivations.yml +++ b/.github/workflows/autoupdate-other-derivations.yml @@ -76,6 +76,8 @@ jobs: ref: '${{ needs.update.outputs.branch }}' - name: Install Nix uses: cachix/install-nix-action@v25 + with: + nix_path: nixpkgs=channel:nixos-unstable - name: Setup binary cache uses: cachix/cachix-action@v14 with: From 696e4c13feca34f6c1e48664a08b46375fd4e6f7 Mon Sep 17 00:00:00 2001 From: ci-infra Date: Thu, 15 Feb 2024 11:22:21 +0100 Subject: [PATCH 09/16] try --- default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/default.nix b/default.nix index 52feccff..2912b403 100644 --- a/default.nix +++ b/default.nix @@ -8,7 +8,7 @@ let - providersSource = pkgs.lib.importJSON ./providers.json; + providersSource = pkgs.lib.importJSON ./autoupdate/providers.json; automated-providers = pkgs.lib.mapAttrs (_: attrs: pkgs.terraform-providers.mkProvider attrs) providersSource; special-providers = { harbor = automated-providers.harbor.override {mkProviderGoModule = pkgs.buildGo122Module;}; From 33772be2d94cfdb26cc3a71677367c32d0cbfcf7 Mon Sep 17 00:00:00 2001 From: ci-infra Date: Thu, 15 Feb 2024 11:26:34 +0100 Subject: [PATCH 10/16] try --- nix/sources.json | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/nix/sources.json b/nix/sources.json index 980bae70..6c265aee 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -106,6 +106,19 @@ "url_template": "https://github.com///archive/v.tar.gz", "version": "1.3.8" }, + "kubectl-node-shell": { + "branch": "master", + "description": "Exec into node via kubectl", + "homepage": "https://github.com/kvaps/kubectl-node-shell", + "owner": "kvaps", + "repo": "kubectl-node-shell", + "rev": "bfa4119425152e9883dc0e139830ff0c0fb35a34", + "sha256": "0wff3361kn1bm82n7nnxal4ssqjrrh2dyfq82ncipcr35154g57q", + "type": "tarball", + "url": "https://github.com/kvaps/kubectl-node-shell/archive/v1.8.0.tar.gz", + "url_template": "https://github.com///archive/v.tar.gz", + "version": "1.8.0" + }, "ksniff": { "branch": "master", "description": "Kubectl plugin to ease sniffing on kubernetes pods using tcpdump and wireshark", From 55d8dd93429a9df18d9d058fe30b9dbbb9ea346a Mon Sep 17 00:00:00 2001 From: ci-infra Date: Thu, 15 Feb 2024 11:35:54 +0100 Subject: [PATCH 11/16] try --- .github/workflows/autoupdate-terraform-providers.yaml | 2 ++ .github/workflows/nixpkgs.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/autoupdate-terraform-providers.yaml b/.github/workflows/autoupdate-terraform-providers.yaml index a44dea57..0fc025dc 100644 --- a/.github/workflows/autoupdate-terraform-providers.yaml +++ b/.github/workflows/autoupdate-terraform-providers.yaml @@ -76,6 +76,8 @@ jobs: ref: '${{ needs.update.outputs.branch }}' - name: Install Nix uses: cachix/install-nix-action@v25 + with: + nix_path: nixpkgs=nixpkgs=channel:nixos-unstable - name: Setup binary cache uses: cachix/cachix-action@v14 with: diff --git a/.github/workflows/nixpkgs.yml b/.github/workflows/nixpkgs.yml index 269f632a..d28ac742 100644 --- a/.github/workflows/nixpkgs.yml +++ b/.github/workflows/nixpkgs.yml @@ -69,6 +69,8 @@ jobs: ref: '${{needs.update.outputs.branch_name}}' - name: Install Nix uses: cachix/install-nix-action@v25 + with: + nix_path: nixpkgs=channel:nixos-unstable - name: Setup binary cache uses: cachix/cachix-action@v14 with: From e6a24a0ead45289615c560dbae10e41f3cddc685 Mon Sep 17 00:00:00 2001 From: ci-infra Date: Thu, 15 Feb 2024 11:39:00 +0100 Subject: [PATCH 12/16] try --- pkgs/helm-plugins/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/helm-plugins/default.nix b/pkgs/helm-plugins/default.nix index 2a8621e3..9be3e510 100644 --- a/pkgs/helm-plugins/default.nix +++ b/pkgs/helm-plugins/default.nix @@ -18,7 +18,7 @@ sha256 = source.sha256; }; # vendorHash = lib.fakeHash; - vendorHash = "sha256-2tiBFS3gvSbnyighSorg/ar058ZJmiQviaT13zOS8KA="; + vendorHash = "sha256-51xjHGU9TC4Nwa9keR0b7bgwpZcRmG7duT9R1JRr3Uw"; postInstall = '' mv $out/bin/helm-diff $out/bin/diff ''; From 16cd9e401a1b771b0c109b8b92db369d109e5061 Mon Sep 17 00:00:00 2001 From: ci-infra Date: Thu, 15 Feb 2024 11:42:27 +0100 Subject: [PATCH 13/16] try --- .github/workflows/autoupdate-terraform-providers.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/autoupdate-terraform-providers.yaml b/.github/workflows/autoupdate-terraform-providers.yaml index 0fc025dc..841d7e99 100644 --- a/.github/workflows/autoupdate-terraform-providers.yaml +++ b/.github/workflows/autoupdate-terraform-providers.yaml @@ -54,10 +54,10 @@ jobs: git add changes_wip.md git commit -m "add update changelog" git push origin $BRANCH_NAME - echo ::set-output name=updates::true - echo ::set-output name=branch::$BRANCH_NAME - else - echo ::set-output name=updates::false + echo updates=true >> ${GITHUB_OUTPUT}$ + echo branch=$BRANCH_NAME >> ${GITHUB_OUTPUT}$ + else$ + echo updates=false >> ${GITHUB_OUTPUT}$ fi env: GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}' From 97bbda3b810a4966c7e1476159396ecf7fc40c0a Mon Sep 17 00:00:00 2001 From: ci-infra Date: Thu, 15 Feb 2024 11:50:46 +0100 Subject: [PATCH 14/16] try --- pkgs/kubectl-plugins/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/kubectl-plugins/default.nix b/pkgs/kubectl-plugins/default.nix index d455c7b7..67e7ab95 100644 --- a/pkgs/kubectl-plugins/default.nix +++ b/pkgs/kubectl-plugins/default.nix @@ -70,7 +70,7 @@ rec { sha256 = sources.kail.sha256; }; # vendorHash= lib.fakeHash; - vendorHash= "sha256-W+/vIq7qC+6apk+1GOWvmcwyyjFRkndq8X5m/lRYOu4="; + vendorHash= "sha256-u6/LsLphaqYswJkAuqgrgknnm+7MnaeH+kf9BPcdtrc=; subPackages = ["cmd/kail/"]; }; From b07dc3aac7d48e412c4c88f348b5f763589d1f61 Mon Sep 17 00:00:00 2001 From: ci-infra Date: Thu, 15 Feb 2024 14:06:32 +0100 Subject: [PATCH 15/16] try --- pkgs/kubectl-plugins/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/kubectl-plugins/default.nix b/pkgs/kubectl-plugins/default.nix index 67e7ab95..bef42cd7 100644 --- a/pkgs/kubectl-plugins/default.nix +++ b/pkgs/kubectl-plugins/default.nix @@ -70,7 +70,7 @@ rec { sha256 = sources.kail.sha256; }; # vendorHash= lib.fakeHash; - vendorHash= "sha256-u6/LsLphaqYswJkAuqgrgknnm+7MnaeH+kf9BPcdtrc=; + vendorHash= "sha256-u6/LsLphaqYswJkAuqgrgknnm+7MnaeH+kf9BPcdtrc="; subPackages = ["cmd/kail/"]; }; From fa478b6318c2029172c665bd000840d8f95c5820 Mon Sep 17 00:00:00 2001 From: ci-infra Date: Thu, 15 Feb 2024 16:03:04 +0100 Subject: [PATCH 16/16] rswitch: remove pkgs override --- default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/default.nix b/default.nix index 2912b403..0a3c2ea8 100644 --- a/default.nix +++ b/default.nix @@ -102,7 +102,7 @@ rec { sd = callPackage ./pkgs/sd {}; - rswitch = import sources.rswitch {inherit pkgs; poetry2nixStandalone = poetry2nixStandalone;}; + rswitch = import sources.rswitch {poetry2nixStandalone = poetry2nixStandalone;}; get-rancher-creds = (import sources.conformity-tooling { inherit pkgs;}).getranchercreds;