Fix workflows, documentation, autoupdate, and simplification

.github/workflows/autoupdate-other-derivations.yml

@@ -18,14 +18,20 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install Nix
         uses: cachix/install-nix-action@v25
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
       - name: Setup binary cache
         uses: cachix/cachix-action@v14
         with:
           name: toolbox
           skipPush: true
       - name: Check for updates
         run : |
-          nix-shell --command autoupdate/update.py
+          nix-shell --command ./autoupdate/update.py
+          (
+            cd autoupdate
+            ./update-all-providers
+          )
         env:
           GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
           SOURCE_FILTER: '(?!terraform.*)'
@@ -48,10 +54,10 @@ jobs:
             git add changes_wip.md
             git commit -m "add update changelog"
             git push origin $BRANCH_NAME
-            echo ::set-output name=updates::true
-            echo ::set-output name=branch::$BRANCH_NAME
+            echo updates=true >> ${GITHUB_OUTPUT}
+            echo branch=$BRANCH_NAME >> ${GITHUB_OUTPUT}
           else
-            echo ::set-output name=updates::false
+            echo updates=false >> ${GITHUB_OUTPUT}
           fi
         env:
           GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
@@ -70,6 +76,8 @@ jobs:
           ref: '${{ needs.update.outputs.branch }}'
       - name: Install Nix
         uses: cachix/install-nix-action@v25
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
       - name: Setup binary cache
         uses: cachix/cachix-action@v14
         with:

.github/workflows/autoupdate-terraform-providers.yaml

@@ -18,14 +18,20 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install Nix
         uses: cachix/install-nix-action@v25
+        with:
+          nix_path: nixpkgs=nixpkgs=channel:nixos-unstable
       - name: Setup binary cache
         uses: cachix/cachix-action@v14
         with:
           name: toolbox
           skipPush: true
       - name: Check for updates
         run : |
-          nix-shell --command autoupdate/update.py
+          nix-shell --command ./autoupdate/update.py
+          (
+            cd autoupdate
+          ./update-all-providers
+          )
         env:
           GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
           SOURCE_FILTER: 'terraform'
@@ -48,10 +54,10 @@ jobs:
             git add changes_wip.md
             git commit -m "add update changelog"
             git push origin $BRANCH_NAME
-            echo ::set-output name=updates::true
-            echo ::set-output name=branch::$BRANCH_NAME
-          else
-            echo ::set-output name=updates::false
+            echo updates=true >> ${GITHUB_OUTPUT}$
+            echo branch=$BRANCH_NAME >> ${GITHUB_OUTPUT}$
+          else$
+            echo updates=false >> ${GITHUB_OUTPUT}$
           fi
         env:
           GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
@@ -70,6 +76,8 @@ jobs:
           ref: '${{ needs.update.outputs.branch }}'
       - name: Install Nix
         uses: cachix/install-nix-action@v25
+        with:
+          nix_path: nixpkgs=nixpkgs=channel:nixos-unstable
       - name: Setup binary cache
         uses: cachix/cachix-action@v14
         with:

.github/workflows/build-non-tf-derivations.yaml

@@ -19,6 +19,8 @@ jobs:
     - uses: actions/checkout@v4
     - name: Install Nix
       uses: cachix/install-nix-action@v25
+      with:
+        nix_path: nixpkgs=channel:nixos-unstable
     - name: Setup binary cache
       uses: cachix/cachix-action@v14
       with:

.github/workflows/build-tf-derivations.yaml

@@ -19,6 +19,8 @@ jobs:
     - uses: actions/checkout@v4
     - name: Install Nix
       uses: cachix/install-nix-action@v25
+      with:
+        nix_path: nixpkgs=channel:nixos-unstable
     - name: Setup binary cache
       uses: cachix/cachix-action@v14
       with:

.github/workflows/nixpkgs.yml

@@ -17,6 +17,8 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install Nix
         uses: cachix/install-nix-action@v25
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
       - name: Setup binary cache
         uses: cachix/cachix-action@v14
         with:
@@ -67,6 +69,8 @@ jobs:
           ref: '${{needs.update.outputs.branch_name}}'
       - name: Install Nix
         uses: cachix/install-nix-action@v25
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
       - name: Setup binary cache
         uses: cachix/cachix-action@v14
         with:

.github/workflows/publish-other-derivations.yaml

@@ -20,6 +20,8 @@ jobs:
     - uses: actions/checkout@v4
     - name: Install Nix
       uses: cachix/install-nix-action@v25
+      with:
+        nix_path: nixpkgs=channel:nixos-unstable
     - name: Setup binary cache
       uses: cachix/cachix-action@v14
       with:

.github/workflows/publish-terraform-providers.yaml

@@ -20,6 +20,8 @@ jobs:
     - uses: actions/checkout@v4
     - name: Install Nix
       uses: cachix/install-nix-action@v25
+      with:
+        nix_path: nixpkgs=channel:nixos-unstable
     - name: Setup binary cache
       uses: cachix/cachix-action@v14
       with:

README.md

@@ -200,8 +200,49 @@ Examples:
 * `cue: 0.0.14 -> 0.0.15`
 
 ## Maintainers
+### Updating
+We manage sources within 2 files:
+* nix/sources.json: handled by niv
+* providers.json: terraform providers handled by [update-provider](./update-provider) and [update-all-providers](./update-all-providers)
 
-### Managing sources
+Quickly, a full toolbox update should be performed this way:
+
+```code
+niv update nixpkgs # update nixpkgs
+nix-shell --command ./autoupdate/update.py # update all sources managed by niv with autoupdate set to true
+( 
+    cd autoupdate
+    ./update-all-providers # update all terraform providers which are not in nixpkgs
+)
+```
+
+Checking:
+
+```code
+nix-build
+nix-build -A terraform-providers
+```
+
+Pushing to cachix:
+
+```code
+nix-build | cachix push toolbox
+nix-build -A terraform-providers | cachix push toolbox
+```
+
+#### Managing sources with autoupdate
+We have a small helper to autoupdate entries in nix/sources.json without the attribute autoupdate set to false.
+The helper will try to build everything and in case a vendorSha256 is outputed will add it to nix/sources.json.
+
+niv ignore this attribute and won't try to delete it unless you drop the source (niv drop ...).
+
+```code
+./autoupdate/update.py
+```
+
+Entries with autoupdate attribute set to true should be treated manually with niv if relevant (ie we did not delibarately pinned the package version).
+
+#### Managing sources with niv
 
 Sources of `nixpkgs` or custom packages are managed with [niv](https://github.com/nmattia/niv). You can install it this way:
 ```sh
@@ -223,9 +264,17 @@ To update sources to a particular version:
 niv update concourse -v 7.6.0
 ```
 
+[!NOTE]
+nixpkgs must be updated with niv. In providers.json its autoupdate attribute is set to false:
+
+```code
+niv update nixpkgs
+```
+
 #### golang sources
 Currently nixpkgs moves to hash and vendorHash attributes populated with SRI hashes values.
-Currenty niv does not support vendorHash attribute so we need to add it directly in goBuild.* helpers.
+Currenty niv does not support vendorHash attribute so we need to add it directly in buildGo.* helpers.
+
 ```
 buildGoModule rec {
     ...
@@ -234,6 +283,11 @@ buildGoModule rec {
     }
 ```
 
+
+Moreover a golang project from the old style building (packages) to the new one (modules). In golang modules, the vendor directory can be there or not. nix can trust it if asked. This situation creates a lot of case and situations where updating toolbox will break golang builds.
+
+The [autoupdate script](./autoupdate/update.py) takes care to add vendorSha256
+
 ### Testing a new package locally
 
 After adding a new package in the toolbox you can build it with:
@@ -264,6 +318,28 @@ export NIXPKGS_ALLOW_UNFREE=1
 In the current repo you will find an envrc.EXAMPLE file to source.
 
 ### Managing terraform providers sources
+#### Automated management
+
+Adding a new provider or update an existing one. Its code must be located on Github:
+
+```code
+./update-provider <owner>/<repo>
+```
+
+Example:
+
+```code
+./update-provider terraform-provider-concourse/concourse
+```
+
+If the build fails because of vendor dir try to set vendorHash to null in providers.json then re-build:
+
+```code
+./update-provider terraform-provider-concourse/concourse --force
+```
+
+#### Detailed management
+
 We manage few providers with the same mechanism used in nixpkgs. Our custom providers are managed through [a json file](./providers.json)
 
 A provider is defined by this block:

autoupdate/update-all-providers

@@ -0,0 +1,27 @@
+#!/usr/bin/env nix-shell
+#! nix-shell -i bash -p jq
+# shellcheck shell=bash
+
+# Took from nixpkgs unstable @ pkgs/applications/networking/cluster/terraform-providers/
+
+# Update all providers which have specified provider source address
+set -euo pipefail
+
+readarray -t providers < <(
+  jq -r 'to_entries
+  | map_values(.value + { alias: .key })
+  | .[]
+  | select(."homepage"?)
+  | .alias' providers.json
+)
+
+cat <<EOF
+Will update ${#providers[@]} providers:
+
+${providers[*]}
+
+EOF
+
+for provider in "${providers[@]}"; do
+  ./update-provider --no-spdx "$@" "${provider}"
+done