From dd355ea1f0c9b7848ae95aa830eb7cbf86e28de9 Mon Sep 17 00:00:00 2001 From: Leorize Date: Mon, 24 Jun 2024 22:00:32 -0500 Subject: [PATCH 1/3] release_manifest: add `get` subcommand Add a new `get` subcommand for querying the file name of the release binary for a given target triplet. This will be used for obtaining generated docs. --- tools/release_manifest.nim | 40 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tools/release_manifest.nim b/tools/release_manifest.nim index 89e32cc1baf..5d1c14aaad9 100644 --- a/tools/release_manifest.nim +++ b/tools/release_manifest.nim @@ -255,6 +255,23 @@ proc addCommand(manifest: string, archiveData: varargs[string]) = # Serialize a new manifest writeFile(manifest, $database.serialize()) +proc getCommand(manifest, target: string): int = + ## Implementation for the `get` subcommand. + ## + ## :manifest: + ## The filename of the release manifest to inspect. + ## + ## :target: + ## The triplet of interest. + let database = json.parseFile(manifest).deserialize() + + let idx = database.triplet.find(target) + if idx < 0: + stderr.writeLine("error: target $1 could not be found in database") + return 1 + + stdout.writeLine(database.file[idx]) + func escapeDataForGithubActions(s: string): string = ## Escape the string `s` so that it can be used as data for workflow commands. # The list is obtained from here: @@ -314,6 +331,7 @@ type Help = "help" Add = "add" FilesToUpload = "files-to-upload" + Get = "get" Version = "version" Flag {.pure.} = enum @@ -363,6 +381,7 @@ Usage: $app [args]... Commands: add Add artifacts to the manifest files-to-upload List the files to be uploaded + get Get release artifact for a target version Print the release version help Display help for any subcommand @@ -393,6 +412,15 @@ Options: format such that it can be used in workflow commands (ie. set-output) without losing data. +$globalOpt +""" + + GetHelp = """ +Usage: $app get [options] [--] + +Print the artifact file name of the given target triplet. An error will be raised +if no artifact can be found for the given target. + $globalOpt """ @@ -432,6 +460,8 @@ proc printHelp(action: Action) = stdout.write(AddHelp % defaultHelpFormat) of FilesToUpload: stdout.write(FilesToUploadHelp % defaultHelpFormat) + of Get: + stdout.write(GetHelp % defaultHelpFormat) of Version: stdout.write(VersionHelp % defaultHelpFormat) @@ -504,6 +534,16 @@ proc dispatch(cli: Cli): int = result = 1 else: filesToUploadCommand(manifest, format.get) + of Get: + let manifest = cli.flags.getOrDefault(Flag.File, DefaultManifestFile) + if cli.args.len == 1: + result = getCommand(manifest, cli.args[0]) + else: + # No or more than one targets were given, print the help text and set failure. + if cli.args.len > 1: + stderr.writeLine("error: only one target is expected") + printHelp(cli.action) + result = 1 of Version: let manifest = cli.flags.getOrDefault(Flag.File, DefaultManifestFile) versionCommand(manifest) From bd8a44941f5e33ed6d7d42a7957a767539dce005 Mon Sep 17 00:00:00 2001 From: Leorize Date: Wed, 3 Jul 2024 13:39:26 -0500 Subject: [PATCH 2/3] publisher: split docs deployment into a workflow Documentation will now be built using a separated workflow that will be run whenever publisher finishes. This allows us to use the latest page deployment code regardless of the commit that triggered publisher. Some auxilliary changes also come along: - New deploy-docs workflow can also be run manually. - "Generated docs" artifacts are now removed. Instead the generated documentation will be obtained from the release tarball for Linux. This is done since artifacts have a timeout to them, which meddle with our ability to run docs deployment at will. - Implemented basic version segmentation: `/devel` subdirectory will now contain the latest devel docs. --- .github/workflows/ci.yml | 9 ---- .github/workflows/deploy-docs.yml | 86 +++++++++++++++++++++++++++++++ .github/workflows/publisher.yml | 19 +------ doc/ci.rst | 25 +++++++-- 4 files changed, 107 insertions(+), 32 deletions(-) create mode 100644 .github/workflows/deploy-docs.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9674bfec052..0a75e1a04e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -549,15 +549,6 @@ jobs: echo "archive=$archive" >> $GITHUB_OUTPUT echo "metadata=$metadata" >> $GITHUB_OUTPUT - - name: Upload docs to artifacts - if: matrix.target.shared_builder - uses: actions/upload-artifact@v4 - with: - # If this name is updated, tweak publisher.yml - name: Generated docs - path: doc/html/ - if-no-files-found: error - - name: Upload release package to artifacts uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml new file mode 100644 index 00000000000..64aa5991f93 --- /dev/null +++ b/.github/workflows/deploy-docs.yml @@ -0,0 +1,86 @@ +name: Deploy documentation +on: + # Automatically run after any completed publish + workflow_run: + workflows: + - Publish built artifacts + types: + - completed + + # For manual triggers + workflow_dispatch: + +# Run every script actions in bash +defaults: + run: + shell: bash + +concurrency: doc-publisher + +jobs: + deploy: + runs-on: ubuntu-latest + + permissions: + id-token: write + pages: write + + environment: + name: github-pages + url: ${{ steps.deploy.outputs.page_url }} + + env: + # Triplet to obtain docs from + DOC_TARGET: x86_64-linux-gnu + + steps: + - uses: actions/checkout@v4 + + - name: Setup latest compiler + uses: nim-works/setup-nimskull@0.1.2 + with: + nimskull-version: "*" # Grab the latest nimskull-version + + - name: Compile release_manifest + run: nim c -d:release -o:release_manifest tools/release_manifest.nim + + - id: versions + name: Grab latest release version + run: | + # Stolen from asdf-nimskull + sort_versions() { + sed 'h; s/[+-]/./g; s/$/.z/; G; s/\n/ /' | + LC_ALL=C sort -t. -k 1,1n -k 2,2n -k 3,3n -k 4,4 -k 5,5n | awk '{print $2}' + } + + all_tags=$(gh release list --json tagName --jq '.[] | .tagName') + latest=$(sort_versions <<<"$all_tags" | tail -n 1) + + echo "Latest devel is: $latest" + echo "devel=$latest" >> "$GITHUB_OUTPUT" + env: + GH_TOKEN: ${{ github.token }} + + - name: Construct devel docs + run: | + tmpdir=$(mktemp -dp "$RUNNER_TEMP" devel.XXXXXXXXXX) + # Get the name of the binary archive for the documentation target + release_archive=$(gh release download "$DEVEL" -p manifest.json -O - | ./release_manifest -f /dev/stdin get "$DOC_TARGET") + # Download the latest release binary + gh release download "$DEVEL" -p "$release_archive" -O "$tmpdir/$release_archive" + # Extract and remove the top-level directory + tar -C "$tmpdir" -xf "$tmpdir/$release_archive" --strip-components=1 + + mkdir -p built-docs + cp -rT "$tmpdir/doc/html" built-docs/devel + cp -rT "$tmpdir/doc/html" built-docs + env: + GH_TOKEN: ${{ github.token }} + DEVEL: ${{ steps.versions.outputs.devel }} + + - uses: actions/upload-pages-artifact@v3 + with: + path: built-docs/ + + - id: deploy + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/publisher.yml b/.github/workflows/publisher.yml index f940fb7a183..5d0125ed76a 100644 --- a/.github/workflows/publisher.yml +++ b/.github/workflows/publisher.yml @@ -25,9 +25,6 @@ jobs: url: ${{ steps.release.outputs.url }} steps: - # Publish action needs a checkout - - uses: actions/checkout@v4 - - name: Obtain latest successful run id id: finder run: | @@ -48,17 +45,9 @@ jobs: WORKFLOW: ci.yml CONCLUSION: success GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} # Download the latest instance of artifacts from a build done previously - - name: Download generated docs - uses: actions/download-artifact@v4 - with: - run-id: ${{ steps.finder.outputs.run_id }} - # Keep up-to-date with ci.yml - name: Generated docs - path: doc/html - github-token: ${{ github.token }} - - name: Download generated source archive uses: actions/download-artifact@v4 with: @@ -87,12 +76,6 @@ jobs: path: release-staging github-token: ${{ github.token }} - - name: Publish docs - uses: JamesIves/github-pages-deploy-action@v4.6.1 - with: - branch: gh-pages - folder: doc/html - - id: release-files name: Create release manifest run: | diff --git a/doc/ci.rst b/doc/ci.rst index 7accfb59c84..7e4b6ee2972 100644 --- a/doc/ci.rst +++ b/doc/ci.rst @@ -96,8 +96,11 @@ was developed to leverage this. 4. Assuming that the staging branch passes all tests, ``devel`` is fast-forwarded to the staging branch head commit. -5. The "Publish" pipeline deploys the rendered documentation and binaries - generated by the testing pipeline earlier in the staging branch. +5. The "Publish" pipeline deploys the binaries generated by the testing pipeline + earlier in the staging branch. + +6. The "Deploy documentation" pipeline deploys the latest documentation to GitHub + Pages. Assumptions in CI design ------------------------ @@ -161,9 +164,6 @@ which are used by other pipelines, these artifacts are: * "source archive": generated from the git clone. -* "Generated docs": HTML documentation generated from source. This is the - ``doc/html`` folder after a ``koch docs`` run. - Changes to how these artifacts are packaged must be reviewed carefully to ensure that dependent pipelines will still function. @@ -245,6 +245,21 @@ Only one instance of this pipeline might be run at any given time, due to its mutating nature. Currently Github Action's ``concurrency`` feature is used to block multiple runs. +"Deploy documentation" +---------------------- + +As the name suggests, this pipeline constructs and deploys the documentation +website. This pipeline is dependent on the +`Release manifest tool <#tools-release-manifest-tool>`_. + +Unlike other pipelines, the code ran will always be that of the latest commit +in ``devel`` branch. This meant that the code should always be aware of +differences between versions of |NimSkull| being deployed. + +Currently, only one instance of this pipeline might be run at any given time +to prevent races. GitHub Actions's ``concurrency`` feature is used to +serialize the runs. + Development guidelines ====================== From 72d6373af8da95ea02d80f3cf8a5af6bf7b4e789 Mon Sep 17 00:00:00 2001 From: alaviss Date: Thu, 4 Jul 2024 20:38:48 -0500 Subject: [PATCH 3/3] ci: better English Co-authored-by: Saem Ghani --- doc/ci.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/ci.rst b/doc/ci.rst index 7e4b6ee2972..b27c7273a69 100644 --- a/doc/ci.rst +++ b/doc/ci.rst @@ -252,8 +252,8 @@ As the name suggests, this pipeline constructs and deploys the documentation website. This pipeline is dependent on the `Release manifest tool <#tools-release-manifest-tool>`_. -Unlike other pipelines, the code ran will always be that of the latest commit -in ``devel`` branch. This meant that the code should always be aware of +Unlike other pipelines, the code run will always be that of the latest commit +in ``devel`` branch. This means that the code should always be aware of differences between versions of |NimSkull| being deployed. Currently, only one instance of this pipeline might be run at any given time