From bd3d3dc99680c23b38317d0ca39fce408d1c9a16 Mon Sep 17 00:00:00 2001 From: alaviss Date: Thu, 4 Jul 2024 21:11:45 -0500 Subject: [PATCH] ci: split docs deployment from publisher and deploy directly to pages (#1370) ## Summary Documentation will now be built using a separated workflow that will be run whenever publisher finishes. This allows us to use the latest page deployment code regardless of the commit that triggered publisher. ## Details * Added `get` sub-command to `release_manifest` , which allows one to extract binary archive name from a release manifest. * "Generated docs" artifacts are now removed. Instead the generated documentation will be obtained from the release tarball for Linux amd64. This allows docs publishing to run regardless of whether the artifact for the latest release has expired or not. * Documentation deployment will now skip `gh-pages` branch and deploy directly using `actions/deploy-pages` . This allows us to drop the heavyweight `gh-pages` branch and remove the latency between push and page being deployed. * Documentation deployment can now be triggered on-demand in case of a prior failure. * Documentation deployment will now always be done using latest code in `devel` . * Development documentation is also pushed to `/devel` sub-folder. This is preliminary setup for versioned docs in the future. --------- Co-authored-by: Saem Ghani --- .github/workflows/ci.yml | 9 ---- .github/workflows/deploy-docs.yml | 86 +++++++++++++++++++++++++++++++ .github/workflows/publisher.yml | 19 +------ doc/ci.rst | 25 +++++++-- tools/release_manifest.nim | 40 ++++++++++++++ 5 files changed, 147 insertions(+), 32 deletions(-) create mode 100644 .github/workflows/deploy-docs.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9674bfec052..0a75e1a04e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -549,15 +549,6 @@ jobs: echo "archive=$archive" >> $GITHUB_OUTPUT echo "metadata=$metadata" >> $GITHUB_OUTPUT - - name: Upload docs to artifacts - if: matrix.target.shared_builder - uses: actions/upload-artifact@v4 - with: - # If this name is updated, tweak publisher.yml - name: Generated docs - path: doc/html/ - if-no-files-found: error - - name: Upload release package to artifacts uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml new file mode 100644 index 00000000000..64aa5991f93 --- /dev/null +++ b/.github/workflows/deploy-docs.yml @@ -0,0 +1,86 @@ +name: Deploy documentation +on: + # Automatically run after any completed publish + workflow_run: + workflows: + - Publish built artifacts + types: + - completed + + # For manual triggers + workflow_dispatch: + +# Run every script actions in bash +defaults: + run: + shell: bash + +concurrency: doc-publisher + +jobs: + deploy: + runs-on: ubuntu-latest + + permissions: + id-token: write + pages: write + + environment: + name: github-pages + url: ${{ steps.deploy.outputs.page_url }} + + env: + # Triplet to obtain docs from + DOC_TARGET: x86_64-linux-gnu + + steps: + - uses: actions/checkout@v4 + + - name: Setup latest compiler + uses: nim-works/setup-nimskull@0.1.2 + with: + nimskull-version: "*" # Grab the latest nimskull-version + + - name: Compile release_manifest + run: nim c -d:release -o:release_manifest tools/release_manifest.nim + + - id: versions + name: Grab latest release version + run: | + # Stolen from asdf-nimskull + sort_versions() { + sed 'h; s/[+-]/./g; s/$/.z/; G; s/\n/ /' | + LC_ALL=C sort -t. -k 1,1n -k 2,2n -k 3,3n -k 4,4 -k 5,5n | awk '{print $2}' + } + + all_tags=$(gh release list --json tagName --jq '.[] | .tagName') + latest=$(sort_versions <<<"$all_tags" | tail -n 1) + + echo "Latest devel is: $latest" + echo "devel=$latest" >> "$GITHUB_OUTPUT" + env: + GH_TOKEN: ${{ github.token }} + + - name: Construct devel docs + run: | + tmpdir=$(mktemp -dp "$RUNNER_TEMP" devel.XXXXXXXXXX) + # Get the name of the binary archive for the documentation target + release_archive=$(gh release download "$DEVEL" -p manifest.json -O - | ./release_manifest -f /dev/stdin get "$DOC_TARGET") + # Download the latest release binary + gh release download "$DEVEL" -p "$release_archive" -O "$tmpdir/$release_archive" + # Extract and remove the top-level directory + tar -C "$tmpdir" -xf "$tmpdir/$release_archive" --strip-components=1 + + mkdir -p built-docs + cp -rT "$tmpdir/doc/html" built-docs/devel + cp -rT "$tmpdir/doc/html" built-docs + env: + GH_TOKEN: ${{ github.token }} + DEVEL: ${{ steps.versions.outputs.devel }} + + - uses: actions/upload-pages-artifact@v3 + with: + path: built-docs/ + + - id: deploy + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/publisher.yml b/.github/workflows/publisher.yml index f940fb7a183..5d0125ed76a 100644 --- a/.github/workflows/publisher.yml +++ b/.github/workflows/publisher.yml @@ -25,9 +25,6 @@ jobs: url: ${{ steps.release.outputs.url }} steps: - # Publish action needs a checkout - - uses: actions/checkout@v4 - - name: Obtain latest successful run id id: finder run: | @@ -48,17 +45,9 @@ jobs: WORKFLOW: ci.yml CONCLUSION: success GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} # Download the latest instance of artifacts from a build done previously - - name: Download generated docs - uses: actions/download-artifact@v4 - with: - run-id: ${{ steps.finder.outputs.run_id }} - # Keep up-to-date with ci.yml - name: Generated docs - path: doc/html - github-token: ${{ github.token }} - - name: Download generated source archive uses: actions/download-artifact@v4 with: @@ -87,12 +76,6 @@ jobs: path: release-staging github-token: ${{ github.token }} - - name: Publish docs - uses: JamesIves/github-pages-deploy-action@v4.6.1 - with: - branch: gh-pages - folder: doc/html - - id: release-files name: Create release manifest run: | diff --git a/doc/ci.rst b/doc/ci.rst index 7accfb59c84..b27c7273a69 100644 --- a/doc/ci.rst +++ b/doc/ci.rst @@ -96,8 +96,11 @@ was developed to leverage this. 4. Assuming that the staging branch passes all tests, ``devel`` is fast-forwarded to the staging branch head commit. -5. The "Publish" pipeline deploys the rendered documentation and binaries - generated by the testing pipeline earlier in the staging branch. +5. The "Publish" pipeline deploys the binaries generated by the testing pipeline + earlier in the staging branch. + +6. The "Deploy documentation" pipeline deploys the latest documentation to GitHub + Pages. Assumptions in CI design ------------------------ @@ -161,9 +164,6 @@ which are used by other pipelines, these artifacts are: * "source archive": generated from the git clone. -* "Generated docs": HTML documentation generated from source. This is the - ``doc/html`` folder after a ``koch docs`` run. - Changes to how these artifacts are packaged must be reviewed carefully to ensure that dependent pipelines will still function. @@ -245,6 +245,21 @@ Only one instance of this pipeline might be run at any given time, due to its mutating nature. Currently Github Action's ``concurrency`` feature is used to block multiple runs. +"Deploy documentation" +---------------------- + +As the name suggests, this pipeline constructs and deploys the documentation +website. This pipeline is dependent on the +`Release manifest tool <#tools-release-manifest-tool>`_. + +Unlike other pipelines, the code run will always be that of the latest commit +in ``devel`` branch. This means that the code should always be aware of +differences between versions of |NimSkull| being deployed. + +Currently, only one instance of this pipeline might be run at any given time +to prevent races. GitHub Actions's ``concurrency`` feature is used to +serialize the runs. + Development guidelines ====================== diff --git a/tools/release_manifest.nim b/tools/release_manifest.nim index 89e32cc1baf..5d1c14aaad9 100644 --- a/tools/release_manifest.nim +++ b/tools/release_manifest.nim @@ -255,6 +255,23 @@ proc addCommand(manifest: string, archiveData: varargs[string]) = # Serialize a new manifest writeFile(manifest, $database.serialize()) +proc getCommand(manifest, target: string): int = + ## Implementation for the `get` subcommand. + ## + ## :manifest: + ## The filename of the release manifest to inspect. + ## + ## :target: + ## The triplet of interest. + let database = json.parseFile(manifest).deserialize() + + let idx = database.triplet.find(target) + if idx < 0: + stderr.writeLine("error: target $1 could not be found in database") + return 1 + + stdout.writeLine(database.file[idx]) + func escapeDataForGithubActions(s: string): string = ## Escape the string `s` so that it can be used as data for workflow commands. # The list is obtained from here: @@ -314,6 +331,7 @@ type Help = "help" Add = "add" FilesToUpload = "files-to-upload" + Get = "get" Version = "version" Flag {.pure.} = enum @@ -363,6 +381,7 @@ Usage: $app [args]... Commands: add Add artifacts to the manifest files-to-upload List the files to be uploaded + get Get release artifact for a target version Print the release version help Display help for any subcommand @@ -393,6 +412,15 @@ Options: format such that it can be used in workflow commands (ie. set-output) without losing data. +$globalOpt +""" + + GetHelp = """ +Usage: $app get [options] [--] + +Print the artifact file name of the given target triplet. An error will be raised +if no artifact can be found for the given target. + $globalOpt """ @@ -432,6 +460,8 @@ proc printHelp(action: Action) = stdout.write(AddHelp % defaultHelpFormat) of FilesToUpload: stdout.write(FilesToUploadHelp % defaultHelpFormat) + of Get: + stdout.write(GetHelp % defaultHelpFormat) of Version: stdout.write(VersionHelp % defaultHelpFormat) @@ -504,6 +534,16 @@ proc dispatch(cli: Cli): int = result = 1 else: filesToUploadCommand(manifest, format.get) + of Get: + let manifest = cli.flags.getOrDefault(Flag.File, DefaultManifestFile) + if cli.args.len == 1: + result = getCommand(manifest, cli.args[0]) + else: + # No or more than one targets were given, print the help text and set failure. + if cli.args.len > 1: + stderr.writeLine("error: only one target is expected") + printHelp(cli.action) + result = 1 of Version: let manifest = cli.flags.getOrDefault(Flag.File, DefaultManifestFile) versionCommand(manifest)