Build PR 1870 «WIP: NWB: Archive logfile(s) on NWB export» (MichaelHuth) #375
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build Pull Request | |
| run-name: Build PR ${{ github.event.number }} «${{ github.event.pull_request.title }}» (${{ github.actor}}) | |
| on: | |
| pull_request: | |
| types: | |
| - opened | |
| - synchronize | |
| workflow_call: | |
| inputs: | |
| is_called_workflow: | |
| required: true | |
| default: false | |
| type: boolean | |
| description: | | |
| Should be set to true. This is used to determine of this workflow was called directly or | |
| from another workflow. | |
| do_instrumentation: | |
| required: false | |
| default: false | |
| type: boolean | |
| description: | | |
| Set to true to enable instrumentation of Igor code. The execution will be approx. 30% | |
| slower. | |
| secrets: | |
| GHA_MIES_CERTIFICATE_PIN: | |
| required: true | |
| GHA_MIES_FTP_ARTEFACT_STORAGE_USER: | |
| required: true | |
| GHA_MIES_FTP_ARTEFACT_STORAGE_PWD: | |
| required: true | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| BuildInstaller: | |
| name: 🏗 Build ${{ matrix.kind }} installer | |
| strategy: | |
| matrix: | |
| kind: | |
| # the normal installer | |
| - user | |
| # This installer is *only* indentend for developer internal use! | |
| - dev | |
| runs-on: [ self-hosted, Windows, Certificate ] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 # load all commits | |
| - name: Initial repo config | |
| run: tools/initial-repo-config.sh | |
| - name: Create installer | |
| run: tools/create-installer.sh ${{ fromJSON('["", "dev"]')[ matrix.kind == 'dev' ] }} | |
| - name: Sign installer | |
| run: tools/sign-installer.sh -p "${{ secrets.GHA_MIES_CERTIFICATE_PIN }}" | |
| - name: upload artifacts | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ always() }} | |
| with: | |
| name: BuildInstaller-${{ matrix.kind }}-assets | |
| path: | | |
| tools/installer/MIES-*.exe | |
| if-no-files-found: error | |
| Linting: | |
| name: 🔎 Linting | |
| runs-on: [ self-hosted, Linux, Docker ] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Code Checks | |
| run: tools/check-code.sh | |
| # straight from the documentation, see https://pre-commit.com/#github-actions-example | |
| - name: set PY | |
| run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV | |
| - uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pre-commit | |
| key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} | |
| - name: Pre commit | |
| run: tools/pre-commit/run.sh | |
| CompilationTest: | |
| name: | | |
| 👩🏾🔬 Compilation: ${{ matrix.source }} ${{ fromJSON('["","with XOP"]')[matrix.hardware] }} | |
| needs: | |
| - BuildInstaller | |
| - Linting | |
| strategy: | |
| matrix: | |
| source: [ git, installer ] | |
| hardware: [ false, true ] | |
| exclude: | |
| # the git source is always checked with hardware support | |
| - source: git | |
| hardware: false | |
| uses: ./.github/workflows/test-igor-workflow.yml | |
| with: | |
| job_name: | | |
| 👩🏾🔬 Compilation: ${{ matrix.source }} ${{ fromJSON('["","with XOP"]')[matrix.hardware] }} | |
| overwrite_job_name: ${{ inputs.is_called_workflow || false }} | |
| experiment: Packages/tests/Compilation/CompilationTester.pxp | |
| installer_artifact_name: BuildInstaller-dev-assets | |
| installer_flags: ${{ fromJSON('["-x skipHardwareXOPs",""]')[matrix.hardware] }} -s ${{ matrix.source }} | |
| artifact_name: CompilationTest-${{ matrix.source }}-${{ fromJSON('["no-hardware","hardware"]')[matrix.hardware] }}-assets | |
| expensive_checks: "1" | |
| Documentation: | |
| name: 👷 Documentation | |
| runs-on: [ self-hosted, Linux, Docker ] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 # load all commits | |
| - name: Initial repo config | |
| run: tools/initial-repo-config.sh | |
| - name: Build documentation | |
| run: tools/documentation/run.sh | |
| - name: upload artifacts | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ always() }} | |
| with: | |
| name: Documentation-assets | |
| path: | | |
| Packages/doc/*.zip | |
| if-no-files-found: error | |
| TestWithoutHardware: | |
| name: 🧪 Test ${{ matrix.name }} | |
| needs: | |
| - BuildInstaller | |
| - CompilationTest | |
| strategy: | |
| matrix: | |
| include: | |
| - experiment: Packages/tests/Basic/Basic.pxp | |
| name: Basic | |
| - experiment: Packages/tests/PAPlot/PAPlot.pxp | |
| name: PAPlot | |
| - experiment: Packages/tests/HistoricData/HistoricData.pxp | |
| name: HistoricData | |
| uses: ./.github/workflows/test-igor-workflow.yml | |
| with: | |
| job_name: 🧪 Test ${{ matrix.name }} | |
| overwrite_job_name: ${{ inputs.is_called_workflow || false }} | |
| experiment: ${{ matrix.experiment }} | |
| artifact_name: test-without-hw-assets | |
| expensive_checks: "1" | |
| instrument_tests: ${{ fromJson('["0", "1"]')[inputs.do_instrumentation] }} | |
| TestNI: | |
| name: 🧪 Test NI ${{ matrix.name }} | |
| needs: | |
| - BuildInstaller | |
| - CompilationTest | |
| strategy: | |
| matrix: | |
| include: | |
| - experiment: Packages/tests/HardwareBasic/HardwareBasic-NI.pxp | |
| name: HardwareBasic | |
| - experiment: Packages/tests/HardwareAnalysisFunctions/HardwareAnalysisFunctions-NI.pxp | |
| name: HardwareAnalysisFunctions | |
| uses: ./.github/workflows/test-igor-workflow.yml | |
| with: | |
| job_name: 🧪 Test NI ${{ matrix.name }} | |
| overwrite_job_name: ${{ inputs.is_called_workflow || false }} | |
| experiment: ${{ matrix.experiment }} | |
| target: "[ 'self-hosted', 'Windows', 'IgorPro', 'NI' ]" | |
| artifact_name: test-ni-assets | |
| expensive_checks: "1" | |
| instrument_tests: ${{ fromJson('["0", "1"]')[inputs.do_instrumentation] }} | |
| TestITC: | |
| name: 🧪 Test ITC ${{ matrix.name }} | |
| needs: | |
| - BuildInstaller | |
| - CompilationTest | |
| strategy: | |
| matrix: | |
| include: | |
| - experiment: Packages/tests/HardwareBasic/HardwareBasic.pxp | |
| name: HardwareBasic | |
| - experiment: Packages/tests/HardwareAnalysisFunctions/HardwareAnalysisFunctions.pxp | |
| name: HardwareAnalysisFunctions | |
| uses: ./.github/workflows/test-igor-workflow.yml | |
| with: | |
| job_name: 🧪 Test ITC ${{ matrix.name }} | |
| overwrite_job_name: ${{ inputs.is_called_workflow || false }} | |
| experiment: ${{ matrix.experiment }} | |
| target: "[ 'self-hosted', 'Windows', 'IgorPro', 'ITC' ]" | |
| artifact_name: test-itc-assets | |
| expensive_checks: "1" | |
| instrument_tests: ${{ fromJson('["0", "1"]')[inputs.do_instrumentation] }} | |
| ValidateNwb: | |
| name: 👮🏼 Validate NWBv2 | |
| runs-on: [ self-hosted, Linux, Docker ] | |
| needs: | |
| - TestITC | |
| - TestNI | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - name: Initial repo config | |
| run: tools/initial-repo-config.sh | |
| - name: Download ITC artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: test-itc-assets | |
| - name: Download NI artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: test-ni-assets | |
| - name: Validate and read NWBv2 files | |
| run: tools/nwb-read-tests/run.sh | |
| FTPUpload: | |
| name: 📦 FTP Upload Artifacts | |
| runs-on: [ self-hosted, Linux, Docker ] | |
| if: always() && !inputs.is_called_workflow | |
| needs: | |
| # All jobs that create new artifacts | |
| - BuildInstaller | |
| - CompilationTest | |
| - Documentation | |
| - TestWithoutHardware | |
| - TestNI | |
| - TestITC | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Initial repo config | |
| run: tools/initial-repo-config.sh | |
| - name: Download all artifacts | |
| id: download | |
| uses: actions/download-artifact@v3 | |
| with: | |
| path: artifacts | |
| - name: Flatten artifact structure | |
| run: | | |
| find "${{ steps.download.outputs.download-path }}" -mindepth 1 -maxdepth 1 -type d |\ | |
| xargs -I {} tools/ftp-upload/flatten-files.sh "{}" | |
| - name: Compress NWB artifacts | |
| run: | | |
| for dir in test-itc-assets test-ni-assets | |
| do | |
| tar --remove-files --use-compress-program=zstd -cvf $dir/NWB.tar.zst $dir/*nwb | |
| done | |
| working-directory: ${{ steps.download.outputs.download-path }} | |
| - name: Upload artifacts using FTP | |
| run: | | |
| tools/ftp-upload/upload-files.sh \ | |
| -s "${{ vars.GHA_MIES_FTP_ARTEFACT_STORAGE_DNS }}" \ | |
| -u "${{ secrets.GHA_MIES_FTP_ARTEFACT_STORAGE_USER }}" \ | |
| -p "${{ secrets.GHA_MIES_FTP_ARTEFACT_STORAGE_PWD }}" \ | |
| -d "${{ steps.download.outputs.download-path }}" \ | |
| -t "pr/$(echo "${{ github.event.pull_request.head.ref }}" | sed "s@/@_@g")/${{ github.event.pull_request.head.sha }}/${{ github.run_attempt }}" |