testing.yml

name: Testing

permissions: read-all

on:
  push:
  pull_request:
  workflow_dispatch:
    inputs:
      longTests:
        description: 'Force run long tests'
        required: false
        type: boolean
      externalSystem:
        description: 'Force run tests which rely on external connectivity'
        required: false
        type: boolean

env:
  ACTIONS: 1
  LONG_TESTS: 0
  EXTERNAL_SYSTEM: 0
  nvd_api_key: ${{ secrets.NVD_API_KEY }}

jobs:
  docs:
    name: Documentation
    permissions:
      contents: read
    if: |
      ! github.event.pull_request.user.login == 'github-actions[bot]' ||
      ! (
        startsWith(github.head_ref, 'chore-sbom-py') ||
        contains(
          fromJSON('["chore-js-dependencies","chore-precommit-config","chore-spdx-header"]'),
          github.head_ref
        )
      )
    runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
        with:
          disable-sudo: true
          egress-policy: block
          allowed-endpoints: >
            files.pythonhosted.org:443
            github.com:443
            pypi.org:443

      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
        with:
          python-version: '3.11'
          cache: 'pip'
          cache-dependency-path: 'doc/requirements.txt'
      - name: Install doc dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install --upgrade setuptools
          python -m pip install --upgrade wheel
          python -m pip install --upgrade -r doc/requirements.txt
      - name: Build docs
        run: |
          cd doc/
          sphinx-build -b html . _build

  tests:
    name: Linux tests
    permissions:
      contents: read
    runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
    strategy:
      matrix:
        python: ['3.8', '3.9', '3.11', '3.12']
    timeout-minutes: 90
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
        with:
          egress-policy: block
          allowed-endpoints: >
            access.redhat.com:443
            archives.fedoraproject.org:443
            azure.archive.ubuntu.com:80
            curl.se:443
            epss.cyentia.com:443
            esm.ubuntu.com:443
            files.pythonhosted.org:443
            ftp.fr.debian.org:80
            github.com:443
            gitlab.com:443
            mirror.cveb.in:443
            mirror.cveb.in:80
            motd.ubuntu.com:443
            nvd.nist.gov:443
            osv-vulnerabilities.storage.googleapis.com:443
            packages.microsoft.com:443
            ppa.launchpadcontent.net:443
            pypi.org:443
            raw.githubusercontent.com:443
            release-monitoring.org:443
            rpmfind.net:443
            security-tracker.debian.org:443
            services.nvd.nist.gov:443
            storage.googleapis.com:443
            www.cisa.gov:443
            www.sqlite.org:443

      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
        with:
          python-version: ${{ matrix.python }}
          cache: 'pip'

      - name: "Skip tests if this is an automated sbom job"
        env:
          COMMIT_VAR: ${{ startsWith(github.head_ref, 'chore-sbom-py') && github.event.pull_request.user.login == 'github-actions[bot]' }}
        run: |
          if ${COMMIT_VAR} == true; then
            echo "sbom=true" >> $GITHUB_ENV
            echo "sbom set to true"
          else
            echo "sbom=false" >> $GITHUB_ENV
            echo "sbom set to false"
          fi

      - name: Get date
        id: get-date
        run: |
          echo "date=$(/bin/date -u "+%Y%m%d")" >> $GITHUB_OUTPUT
          echo "yesterday=$(/bin/date -d "-1 day" -u "+%Y%m%d")" >> $GITHUB_OUTPUT
      - name: Print Cache Keys
        run: |
          echo "Today's Cache Key: Linux-cve-bin-tool-${{ steps.get-date.outputs.date }}"
          echo "Yesterday's Cache Key: Linux-cve-bin-tool-${{ steps.get-date.outputs.yesterday }}"
      - name: Get today's cached database
        uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
        id: todays-cache
        with:
          path: cache
          key: Linux-cve-bin-tool-${{ steps.get-date.outputs.date }}
      - name: Get yesterday's cached database if today's is not available
        uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
        if: steps.todays-cache.outputs.cache-hit != 'true'
        with:
          path: cache
          key: Linux-cve-bin-tool-${{ steps.get-date.outputs.yesterday }}
      - name: Install cabextract
        if: env.sbom != 'true'
        run: sudo apt-get update && sudo apt-get install cabextract
      - name: Install OS dependencies for testing PDF
        if: env.sbom != 'true'
        run: sudo apt-get install build-essential libpoppler-cpp-dev pkg-config python3-dev
      - name: Install pdftotext, reportlab and cve-bin-tool
        if: env.sbom != 'true'
        run: |
          python -m pip install --upgrade pip
          python -m pip install --upgrade setuptools
          python -m pip install --upgrade wheel
          python -m pip install --upgrade pdftotext
          python -m pip install --upgrade reportlab
          python -m pip install --upgrade -r dev-requirements.txt
          python -m pip install --upgrade .
      - name: Try single CLI run of tool
        if: env.sbom != 'true'
        run: |
          [[ -e cache ]] && mkdir -p .cache && mv cache ~/.cache/cve-bin-tool
          NO_EXIT_CVE_NUM=1 python -m cve_bin_tool.cli test/assets/test-kerberos-5-1.15.1.out
          cp -r ~/.cache/cve-bin-tool cache
      - name: Run async tests
        if: env.sbom != 'true'
        run: >
          pytest -n 4 -v --durations=50
          --ignore=test/test_cli.py
          --ignore=test/test_cvedb.py
          --ignore=test/test_requirements.py
          --ignore=test/test_html.py
          --ignore=test/test_json.py
      - name: Run synchronous tests
        if: env.sbom != 'true'
        run: >
          pytest -v --durations=50
          test/test_cli.py
          test/test_cvedb.py

  long_tests:
    name: Long tests on Python 3.10
    permissions:
      contents: read
    if: |
      ! github.event.pull_request.user.login == 'github-actions[bot]' ||
      ! (
        startsWith(github.head_ref, 'chore-sbom-py') ||
        contains(
          fromJSON('["chore-update-table","chore-precommit-config","chore-spdx-header"]'),
          github.head_ref
        )
      )
    runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
    timeout-minutes: 120
    env:
      LONG_TESTS: 1
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
        with:
          egress-policy: block
          allowed-endpoints: >
            access.redhat.com:443
            api.codecov.io:443
            api.github.com:443
            archives.fedoraproject.org:443
            azure.archive.ubuntu.com:80
            cli.codecov.io:443
            codecov.io:443
            curl.se:443
            epss.cyentia.com:443
            esm.ubuntu.com:443
            files.pythonhosted.org:443
            ftp.fr.debian.org:80
            github.com:443
            gitlab.com:443
            mirror.cveb.in:443
            mirror.cveb.in:80
            motd.ubuntu.com:443
            nvd.nist.gov:443
            osv-vulnerabilities.storage.googleapis.com:443
            packages.microsoft.com:443
            ppa.launchpadcontent.net:443
            pypi.org:443
            raw.githubusercontent.com:443
            release-monitoring.org:443
            rpmfind.net:443
            security-tracker.debian.org:443
            services.nvd.nist.gov:443
            storage.googleapis.com:443
            uploader.codecov.io:443
            www.cisa.gov:443
            www.sqlite.org:443

      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
        with:
          python-version: '3.10'
          cache: 'pip'

      - name: "Skip tests if this is an automated sbom job"
        env:
          COMMIT_VAR: ${{ startsWith(github.head_ref, 'chore-sbom-py') && github.event.pull_request.user.login == 'github-actions[bot]' }}
        run: |
          if ${COMMIT_VAR} == true; then
            echo "sbom=true" >> $GITHUB_ENV
            echo "sbom set to true"
          else
            echo "sbom=false" >> $GITHUB_ENV
            echo "sbom set to false"
          fi

      - name: Get date
        id: get-date
        run: |
          echo "date=$(/bin/date -u "+%Y%m%d")" >> $GITHUB_OUTPUT
          echo "yesterday=$(/bin/date -d "-1 day" -u "+%Y%m%d")" >> $GITHUB_OUTPUT
      - name: Print Cache Keys
        run: |
          echo "Today's Cache Key: Linux-cve-bin-tool-${{ steps.get-date.outputs.date }}"
          echo "Yesterday's Cache Key: Linux-cve-bin-tool-${{ steps.get-date.outputs.yesterday }}"
      - name: Get today's cached database
        uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
        id: todays-cache
        with:
          path: cache
          key: Linux-cve-bin-tool-${{ steps.get-date.outputs.date }}
      - name: Get yesterday's cached database if today's is not available
        uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
        if: steps.todays-cache.outputs.cache-hit != 'true'
        with:
          path: cache
          key: Linux-cve-bin-tool-${{ steps.get-date.outputs.yesterday }}
      - uses: technote-space/get-diff-action@f27caffdd0fb9b13f4fc191c016bb4e0632844af # v6.1.2
        with:
          PATTERNS: |
            cve_bin_tool/*.py
            cve_bin_tool/data_sources/*.py
            cve_bin_tool/checkers/*.py
            test/condensed-downloads/*
          FILES: |
            cvedb.py
            test_scanner.py
            test_cli.py
            cli.py
      - uses: haya14busa/action-cond@94f77f7a80cd666cb3155084e428254fea4281fd # v1.2.1
        id: git-diff
        with:
          cond: ${{ (github.event.inputs.longTests == 'true') || (env.GIT_DIFF != '') }}
          if_true: '1'
          if_false: '0'
      - name: Install cabextract
        if: env.sbom != 'true'
        run: sudo apt-get update && sudo apt-get install cabextract
      - name: Install OS dependencies for testing PDF
        if: env.sbom != 'true'
        run: sudo apt-get install build-essential libpoppler-cpp-dev pkg-config python3-dev
      - name: Install pdftotext, reportlab and cve-bin-tool
        if: env.sbom != 'true'
        run: |
          python -m pip install --upgrade pip
          python -m pip install --upgrade setuptools
          python -m pip install --upgrade wheel
          python -m pip install --upgrade pdftotext
          python -m pip install --upgrade reportlab
          python -m pip install --upgrade -r dev-requirements.txt
          python -m pip install --editable .
      - name: Try single CLI run of tool
        if: env.sbom != 'true'
        run: |
          [[ -e cache ]] && mkdir -p .cache && mv cache ~/.cache/cve-bin-tool
          NO_EXIT_CVE_NUM=1 python -m cve_bin_tool.cli test/assets/test-kerberos-5-1.15.1.out
          cp -r ~/.cache/cve-bin-tool cache
      - name: Run async tests
        if: env.sbom != 'true'
        env:
          LONG_TESTS: ${{ steps.git-diff.outputs.value }}
        run: >
          pytest --cov --cov-append -n 4 -v --durations=50
          --ignore=test/test_cli.py
          --ignore=test/test_cvedb.py
          --ignore=test/test_requirements.py
          --ignore=test/test_html.py
          --ignore=test/test_json.py
      - name: Run synchronous tests
        if: env.sbom != 'true'
        env:
          LONG_TESTS: ${{ steps.git-diff.outputs.value }}
        run: >
          pytest -v --cov --cov-append --cov-report=xml --durations=50
          test/test_cli.py
          test/test_cvedb.py
      - name: Upload code coverage to codecov
        if: env.sbom != 'true'
        uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0
        with:
          files: ./coverage.xml
          flags: longtests
          name: codecov-umbrella
          fail_ci_if_error: false

  linux-mayfail:
    name: Tests that may fail due to network or HTML
    permissions:
      contents: read
    if: |
      ! github.event.pull_request.user.login == 'github-actions[bot]' ||
      ! (
        startsWith(github.head_ref, 'chore-sbom-py') ||
        contains(
          fromJSON('["chore-update-table","chore-precommit-config","chore-spdx-header"]'),
          github.head_ref
        )
      )
    runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }}
    timeout-minutes: 45
    env:
      EXTERNAL_SYSTEM: 1
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
        with:
          egress-policy: block
          allowed-endpoints: >
            access.redhat.com:443
            api.github.com:443
            azure.archive.ubuntu.com:80
            csrc.nist.gov:443
            curl.se:443
            epss.cyentia.com:443
            esm.ubuntu.com:443
            files.pythonhosted.org:443
            github.com:443
            gitlab.com:443
            mirror.cveb.in:443
            mirror.cveb.in:80
            motd.ubuntu.com:443
            nvd.nist.gov:443
            osv-vulnerabilities.storage.googleapis.com:443
            packages.microsoft.com:443
            playwright.azureedge.net:443
            ppa.launchpadcontent.net:443
            pypi.org:443
            release-monitoring.org:443
            scap.nist.gov:443
            security-tracker.debian.org:443
            services.nvd.nist.gov:443
            storage.googleapis.com:443
            www.cisa.gov:443
            www.sqlite.org:443

      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
        with:
          python-version: '3.10'
          cache: 'pip'
      - name: Get date
        id: get-date
        run: |
          echo "date=$(/bin/date -u "+%Y%m%d")" >> $GITHUB_OUTPUT
          echo "yesterday=$(/bin/date -d "-1 day" -u "+%Y%m%d")" >> $GITHUB_OUTPUT
      - name: Print Cache Keys
        run: |
          echo "Today's Cache Key: Linux-cve-bin-tool-${{ steps.get-date.outputs.date }}"
          echo "Yesterday's Cache Key: Linux-cve-bin-tool-${{ steps.get-date.outputs.yesterday }}"
      - name: Get today's cached database
        uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
        id: todays-cache
        with:
          path: cache
          key: Linux-cve-bin-tool-${{ steps.get-date.outputs.date }}
      - name: Get yesterday's cached database if today's is not available
        uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
        if: steps.todays-cache.outputs.cache-hit != 'true'
        with:
          path: cache
          key: Linux-cve-bin-tool-${{ steps.get-date.outputs.yesterday }}
      - uses: technote-space/get-diff-action@f27caffdd0fb9b13f4fc191c016bb4e0632844af # v6.1.2
        with:
          PATTERNS: |
            cve_bin_tool/data_sources/*.py
          FILES: |
            test_available_fix.py
            test_source_osv.py
            test_source_gad.py
            test_source_nvd.py
            test_cli.py
            test_nvd_api.py
            test_cvedb.py
            test_scanner.py
            cli.py
            nvd_api.py
            cvedb.py
      - uses: haya14busa/action-cond@94f77f7a80cd666cb3155084e428254fea4281fd # v1.2.1
        id: git-diff
        with:
          cond: ${{ (github.event.inputs.externalSystem == 'true') || (env.GIT_DIFF != '') }}
          if_true: '1'
          if_false: '0'
      - name: Install cve-bin-tool
        run: |
          python -m pip install --upgrade pip
          python -m pip install --upgrade setuptools
          python -m pip install --upgrade wheel
          python -m pip install --upgrade -r dev-requirements.txt
          python -m pip install --editable .
      - name: Install playwright dependencies for HTML tests
        run: |
          python -m playwright install chromium --with-deps
      - name: Try single CLI run of tool
        run: |
          [[ -e cache ]] && mkdir -p .cache && mv cache ~/.cache/cve-bin-tool
          NO_EXIT_CVE_NUM=1 python -m cve_bin_tool.cli test/assets/test-kerberos-5-1.15.1.out
          cp -r ~/.cache/cve-bin-tool cache
      - name: Run all tests which rely on external connectivity
        env:
          EXTERNAL_SYSTEM: ${{ steps.git-diff.outputs.value }}
        run: >
          pytest -v --durations=0
          test/test_source_osv.py
          test/test_source_gad.py
          test/test_source_nvd.py
          test/test_nvd_api.py
          test/test_cvedb.py
          test/test_available_fix.py
      - name: Run HTML tests
        run: pytest -v -n auto test/test_html.py --durations=0
      - name: Run json tests as they may fail due to NVD data validation issue or download timing
        run: >
          pytest -v --durations=0
          test/test_json.py

  windows_long_tests:
    name: Windows long tests
    permissions:
      contents: read
    if: |
      ! github.event.pull_request.user.login == 'github-actions[bot]' ||
      ! (
        startsWith(github.head_ref, 'chore-sbom-py') ||
        contains(
          fromJSON('["chore-update-table","chore-precommit-config","chore-spdx-header"]'),
          github.head_ref
        )
      )
    runs-on: windows-latest
    timeout-minutes: 120
    env:
      LONG_TESTS: 1
      NO_EXIT_CVE_NUM: 1
      PYTHONIOENCODING: 'utf8'
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
        with:
          egress-policy: audit

      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      - uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
        with:
          python-version: '3.12'
          cache: 'pip'
      - name: Get date
        id: get-date
        run: |
          echo "DATE=$(get-date -format "yyyyMMdd")" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
          echo "YESTERDAY=$(get-date (get-date).addDays(-1)  -format "yyyyMMdd")" | Out-File -FilePath $env:GITHUB_OUTPUT -Encoding utf8 -Append
      - name: Print Cache Keys
        run: |
          echo "Today's Cache Key: Linux-cve-bin-tool-${{ steps.get-date.outputs.DATE }}"
          echo "Yesterday's Cache Key: Linux-cve-bin-tool-${{ steps.get-date.outputs.YESTERDAY }}"
      - name: Get today's cached database
        uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
        id: todays-cache
        with:
          path: cache
          key: Linux-cve-bin-tool-${{ steps.get-date.outputs.DATE }}
          enableCrossOsArchive: true
      - name: Get yesterday's cached database if today's is not available
        uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
        if: steps.todays-cache.outputs.cache-hit != 'true'
        with:
          path: cache
          key: Linux-cve-bin-tool-${{ steps.get-date.outputs.YESTERDAY }}
          enableCrossOsArchive: true
      - name: Move cache to ~/.cache/cve-bin-tool
        run: |
          mkdir '~\.cache'
          if (Test-Path -Path cache) { mv cache '~\.cache\cve-bin-tool' }
      - name: Install cve-bin-tool
        run: |
          python -m pip install --upgrade pip
          python -m pip install --upgrade setuptools
          python -m pip install --upgrade wheel
          python -m pip install --upgrade -r dev-requirements.txt
          python -m pip install --upgrade .
      - name: Try single CLI run of tool
        run: |
          python -m cve_bin_tool.cli test/assets/test-kerberos-5-1.15.1.out
      - name: Run async tests
        run: >
          pytest --cov --cov-append -n 4 -v --durations=50
          --ignore=test/test_cli.py
          --ignore=test/test_cvedb.py
          --ignore=test/test_requirements.py
          --ignore=test/test_html.py
          --ignore=test/test_json.py
      - name: Run synchronous tests
        run: >
          pytest -v --cov --cov-append --cov-report=xml --durations=50
          test/test_cli.py
          test/test_cvedb.py
      - name: Cache conda
        uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
        env:
          # Increase to reset cache if requirements.txt file has not changed
          CACHE_NUMBER: 0
        with:
          path: ~/conda_pkgs_dir
          key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{
            hashFiles('requirements.txt') }}
      - uses: conda-incubator/setup-miniconda@d2e6a045a86077fb6cad6f5adf368e9076ddaa8d # v3.1.0
        with:
          auto-update-conda: true
          activate-environment: pdftotext
          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
      - name: Install pdftotext, reportlab and cve-bin-tool
        run: |
          conda install -c conda-forge python=3.9 poppler pdftotext
          python -m pip install --upgrade pip
          python -m pip install --upgrade setuptools
          python -m pip install --upgrade wheel
          python -m pip install --upgrade reportlab
          python -m pip install --upgrade -r dev-requirements.txt
          python -m pip install --upgrade .
      - name: Test PDF generation on Windows
        run: pytest test/test_output_engine.py -k test_output_pdf --cov --cov-append --cov-report=xml --durations=50
      - name: Upload code coverage to codecov
        uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0
        with:
          files: ./coverage.xml
          flags: win-longtests
          name: codecov-umbrella
          fail_ci_if_error: false