build_test.yml

# Copyright (c) the JPEG XL Project Authors. All rights reserved.
#
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.

# Workflow for building and running tests.

name: Build/Test *nix
on:
  merge_group:
  push:
    branches:
      - main
      - v*.*.x
  pull_request:
    types: [opened, reopened, labeled, unlabeled, synchronize]

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  build_test:
    name: ${{ startsWith(matrix.os, 'macos-') && 'MacOS' || 'Ubuntu' }} Build ${{ matrix.name }}
    if: ${{ !contains(github.event.pull_request.labels.*.name, 'CI:none') }}
    runs-on: ${{ matrix.os || 'ubuntu-latest' }}
    strategy:
      fail-fast: false
      matrix:
        # We have one job per "name" in the matrix. Attributes are set on the
        # specific job names.
        name: [release, debug, asan, msan, tsan, scalar]
        include:
          - name: release
            mode: release
            run_bench: true
            test_in_pr: true
            cmake_args: >-
              -DJPEGXL_TEST_TOOLS=ON
              -DJPEGLI_LIBJPEG_LIBRARY_VERSION="8.2.2"
              -DJPEGLI_LIBJPEG_LIBRARY_SOVERSION="8"
            # Track static stack size on build and check it doesn't exceed 3 kB.
            env_stack_size: 1
            max_stack: 2400
            # Conformance tooling test requires numpy.
            apt_pkgs: doxygen graphviz python3-numpy
          - name: lowprecision
            mode: release
            run_bench: true
            test_in_pr: true
            cmake_args: -DCMAKE_CXX_FLAGS=-DJXL_HIGH_PRECISION=0
          - name: debug
            # Runs on AVX3 CPUs require more stack than others. Make sure to
            # test on AVX3-enabled CPUs when changing this value.
            env_test_stack_size: 4000
          # Build scalar-only hwy instructions.
          - name: scalar
            mode: release
            cxxflags: -DHWY_COMPILE_ONLY_SCALAR -DFJXL_ENABLE_AVX2=0 -DFJXL_ENABLE_AVX512=0
          # Disabling optional features to speed up MSAN build a little bit.
          - name: msan
            skip_install: true
            cmake_args: >-
              -DJPEGXL_ENABLE_DEVTOOLS=OFF -DJPEGXL_ENABLE_PLUGINS=OFF
              -DJPEGXL_ENABLE_VIEWERS=OFF
            apt_pkgs: clang-15
            cc: clang-15
            cxx: clang++-15
          - name: asan
            skip_install: true
          - name: tsan
            skip_install: true
          - name: coverage
            env_test_stack_size: 2048
            skip_install: true
          # Build with support for decoding to JPEG bytes disabled. Produces a
          # smaller build if only decoding to pixels is needed.
          - name: release-nojpeg
            mode: release
            cxxflags: -DJXL_DEBUG_ON_ABORT=0
            cmake_args: >-
              -DJPEGXL_ENABLE_TRANSCODE_JPEG=OFF
              -DJPEGXL_ENABLE_PLUGINS=OFF
              -DJPEGXL_ENABLE_VIEWERS=OFF
          # Build with jxl_cms based on lcms2 library.
          - name: release-lcms2
            mode: release
            cmake_args: >-
              -DJPEGXL_ENABLE_SKCMS=OFF
          - name: release-system-lcms2
            mode: release
            cmake_args: >-
              -DJPEGXL_ENABLE_SKCMS=OFF
              -DJPEGXL_FORCE_SYSTEM_LCMS2=ON
            apt_pkgs: liblcms2-dev
            # static build is impossible
            skip_install: true
          # Build optimized for binary size, all features not needed for
          # reconstructing pixels is disabled.
          - name: release:minimal
            mode: release
            cxxflags: -DJXL_DEBUG_ON_ABORT=0
            cmake_args: >-
              -DJPEGXL_ENABLE_TRANSCODE_JPEG=OFF
              -DJPEGXL_ENABLE_BOXES=OFF
              -DJPEGXL_ENABLE_PLUGINS=OFF
              -DJPEGXL_ENABLE_VIEWERS=OFF
          # Builds with gcc in release mode
          - name: release:gcc8
            os: ubuntu-20.04
            mode: release
            apt_pkgs: gcc-8 g++-8
            cmake_args: >-
              -DCMAKE_C_COMPILER=gcc-8 -DCMAKE_CXX_COMPILER=g++-8
          # Builds with clang-7 in release mode
          - name: release:clang-7
            os: ubuntu-20.04
            mode: release
            skip_install: true
            apt_pkgs: clang-7
            cc: clang-7
            cxx: clang++-7
          - name: release:osx
            os: macos-latest
            mode: release
            skip_install: true
            cmake_args: >-
              -DCMAKE_FIND_FRAMEWORK=NEVER

    env:
      CCACHE_DIR: ${{ github.workspace }}/.ccache
      # Whether we track the stack size.
      STACK_SIZE: ${{ matrix.env_stack_size }}
      TEST_STACK_LIMIT: ${{ matrix.env_test_stack_size }}
      WILL_TEST: ${{  github.event_name == 'push' || (github.event_name == 'pull_request' && matrix.name != 'coverage' && (matrix.test_in_pr || contains(github.event.pull_request.labels.*.name, 'CI:full'))) }}
      WILL_BUILD: ${{ github.event_name == 'push' || (github.event_name == 'pull_request' && matrix.name != 'coverage') }}
      WILL_BENCH: ${{ github.event_name != 'merge_group' && matrix.run_bench }}
      # Temporarily disable doc-building, re-enable when sphinx is working again.
      # WILL_DOC: ${{ github.event_name != 'merge_group' && matrix.name == 'release' }}
      WILL_DOC: false
      WILL_COV: ${{ github.event_name == 'push' && matrix.name == 'coverage' }}
      JPEGXL_OPT_DBG: true
      FASTER_MSAN_BUILD: 1

    steps:
    - name: Harden Runner
      uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0
      with:
        egress-policy: audit

    - name: Install build deps Ubuntu
      if: startsWith(matrix.os, 'macos-') == false
      run: |
        sudo rm -f /var/lib/man-db/auto-update
        sudo apt update
        sudo apt install -y \
          ccache \
          clang \
          cmake \
          graphviz \
          imagemagick \
          libbenchmark-dev \
          libbenchmark-tools \
          libbrotli-dev \
          libgdk-pixbuf2.0-dev \
          libgif-dev \
          libgtest-dev \
          libgtk2.0-dev  \
          libjpeg-dev \
          libjpeg-turbo-progs \
          libopenexr-dev \
          libpng-dev \
          libwebp-dev \
          ninja-build \
          pkg-config \
          xvfb \
          ${{ matrix.apt_pkgs }} \
        #
        echo "CC=${{ matrix.cc || 'clang' }}" >> $GITHUB_ENV
        echo "CXX=${{ matrix.cxx || 'clang++' }}" >> $GITHUB_ENV
    - name: Install build deps MacOS
      if: startsWith(matrix.os, 'macos-')
      run: |
        # Should be already installed:
        #  brew install brotli giflib jpeg-turbo libpng zlib
        # Not required, since we skip building documentation
        #  brew install doxygen
        brew install binutils ccache coreutils google-benchmark googletest ninja sdl2

    - name: Checkout the source
      uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3
      with:
        submodules: true
        fetch-depth: 2

    - name: Setup the Homebrew prefixes
      if: startsWith(matrix.os, 'macos-')
      run: |
          CMAKE_PREFIX_PATH=`brew --prefix brotli`:`brew --prefix giflib`:`brew --prefix google-benchmark`:`brew --prefix jpeg-turbo`:`brew --prefix libpng`:`brew --prefix sdl2`:`brew --prefix zlib`
          echo "CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}" >> $GITHUB_ENV

    - name: Suppress doxygen target
      if: matrix.name != 'release'
      run: |
        echo "TARGETS=all" >> $GITHUB_ENV

    - name: Setup the LLVM source path
      if: matrix.name == 'msan' && env.WILL_BUILD == 'true'
      run: |
        LLVM_ROOT=${GITHUB_WORKSPACE}/llvm_root
        mkdir -p ${LLVM_ROOT}
        echo "LLVM_ROOT=${LLVM_ROOT}" >> $GITHUB_ENV

    - name: Cache LLVM sources
      if: matrix.name == 'msan' && env.WILL_BUILD == 'true'
      uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
      with:
        path: ${{ env.LLVM_ROOT }}
        key: llvm

    - name: Checkout the LLVM source
      if: matrix.name == 'msan' && env.WILL_BUILD == 'true'
      uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # v4.1.3
      with:
        submodules: false
        repository: llvm/llvm-project
        ref: llvmorg-14.0.0 # NB: 15.0.0 does not build ¯\_(ツ)_/¯
        path: llvm_root

    - name: Sphinx dependencies
      # Dependencies for sphinx HTML documentation
      if: env.WILL_DOC == 'true'
      run: |
        pip3 install -r doc/sphinx/requirements.txt

    - name: Install gcovr
      if: env.WILL_COV == 'true'
      run: pip install gcovr

    - name: Git environment
      id: git-env
      run: |
        echo "parent=$(git rev-parse ${{ github.sha }}^)" >> $GITHUB_OUTPUT
      shell: bash

    - name: ccache
      uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
      with:
        path: ${{ env.CCACHE_DIR }}
        # When the cache hits the key it is not updated, so if this is a rebuild
        # of the same Pull Request it will reuse the cache if still around. For
        # either Pull Requests or new pushes to main, this will use the parent
        # hash as the starting point from the restore-keys entry.
        key: build-${{ runner.os }}-${{ github.sha }}-${{ matrix.name }}
        restore-keys: |
          build-${{ runner.os }}-${{ steps.git-env.outputs.parent }}-${{ matrix.name }}

    - name: Build
      if: env.WILL_BUILD == 'true'
      run: |
        mkdir -p ${CCACHE_DIR}
        echo "max_size = 200M" > ${CCACHE_DIR}/ccache.conf
        mode="${{ matrix.mode }}"
        build_tests=$([ "$WILL_TEST" == "true" ] && echo "ON" || echo "OFF")
        [[ -n "${mode}" ]] || mode="${{ matrix.name }}"
        ./ci.sh ${mode} -DJPEGXL_FORCE_SYSTEM_BROTLI=ON \
          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
          -DCMAKE_C_COMPILER_LAUNCHER=ccache \
          -DBUILD_TESTING=${build_tests} \
          ${{ matrix.cmake_args }}
      env:
        SKIP_TEST: 1
        CMAKE_CXX_FLAGS: ${{ matrix.cxxflags }}

    - name: Build stats
      if: env.WILL_BUILD == 'true'
      run: |
        awk '!/^#/ {total[$4]+=($2-$1);cntr[$4]+=1} END {for (key in total) print total[key]/cntr[key] " " key}' build/.ninja_log | sort -n | tail -n 25

    - name: ccache stats
      run: ccache --show-stats

    - name: Build stats ${{ matrix.name }}
      if: env.WILL_BUILD == 'true' && matrix.mode == 'release'
      run: |
        SHARED_LIB_EXT="${{ startsWith(matrix.os, 'macos-') && 'dylib' || 'so' }}"
        SELECT_BINUTILS="${{ startsWith(matrix.os, 'macos-') && '--binutils `brew --prefix binutils`/bin/' || '' }}"
        tools/scripts/build_stats.py --save build/stats.json \
          --max-stack ${{ matrix.max_stack || '0' }} ${SELECT_BINUTILS} \
          cjxl djxl libjxl.${SHARED_LIB_EXT} libjxl_dec.${SHARED_LIB_EXT}

    # Check that we can build the example project against the installed libs.
    - name: Install and build examples
      if: env.WILL_BUILD == 'true' && matrix.mode == 'release' && !matrix.skip_install
      run: |
        set -x
        sudo cmake --build build -- install
        cmake -Bbuild-example -Hexamples -G Ninja
        cmake --build build-example
        # Test that the built binaries run.
        echo -e -n "PF\n1 1\n-1.0\n\0\0\x80\x3f\0\0\x80\x3f\0\0\x80\x3f" > test.pfm
        build-example/encode_oneshot test.pfm test.jxl
        build-example/decode_oneshot test.jxl dec.pfm dec.icc

    # Run the tests on push and when requested in pull_request.
    - name: Test ${{ matrix.mode }}
      if: env.WILL_TEST == 'true'
      run: |
        ./ci.sh test ${{ matrix.ctest_args }}

    # Print the running time summary for the slowest tests.
    - name: Test runtime stats
      if: env.WILL_TEST == 'true'
      run: |
        sort build/Testing/Temporary/CTestCostData.txt -k 3 -n | tail -n 20 || true

    - name: Build HTML documentation (sphinx/readthetdocs)
      if: env.WILL_DOC == 'true'
      run: |
        cmake --build build -- rtd-html

    - name: Coverage report
      if: env.WILL_COV == 'true'
      run: |
        ./ci.sh coverage_report

    - name: Coverage upload to Codecov
      if: env.WILL_COV == 'true'
      uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673 # v4.5.0
      with:
        flags: unittests
        files: build/coverage.xml

    - name: Fast benchmark ${{ matrix.mode }}
      if: env.WILL_BENCH == 'true'
      run: |
        cat /proc/cpuinfo | grep MHz | sort | uniq
        lscpu
        BENCHMARK_NUM_THREADS=3 STORE_IMAGES=0 ./ci.sh fast_benchmark

    # Run gbench once, just to make sure it runs, not for actual benchmarking.
    # This doesn't work on MSAN because we use gbench library from the system
    # which is not instrumented by MSAN.
    - name: gbench check
      if: env.WILL_BENCH == 'true'
      run: |
        ./ci.sh gbench --benchmark_min_time=0