Reorganize HyperHDR libs (#887)

* Prepare for shared libs * Reorganize headers * Reorganize cmake * Add nanopb as an external submodule * Create HyperHDR utils-xz lib * More HyperHDR libs * Reorganize sources * Update Github scripts * Fix
awawa-dev · Jul 5, 2024 · d1264d1 · d1264d1
1 parent 12ed1c3
commit d1264d1
Show file tree

Hide file tree

Showing 602 changed files with 4,849 additions and 9,610 deletions.
diff --git a/.ci/ci_build.sh b/.ci/ci_build.sh
@@ -6,79 +6,103 @@ if [ "$SYSTEM_COLLECTIONID" != "" ]; then
 	echo "Azure detected"
 	CI_NAME="$(echo "$AGENT_OS" | tr '[:upper:]' '[:lower:]')"
 	CI_BUILD_DIR="$BUILD_SOURCESDIRECTORY"
+	CI_TYPE="azure"
 elif [ "$HOME" != "" ]; then
 	# GitHub Actions
 	echo "Github Actions detected"
 	CI_NAME="$(uname -s | tr '[:upper:]' '[:lower:]')"
 	CI_BUILD_DIR="$GITHUB_WORKSPACE"
+	CI_TYPE="github_action"
 else
 	# for executing in non ci environment
 	CI_NAME="$(uname -s | tr '[:upper:]' '[:lower:]')"
+	CI_TYPE="other"
 fi
 
-if [[ "$BUILD_ARCHIVES" == '0' ]]; then
-	IS_ARCHIVE_SKIPPED=" -DDO_NOT_BUILD_ARCHIVES=ON"
+if [ ${BUILD_ARCHIVES} = true ]; then
+	echo "Build the package archive"
+	IS_ARCHIVE_SKIPPED=" -DDO_NOT_BUILD_ARCHIVES=OFF"	
 else
-	IS_ARCHIVE_SKIPPED=" -DDO_NOT_BUILD_ARCHIVES=OFF"
+	echo "Do not build the package archive"
+	IS_ARCHIVE_SKIPPED=" -DDO_NOT_BUILD_ARCHIVES=ON"
 fi
 
 # set environment variables if not exists (debug)
 [ -z "${BUILD_TYPE}" ] && BUILD_TYPE="Release"
 
 [ -z "${USE_STANDARD_INSTALLER_NAME}" ] && USE_STANDARD_INSTALLER_NAME="OFF"
 
-echo "Platform: ${PLATFORM}, build type: ${BUILD_TYPE}, CI_NAME: $CI_NAME, docker image: ${DOCKER_IMAGE}, docker type: ${DOCKER_TAG}, is archive enabled: ${IS_ARCHIVE_SKIPPED}"
+echo "Platform: ${PLATFORM}, build type: ${BUILD_TYPE}, CI_NAME: $CI_NAME, docker image: ${DOCKER_IMAGE}, docker type: ${DOCKER_TAG}, is archive enabled: ${IS_ARCHIVE_SKIPPED}, use ccache: ${USE_CCACHE}, reset ccache: ${RESET_CACHE}"
+
+# clear ccache if neccesery
+if [ ${RESET_CACHE} = true ]; then
+	echo "Clearing ccache"
+	rm -rf .ccache || true
+	rm -rf build/.ccache || true
+fi
 
 # Build the package on osx or linux
 if [[ "$CI_NAME" == 'osx' || "$CI_NAME" == 'darwin' ]]; then
 	echo "Start: osx or darwin"
-	if [[ "$USE_CCACHE" == '1' ]]; then
-		echo "Using ccache"
 
-		# Init ccache
-		mkdir -p .ccache
-		cd .ccache
-
-		if [[ "$RESET_CACHE" == '1' ]]; then
-			echo "Clearing ccache"
-			rm -rf ..?* .[!.]* *
+	if [ ${USE_CCACHE} = true ]; then
+		echo "Using ccache"		
+		if [[ $(uname -m) == 'arm64' ]]; then
+			BUILD_OPTION=""
+		else
+			BUILD_OPTION="-DUSE_PRECOMPILED_HEADERS=OFF"
+			export CCACHE_COMPILERCHECK=content
 		fi
-
-		CCACHE_PATH=$PWD
-		cd ..
-        cachecommand="-DCMAKE_C_COMPILER_LAUNCHER=ccache ${IS_ARCHIVE_SKIPPED}"
-		export CCACHE_DIR=${CCACHE_PATH} && export CCACHE_COMPRESS=true && export CCACHE_COMPRESSLEVEL=1 && export CCACHE_MAXSIZE=400M
-		echo "CCache parameters: ${cachecommand}"		
-		ls -a .ccache
-
-		mkdir build || exit 1
-		cd build
-		ccache -p
-		cmake ${cachecommand} -DPLATFORM=${PLATFORM} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_OSX_DEPLOYMENT_TARGET=10.15 -DCMAKE_INSTALL_PREFIX:PATH=/usr/local ../ || exit 2
-		make -j $(sysctl -n hw.ncpu) || exit 3
-		sudo cpack || exit 3
-		exit 0;
-		exit 1 || { echo "---> HyperHDR compilation failed! Abort"; exit 5; }
 	else
 		echo "Not using ccache"
-		mkdir build || exit 1
-		cd build
-		cmake -DPLATFORM=${PLATFORM} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_OSX_DEPLOYMENT_TARGET=10.15 -DCMAKE_INSTALL_PREFIX:PATH=/usr/local ../ || exit 2
-		make -j $(sysctl -n hw.ncpu) || exit 3
-		sudo cpack || exit 3
-		exit 0;
-		exit 1 || { echo "---> HyperHDR compilation failed! Abort"; exit 5; }
+		BUILD_OPTION="-DUSE_CCACHE_CACHING=OFF"
 	fi
+
+	echo "Build option: ${BUILD_OPTION}"
+
+	mkdir -p build/.ccache
+	ls -a build/.ccache
+	cd build
+	ccache -z -d ./.ccache || true
+	cmake -DPLATFORM=${PLATFORM} ${BUILD_OPTION} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_OSX_DEPLOYMENT_TARGET=10.15 -DCMAKE_INSTALL_PREFIX:PATH=/usr/local ../ || exit 2
+	make -j $(sysctl -n hw.ncpu) || exit 3
+	sudo cpack || exit 3
+	ccache -sv -d ./.ccache || true
+	exit 0;
+	exit 1 || { echo "---> HyperHDR compilation failed! Abort"; exit 5; }
+
 elif [[ $CI_NAME == *"mingw64_nt"* || "$CI_NAME" == 'windows_nt' ]]; then
-	echo "Start: windows"
-
-	echo "Number of Cores $NUMBER_OF_PROCESSORS"
-	mkdir build || exit 1
+	echo "Start: windows"	
+	echo "Number of cores: $NUMBER_OF_PROCESSORS"
+
+	if [ ${USE_CCACHE} = true ]; then
+		echo "Using ccache"
+		BUILD_OPTION="${IS_ARCHIVE_SKIPPED}"
+	else
+		echo "Not using ccache"
+		BUILD_OPTION="-DUSE_CCACHE_CACHING=OFF ${IS_ARCHIVE_SKIPPED}"
+	fi
+
+	if [[ $CI_TYPE == "github_action" ]]; then
+		export CCACHE_COMPILERCHECK=content
+		export CCACHE_NOCOMPRESS=true
+		BUILD_OPTION="${BUILD_OPTION} -DCMAKE_GITHUB_ACTION=ON"
+	else
+		BUILD_OPTION="${BUILD_OPTION} -DCMAKE_GITHUB_ACTION=OFF"
+	fi
+
+	echo "Build option: ${BUILD_OPTION}"
+
+	mkdir -p build/.ccache
+
 	cd build
-	cmake -G "Visual Studio 17 2022" -A x64 -DPLATFORM=${PLATFORM} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_GITHUB_ACTION=1 ${IS_ARCHIVE_SKIPPED} ../ || exit 2
+	cmake -G "Visual Studio 17 2022" ${BUILD_OPTION} -A x64 -DPLATFORM=${PLATFORM} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} ../ || exit 2
+	./ccache.exe -zp || true
 	cmake --build . --target package --config Release -- -nologo -v:m -maxcpucount || exit 3
+	./ccache.exe -sv || true
+
 	exit 0;
-	exit 1 || { echo "---> Hyperhdr compilation failed! Abort"; exit 5; }
+
 elif [[ "$CI_NAME" == 'linux' ]]; then
 	echo "Compile Hyperhdr with DOCKER_IMAGE = ${DOCKER_IMAGE}, DOCKER_TAG = ${DOCKER_TAG} and friendly name DOCKER_NAME = ${DOCKER_NAME}"
 
@@ -87,83 +111,53 @@ elif [[ "$CI_NAME" == 'linux' ]]; then
 
 	# take ownership of deploy dir
 	mkdir -p ${CI_BUILD_DIR}/deploy
+	mkdir -p .ccache
+
+	if [ ${USE_CCACHE} = true ]; then
+		echo "Using ccache"
+		BUILD_OPTION="${IS_ARCHIVE_SKIPPED}"
+		cache_env="export CCACHE_DIR=/.ccache && ccache -z"
+		ls -a .ccache
+	else
+		echo "Not using ccache"		
+		BUILD_OPTION="-DUSE_CCACHE_CACHING=OFF ${IS_ARCHIVE_SKIPPED}"
+		cache_env="true"
+	fi
+
+	echo "Build option: ${BUILD_OPTION}, ccache: ${cache_env}"
 
 	if [[ "$DOCKER_TAG" == "ArchLinux" ]]; then
 		echo "Arch Linux detected"
-		cp cmake/arch/* .
+		cp cmake/linux/arch/* .
 		executeCommand="makepkg"
 		chmod -R a+rw ${CI_BUILD_DIR}/deploy
 		versionFile=`cat version`
 		sed -i "s/{VERSION}/${versionFile}/" PKGBUILD
-	fi
-
-	if [[ "$USE_CCACHE" == '1' ]]; then
-		echo "Using cache"
-
-		mkdir -p .ccache
-
-        cachecommand="-DCMAKE_C_COMPILER_LAUNCHER=ccache ${IS_ARCHIVE_SKIPPED}"
-
-		if [[ "$RESET_CACHE" == '1' ]]; then
-			echo "Clearing ccache"
-			cache_env="export CCACHE_SLOPPINESS=pch_defines,time_macros && export CCACHE_DIR=/.ccache && export CCACHE_NOCOMPRESS=true && export CCACHE_MAXSIZE=600M && cd /.ccache && rm -rf ..?* .[!.]* *"
-		else
-			cache_env="export CCACHE_SLOPPINESS=pch_defines,time_macros && export CCACHE_DIR=/.ccache && export CCACHE_NOCOMPRESS=true && export CCACHE_MAXSIZE=600M"
-		fi
-
-		echo "CCache parameters: ${cachecommand}, env: ${cache_env}"
-
-		if [[ "$DOCKER_TAG" == "ArchLinux" ]]; then
-			sed -i "s/{CACHE}/${cachecommand}/" PKGBUILD
-			echo "Using makepkg"
-			cat PKGBUILD
-			chmod -R a+rw ${CI_BUILD_DIR}/.ccache
+		if [ ${USE_CCACHE} = true ]; then
+			sed -i "s/{BUILD_OPTION}/${BUILD_OPTION} -DUSE_PRECOMPILED_HEADERS=OFF/" PKGBUILD
 		else
-			executeCommand="cd build && ( cmake ${cachecommand} -DPLATFORM=${PLATFORM} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DDEBIAN_NAME_TAG=${DOCKER_TAG} -DUSE_STANDARD_INSTALLER_NAME=${USE_STANDARD_INSTALLER_NAME} ../ || exit 2 )"
-			executeCommand+=" && ( make -j $(nproc) package || exit 3 )"
+			sed -i "s/{BUILD_OPTION}/${BUILD_OPTION}/" PKGBUILD
 		fi
-
-		ls -a .ccache
-		# run docker
-		docker run --rm \
-		-v "${CI_BUILD_DIR}/.ccache:/.ccache" \
-		-v "${CI_BUILD_DIR}/deploy:/deploy" \
-		-v "${CI_BUILD_DIR}:/source:ro" \
-		$REGISTRY_URL:$DOCKER_TAG \
-		/bin/bash -c "${cache_env} && cd / && mkdir -p hyperhdr && cp -r source/. /hyperhdr &&
-		cd /hyperhdr && mkdir build && (${executeCommand}) &&
-		(cp /hyperhdr/build/bin/h* /deploy/ 2>/dev/null || : ) &&
-		(cp /hyperhdr/build/Hyper* /deploy/ 2>/dev/null || : ) &&
-		(cp /hyperhdr/Hyper*.zst /deploy/ 2>/dev/null || : ) &&
-		ccache -s &&
-		exit 0;
-		exit 1 " || { echo "---> HyperHDR compilation failed! Abort"; exit 5; }
-		ls -a .ccache
+		chmod -R a+rw ${CI_BUILD_DIR}/.ccache
 	else
-		echo "Not using cache"
-
-		if [[ "$DOCKER_TAG" == "ArchLinux" ]]; then
-			sed -i "s/{CACHE}//" PKGBUILD
-			echo "Using makepkg"
-			cat PKGBUILD
-		else
-			executeCommand="cd build && ( cmake -DPLATFORM=${PLATFORM} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DDEBIAN_NAME_TAG=${DOCKER_TAG} -DUSE_STANDARD_INSTALLER_NAME=${USE_STANDARD_INSTALLER_NAME} ../ || exit 2 )"
-			executeCommand+=" && ( make -j $(nproc) package || exit 3 )"
-		fi
-
-		# run docker
-		docker run --rm \
-		-v "${CI_BUILD_DIR}/deploy:/deploy" \
-		-v "${CI_BUILD_DIR}:/source:ro" \
-		$REGISTRY_URL:$DOCKER_TAG \
-		/bin/bash -c "cd / && mkdir -p hyperhdr && cp -r source/. /hyperhdr &&
-		cd /hyperhdr && mkdir build && (${executeCommand}) &&
-		(cp /hyperhdr/build/bin/h* /deploy/ 2>/dev/null || : ) &&
-		(cp /hyperhdr/build/Hyper* /deploy/ 2>/dev/null || : ) &&
-		(cp /hyperhdr/Hyper*.zst /deploy/ 2>/dev/null || : ) &&
-		exit 0;
-		exit 1 " || { echo "---> HyperHDR compilation failed! Abort"; exit 5; }
+		executeCommand="cd build && ( cmake ${BUILD_OPTION} -DPLATFORM=${PLATFORM} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DDEBIAN_NAME_TAG=${DOCKER_TAG} -DUSE_STANDARD_INSTALLER_NAME=${USE_STANDARD_INSTALLER_NAME} ../ || exit 2 )"
+		executeCommand+=" && ( make -j $(nproc) package || exit 3 )"
 	fi
+
+	# run docker
+	docker run --rm \
+	-v "${CI_BUILD_DIR}/.ccache:/.ccache" \
+	-v "${CI_BUILD_DIR}/deploy:/deploy" \
+	-v "${CI_BUILD_DIR}:/source:ro" \
+	$REGISTRY_URL:$DOCKER_TAG \
+	/bin/bash -c "${cache_env} && cd / && mkdir -p hyperhdr && cp -r source/. /hyperhdr &&
+	cd /hyperhdr && mkdir build && (${executeCommand}) &&
+	(cp /hyperhdr/build/bin/h* /deploy/ 2>/dev/null || : ) &&
+	(cp /hyperhdr/build/Hyper* /deploy/ 2>/dev/null || : ) &&
+	(cp /hyperhdr/Hyper*.zst /deploy/ 2>/dev/null || : ) &&
+	(ccache -sv || true) &&
+	exit 0;
+	exit 1 " || { echo "---> HyperHDR compilation failed! Abort"; exit 5; }
 
 	# overwrite file owner to current user
 	sudo chown -fR $(stat -c "%U:%G" ${CI_BUILD_DIR}/deploy) ${CI_BUILD_DIR}/deploy

diff --git a/.github/workflows/push-master.yml b/.github/workflows/push-master.yml
@@ -2,12 +2,12 @@ name: HyperHDR CI Build
 
 on:
   push:
-  pull_request:
+#  pull_request:
 
 env:
-    USE_CACHE: "1"
-    RESET_CACHE: "0"
-    USE_CODEQL: "1"
+    USE_CACHE: ${{ vars.USE_CACHE && vars.USE_CACHE || true }}
+    RESET_CACHE: ${{ vars.RESET_CACHE && vars.RESET_CACHE || false }}
+    USE_CODEQL: ${{ vars.USE_CODEQL && vars.USE_CODEQL || false }}
     BUILD_ARCHIVES: ${{ startsWith(github.event.ref, 'refs/tags') && 1 || 0 }}
 
 jobs:
@@ -73,16 +73,16 @@ jobs:
 
       # generate cache id
       - name: Prepare ccache timestamp
-        if: env.USE_CACHE == '1'
+        if: env.USE_CACHE == 'true'
         id: ccache_cache_timestamp
         shell: cmake -P {0}
         run: |
           string(TIMESTAMP current_date "%Y-%m-%d-%H-%M-%S" UTC)
           file(APPEND "$ENV{GITHUB_OUTPUT}" "timestamp=${current_date}")
 
        # download cache
-      - name: ccache cache files
-        if: ( env.USE_CACHE == '1' )
+      - name: Download ccache files
+        if: env.USE_CACHE == 'true'
         uses: actions/cache@v4
         with:
            path: .ccache
@@ -151,26 +151,26 @@ jobs:
 
       # Generate cache id
       - name: Prepare ccache timestamp
-        if: env.USE_CACHE == '1'
+        if: env.USE_CACHE == 'true'
         id: ccache_cache_timestamp
         shell: cmake -P {0}
         run: |
             string(TIMESTAMP current_date "%Y-%m-%d-%H-%M-%S" UTC)
             file(APPEND "$ENV{GITHUB_OUTPUT}" "timestamp=${current_date}")
 
       # Download cache
-      - name: ccache cache files
-        if: ( env.USE_CACHE == '1' )
+      - name: Download ccache files
+        if: env.USE_CACHE == 'true'
         uses: actions/cache@v4
         with:
-            path: .ccache
+            path: build/.ccache
             key: macOS-ccache-${{ matrix.NICE_NAME }}-${{ steps.ccache_cache_timestamp.outputs.timestamp }}
             restore-keys: macOS-ccache-${{ matrix.NICE_NAME }}
 
       # Install deps
       - name: Install deps
         shell: bash
-        run: brew install -q qt@${{ matrix.QT_VERSION }} xz ccache zstd webp jpeg-turbo || true
+        run: brew install -q qt@${{ matrix.QT_VERSION }} xz ccache zstd webp jpeg-turbo flatbuffers pkg-config || true
 
       # Set env
       - name: Set QT5 env
@@ -234,12 +234,40 @@ jobs:
           pip install aqtinstall
           aqt install-qt -O ${{ runner.workspace }}/Qt windows desktop ${{ env.QT_VERSION }} win64_msvc2019_64 -m qtserialport
 
+      # Generate cache id
+      - name: Prepare ccache timestamp
+        if: env.USE_CACHE == 'true'
+        id: ccache_cache_timestamp
+        shell: cmake -P {0}
+        run: |
+            string(TIMESTAMP current_date "%Y-%m-%d-%H-%M-%S" UTC)
+            file(APPEND "$ENV{GITHUB_OUTPUT}" "timestamp=${current_date}")
+
+      # Download cache
+      - name: Download ccache files
+        if: env.USE_CACHE == 'true'
+        uses: actions/cache@v4
+        with:
+            path: build/.ccache
+            key: ${{ runner.os }}-ccache-${{ steps.ccache_cache_timestamp.outputs.timestamp }}
+            restore-keys: ${{ runner.os }}-ccache
+
       - name: Install TurboJPEG
         shell: powershell
         run: |
           Invoke-WebRequest -Method Get -Uri https://github.com/libjpeg-turbo/libjpeg-turbo/releases/download/3.0.3/libjpeg-turbo-3.0.3-vc64.exe -OutFile turbojpeg.exe -UseBasicParsing
           Start-Process -Wait turbojpeg.exe -ArgumentList "/S"
 
+      - name: Download CCache
+        shell: powershell
+        run: |
+          Invoke-WebRequest -Method Get -Uri https://github.com/ccache/ccache/releases/download/v4.10/ccache-4.10-windows-x86_64.zip -OutFile ccache.zip -UseBasicParsing
+
+      - name: Install CCache
+        shell: cmd
+        run: |
+          7z e ccache.zip -o.\build *.exe -r
+
       - name: Set up x64 build architecture environment
         shell: cmd
         run: call "${{env.VCINSTALLDIR}}\Auxiliary\Build\vcvars64.bat"
@@ -277,7 +305,8 @@ jobs:
   analyze:
     name: Analyze (CodeQL)
     runs-on: ubuntu-latest
-    if: ( true )
+    if: ${{ vars.USE_CODEQL == 'true' && vars.USE_CODEQL || false }}
+
     permissions:
       actions: read
       contents: read
@@ -291,11 +320,13 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
+        if: env.USE_CODEQL == 'true'
         with:
           submodules: true
 
       - name: Run composite CodeQL job for '${{matrix.language}}' language
         uses: ./.github/actions/codeql
+        if: env.USE_CODEQL == 'true'
         with:
           language: ${{matrix.language}}