Skip to content

CPP binary SDK packaging #3154

CPP binary SDK packaging

CPP binary SDK packaging #3154

Workflow file for this run

name: CPP binary SDK packaging
on:
push:
branches:
# Run a full packaging step any time a new branch is merged into main.
- main
schedule:
- cron: "0 8 * * *" # 8am UTC = 12am PST / 1am PDT
workflow_dispatch:
inputs:
preserveIntermediateArtifacts:
description: 'preserve intermediate artifacts?'
default: 0
verboseBuild:
description: 'verbose build?'
default: 0
skipIntegrationTests:
description: 'skip integration tests?'
default: 0
downloadPublicVersion:
description: 'public version # to test against'
downloadPreviousRun:
description: 'previous run # to test against'
use_expanded_matrix:
description: 'Use an expanded matrix?'
default: '0'
required: true
env:
# Packaging prerequisites
# Demumble version from March 22, 2022
demumbleVer: "df938e45c2b0e064fb5323d88b692d03b451d271"
# Use SHA256 for hashing files.
hashCommand: "sha256sum"
# Xcode version 15.1 is the version we build the SDK with.
# Our MacOS runners will use the version in /Applications/Xcode_${xcodeVersion}.app
xcodeVersion: "15.1"
# LLVM version with ARM MachO support has no version number yet.
llvmVer: "5f187f0afaad33013ba03454c4749d99b1362534"
GITHUB_TOKEN: ${{ github.token }}
jobs:
log_inputs:
name: log-inputs
runs-on: ubuntu-20.04
steps:
- name: log run inputs
run: |
if [[ -n "${{ github.event.inputs.downloadPublicVersion }}" ]]; then
echo "::warning ::Downloading public SDK package from https://dl.google.com/firebase/sdk/cpp/firebase_cpp_sdk_${{ github.event.inputs.downloadPublicVersion }}.zip"
elif [[ -n "${{ github.event.inputs.downloadPreviousRun }}" ]]; then
echo "::warning ::Downloading SDK package from previous run at https://github.com/firebase/firebase-cpp-sdk/actions/runs/${{ github.event.inputs.downloadPreviousRun }}"
fi
- name: log if skipping integration tests
if: |
github.event.inputs.skipIntegrationTests != 0 && github.event.inputs.skipIntegrationTests != ''
run: echo "::warning ::Skipping integration tests."
- name: log if preserving intermediate artifacts
if: |
github.event.inputs.preserveIntermediateArtifacts != 0 && github.event.inputs.preserveIntermediateArtifacts != '' &&
github.event.inputs.downloadPublicVersion == '' && github.event.inputs.downloadPreviousRun == ''
run: echo "::warning ::Intermediate artifacts will be preserved."
- name: log if verbose build enabled
if: |
github.event.inputs.verboseBuild != 0 && github.event.inputs.verboseBuild != '' &&
github.event.inputs.downloadPublicVersion == '' && github.event.inputs.downloadPreviousRun == ''
run: echo "::warning ::Verbose build enabled."
- name: log if expanded matrix enabled
if: github.event_name == 'schedule' || github.event.inputs.use_expanded_matrix == '1'
run: echo "::warning ::Expanded test matrix enabled."
build_tools:
name: build-tools-${{ matrix.tools_platform }}
runs-on: ${{ matrix.os }}
if: ${{ github.event.inputs.downloadPublicVersion == '' && github.event.inputs.downloadPreviousRun == '' }}
strategy:
matrix:
os: [ubuntu-20.04, macos-13]
include:
- os: ubuntu-20.04
tools_platform: linux
# Binutils 2.35.1 released Sep 19, 2020
binutils_version: "2.35.1"
- os: macos-13
tools_platform: darwin
# Binutils 2.35.1 released Sep 19, 2020
binutils_version: "2.35.1"
steps:
- name: setup Xcode version (macos)
if: runner.os == 'macOS'
run: sudo xcode-select -s /Applications/Xcode_${{ env.xcodeVersion }}.app/Contents/Developer
- name: Setup python
uses: actions/setup-python@v4
with:
python-version: 3.8
- name: Fetch and build binutils
run: |
set +e
# Retry up to 10 times because Curl has a tendency to timeout on
# Github runners.
for retry in {1..10} error; do
if [[ $retry == "error" ]]; then exit 5; fi
curl -L https://ftpmirror.gnu.org/binutils/binutils-${{ matrix.binutils_version }}.tar.xz --output binutils.tar.xz && break
sleep 300
done
set -e
tar -xf binutils.tar.xz
mv ./binutils-${{ matrix.binutils_version }} ./binutils-src
mkdir /tmp/binutils-output
cd binutils-src
./configure --enable-targets=all --prefix=/tmp/binutils-output
make -j2
make install
cd -
mkdir -p packaging-tools
cp -af /tmp/binutils-output/bin/* packaging-tools
- name: Cache LLVM (Mac only)
if: matrix.tools_platform == 'darwin'
id: cache_llvm
uses: actions/cache@v3
with:
path: llvm-src/llvm/build/bin
key: llvm-bin-${{matrix.tools_platform}}-${{env.xcodeVersion}}-${{env.llvmVer}}
- name: Fetch LLVM source (Mac only)
# Only fetch LLVM if we don't already have binaries from the cache.
if: ${{ matrix.tools_platform == 'darwin' && !steps.cache_llvm.outputs.cache-hit }}
uses: actions/checkout@v3
with:
repository: llvm/llvm-project
path: llvm-src
ref: ${{ env.llvmVer }}
- name: Build LLVM (Mac only)
# Only build LLVM if we don't already have binaries from the cache.
if: ${{ matrix.tools_platform == 'darwin' && !steps.cache_llvm.outputs.cache-hit }}
run: |
mkdir llvm-src/llvm/build
cd llvm-src/llvm/build
cmake -DLLVM_ENABLE_PROJECTS=clang -DCMAKE_BUILD_TYPE=Release ..
cmake --build . -j 3 --target llvm-objcopy --target llvm-nm --target llvm-ar
- name: Package LLVM output (Mac only)
if: matrix.tools_platform == 'darwin'
run: |
mkdir -p packaging-tools
cp -af llvm-src/llvm/build/bin/* packaging-tools
- name: fetch demumble
uses: actions/checkout@v3
with:
repository: nico/demumble
path: demumble-src
ref: ${{ env.demumbleVer }}
- name: build demumble
run: |
cd demumble-src
cmake .
cmake --build .
python demumble_test.py
cd -
mkdir -p packaging-tools
cp -af demumble-src/demumble packaging-tools
- name: archive tools
run: |
cd packaging-tools
ls
tar -czhf ../packaging-tools.tgz .
- name: upload artifacts
uses: actions/upload-artifact@v3
with:
name: packaging-tools-${{ matrix.tools_platform }}
path: packaging-tools.tgz
build_and_package_ios_tvos:
name: build-and-package-ios-tvos
runs-on: macos-13
if: ${{ github.event.inputs.downloadPublicVersion == '' && github.event.inputs.downloadPreviousRun == '' }}
steps:
- name: Store git credentials for all git commands
# Forces all git commands to use authenticated https, to prevent throttling.
shell: bash
run: |
git config --global credential.helper 'store --file /tmp/git-credentials'
echo 'https://${{ github.token }}@github.com' > /tmp/git-credentials
- name: setup Xcode version (macos)
if: runner.os == 'macOS'
run: sudo xcode-select -s /Applications/Xcode_${{ env.xcodeVersion }}.app/Contents/Developer
- name: fetch SDK
uses: actions/checkout@v3
with:
path: sdk-src
- name: Setup python
uses: actions/setup-python@v4
with:
python-version: 3.8
- name: install prerequisites
run: sdk-src/build_scripts/ios/install_prereqs.sh
- name: build sdk
run: |
python sdk-src/scripts/gha/build_ios_tvos.py -b firebase-cpp-sdk-ios-tvos-build -s sdk-src
sdk-src/build_scripts/ios/package.sh firebase-cpp-sdk-ios-tvos-build firebase-cpp-sdk-ios-tvos-package
sdk-src/build_scripts/tvos/package.sh firebase-cpp-sdk-ios-tvos-build firebase-cpp-sdk-ios-tvos-package
cd firebase-cpp-sdk-ios-tvos-package
tar -czhf ../firebase-cpp-sdk-ios-tvos-package.tgz .
- name: Print built libraries
shell: bash
run: |
find firebase-cpp-sdk-*-build -name "*.lib"
find firebase-cpp-sdk-*-build -name "*.dll"
find firebase-cpp-sdk-*-build -name "*.dylib"
find firebase-cpp-sdk-*-build -name "*.a"
find firebase-cpp-sdk-*-build -name "*.so"
find firebase-cpp-sdk-*-build -name "*.framework"
- name: Print package contents
shell: bash
run: |
find firebase-cpp-sdk-*-package -type f
- name: upload artifacts
uses: actions/upload-artifact@v3
with:
name: firebase-cpp-sdk-ios-tvos-package
path: firebase-cpp-sdk-ios-tvos-package.tgz
build_and_package_android:
name: build-and-package-android
runs-on: ubuntu-20.04
if: ${{ github.event.inputs.downloadPublicVersion == '' && github.event.inputs.downloadPreviousRun == '' }}
strategy:
fail-fast: false
steps:
- name: fetch SDK
uses: actions/checkout@v3
with:
path: sdk-src
- name: Store git credentials for all git commands
# Forces all git commands to use authenticated https, to prevent throttling.
shell: bash
run: |
git config --global credential.helper 'store --file /tmp/git-credentials'
echo 'https://${{ github.token }}@github.com' > /tmp/git-credentials
- name: Cache NDK
id: cache_ndk
uses: actions/cache@v3
with:
path: /tmp/android-ndk-r21e
key: android-ndk-${{ runner.os }}-r21e
- name: install prerequisites
run: sdk-src/build_scripts/android/install_prereqs.sh
- name: build sdk
run: |
sdk-src/build_scripts/android/build.sh firebase-cpp-sdk-android-build sdk-src
sdk-src/build_scripts/android/package.sh firebase-cpp-sdk-android-build firebase-cpp-sdk-android-package
cd firebase-cpp-sdk-android-package
tar -czhf ../firebase-cpp-sdk-android-package.tgz .
- name: Print built libraries
shell: bash
run: |
find firebase-cpp-sdk-*-build -name "*.lib"
find firebase-cpp-sdk-*-build -name "*.dll"
find firebase-cpp-sdk-*-build -name "*.dylib"
find firebase-cpp-sdk-*-build -name "*.a"
find firebase-cpp-sdk-*-build -name "*.so"
find firebase-cpp-sdk-*-build -name "*.framework"
- name: Print package contents
shell: bash
run: |
find firebase-cpp-sdk-*-package -type f
- name: upload artifacts
uses: actions/upload-artifact@v3
with:
name: firebase-cpp-sdk-android-package
path: firebase-cpp-sdk-android-package.tgz
build_desktop:
name: build-${{ matrix.sdk_platform }}-${{ matrix.architecture }}-${{ matrix.build_type }}-${{ matrix.msvc_runtime }}-${{ matrix.linux_abi }}
runs-on: ${{ matrix.os }}
if: ${{ github.event.inputs.downloadPublicVersion == '' && github.event.inputs.downloadPreviousRun == '' }}
strategy:
fail-fast: false
matrix:
os: [windows-latest, ubuntu-20.04, macos-13]
build_type: ["Release", "Debug"]
architecture: ["x64", "x86", "arm64"]
msvc_runtime: ["static", "dynamic"]
linux_abi: ["legacy", "c++11"]
python_version: [3.8]
include:
- os: windows-latest
vcpkg_triplet_suffix: "windows-static"
sdk_platform: "windows"
- os: windows-latest
msvc_runtime: "dynamic"
vcpkg_triplet_suffix: "windows-static-md"
sdk_platform: "windows"
- os: ubuntu-20.04
vcpkg_triplet_suffix: "linux"
additional_build_flags: ""
sdk_platform: "linux"
- os: macos-13
vcpkg_triplet_suffix: "osx"
additional_build_flags: "--target_format libraries"
sdk_platform: "darwin"
exclude:
- os: windows-latest
linux_abi: "c++11"
- os: macos-13
architecture: "x86"
- os: macos-13
msvc_runtime: "dynamic"
- os: macos-13
linux_abi: "c++11"
- os: macos-13
build_type: "Debug"
- os: ubuntu-20.04
msvc_runtime: "dynamic"
- os: ubuntu-20.04
build_type: "Debug"
- os: ubuntu-20.04
architecture: "arm64"
- os: windows-latest
architecture: "arm64"
steps:
- name: Store git credentials for all git commands
# Forces all git commands to use authenticated https, to prevent throttling.
shell: bash
run: |
git config --global credential.helper 'store --file /tmp/git-credentials'
echo 'https://${{ github.token }}@github.com' > /tmp/git-credentials
- name: setup Xcode version (macos)
if: runner.os == 'macOS'
run: sudo xcode-select -s /Applications/Xcode_${{ env.xcodeVersion }}.app/Contents/Developer
- uses: actions/checkout@v3
with:
submodules: true
- name: Set env variables for subsequent steps (all)
shell: bash
run: |
echo "VCPKG_RESPONSE_FILE=external/vcpkg_${{ matrix.vcpkg_triplet }}_response_file.txt" >> $GITHUB_ENV
echo "MATRIX_UNIQUE_NAME=${{ matrix.os }}-${{ matrix.build_type }}-${{ matrix.architecture }}-${{ matrix.python_version }}-${{ matrix.msvc_runtime }}-${{ matrix.linux_abi }}" >> $GITHUB_ENV
echo "SDK_NAME=${{ matrix.sdk_platform }}-${{ matrix.architecture }}-${{ matrix.build_type }}-${{ matrix.msvc_runtime }}-${{ matrix.linux_abi }}" >> $GITHUB_ENV
if [[ '${{ matrix.sdk_platform }}' == 'darwin' ]]; then
# If Mac, also hash vcpkg cache on Xcode version.
echo "VCPKG_EXTRA_HASH=-xcode${{env.xcodeVersion}}" >> $GITHUB_ENV
fi
- name: Add msbuild to PATH (windows)
if: startsWith(matrix.os, 'windows')
uses: microsoft/setup-msbuild@v1.1
- name: Cache vcpkg C++ dependencies
id: cache_vcpkg
uses: actions/cache@v3
with:
path: external/vcpkg/installed
key: dev-vcpkg-${{ matrix.architecture }}-${{ matrix.vcpkg_triplet_suffix }}-${{ matrix.msvc_runtime }}-${{ matrix.linux_abi }}-${{ hashFiles(format('{0}', env.VCPKG_RESPONSE_FILE)) }}-${{ hashFiles('.git/modules/external/vcpkg/HEAD') }}${{env.VCPKG_EXTRA_HASH}}
- name: Cache ccache files
if: startsWith(matrix.os, 'ubuntu') || startsWith(matrix.os, 'macos')
id: cache_ccache
uses: actions/cache@v3
with:
path: ccache_dir
key: dev-test-ccache-${{ env.MATRIX_UNIQUE_NAME }}
- name: Setup python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python_version }}
- name: Install Desktop SDK prerequisites
uses: nick-invision/retry@v2
with:
timeout_minutes: 15
max_attempts: 3
command: |
python scripts/gha/install_prereqs_desktop.py --gha_build --arch '${{ matrix.architecture }}' --ssl boringssl
- name: Export verbose flag
shell: bash
run: |
verbose_flag=
if [[ -n "${{ github.event.inputs.verboseBuild }}" && "${{ github.event.inputs.verboseBuild }}" -ne 0 ]]; then
verbose_flag=--verbose
fi
echo "VERBOSE_FLAG=${verbose_flag}" >> $GITHUB_ENV
# Run the build in the host OS default shell since Windows can't handle long path names in bash.
- name: Build desktop SDK
run: |
python scripts/gha/build_desktop.py --arch "${{ matrix.architecture }}" --config "${{ matrix.build_type }}" --msvc_runtime_library "${{ matrix.msvc_runtime }}" --linux_abi "${{ matrix.linux_abi }}" --build_dir out-sdk ${VERBOSE_FLAG} ${{ matrix.additional_build_flags }} --gha_build
- name: Archive SDK
shell: bash
run: |
cd out-sdk
find .. -type f -print > src_file_list.txt
# Remove intermediate build files (.o and .obj) files to save space.
find . -type f -name '*.o' -or -name '*.obj' -print0 | xargs -0 rm -f --
tar -czhf ../firebase-cpp-sdk-${{ env.SDK_NAME }}-build.tgz .
- name: Print built libraries
shell: bash
run: |
find out-* -name "*.lib"
find out-* -name "*.dll"
find out-* -name "*.dylib"
find out-* -name "*.a"
find out-* -name "*.so"
find out-* -name "*.framework"
- name: Inspect firebase libraries for cpu arch and msvc runtime.
shell: bash
run: |
python scripts/gha/inspect_built_libraries.py out-sdk/
continue-on-error: true
- name: upload artifacts
uses: actions/upload-artifact@v3
with:
name: firebase-cpp-sdk-${{ env.SDK_NAME }}-build
path: firebase-cpp-sdk-${{ env.SDK_NAME }}-build.tgz
package_desktop:
name: package-${{ matrix.sdk_platform }}${{ matrix.suffix }}
runs-on: ${{ matrix.runs_on_platform }}
needs: [build_tools, build_desktop]
if: ${{ github.event.inputs.downloadPublicVersion == '' && github.event.inputs.downloadPreviousRun == '' }}
strategy:
fail-fast: false
matrix:
sdk_platform: [linux, darwin, windows]
suffix: ['']
runs_on_platform: [ubuntu-20.04]
include:
# Split windows packaging into multiple runners.
- sdk_platform: windows
suffix: '-x86-Release-static'
runs_on_platform: ubuntu-20.04
- sdk_platform: windows
suffix: '-x86-Release-dynamic'
runs_on_platform: ubuntu-20.04
- sdk_platform: windows
suffix: '-x64-Release-static'
runs_on_platform: ubuntu-20.04
- sdk_platform: windows
suffix: '-x64-Release-dynamic'
runs_on_platform: ubuntu-20.04
- sdk_platform: windows
suffix: '-x86-Debug-static'
runs_on_platform: ubuntu-20.04
- sdk_platform: windows
suffix: '-x86-Debug-dynamic'
runs_on_platform: ubuntu-20.04
- sdk_platform: windows
suffix: '-x64-Debug-static'
runs_on_platform: ubuntu-20.04
- sdk_platform: windows
suffix: '-x64-Debug-dynamic'
runs_on_platform: ubuntu-20.04
- sdk_platform: darwin
runs_on_platform: macos-13
exclude:
- sdk_platform: windows
suffix: ''
- sdk_platform: darwin
runs_on_platform: ubuntu-20.04
steps:
- name: setup Xcode version (macos)
if: runner.os == 'macOS'
run: sudo xcode-select -s /Applications/Xcode_${{ env.xcodeVersion }}.app/Contents/Developer
- name: fetch SDK
uses: actions/checkout@v3
with:
path: sdk-src
- name: download artifact
uses: actions/download-artifact@v3
with:
# download-artifact doesn't support wildcards, but by default
# will download all artifacts. Sadly this is what we must do.
path: artifacts
- name: Setup python
uses: actions/setup-python@v4
with:
python-version: 3.8
- name: Install prerequisites
run: |
cd sdk-src
python scripts/gha/install_prereqs_desktop.py --ssl boringssl
cd ..
if [[ $(uname) == "Darwin"* ]]; then
brew install parallel
fi
- name: postprocess and package built SDK
run: |
mkdir -p bin
if [[ $(uname) == "Linux"* ]]; then
tools_platform=linux
else
tools_platform=darwin
fi
verbose_flag=
if [[ -n "${{ github.event.inputs.verboseBuild }}" && "${{ github.event.inputs.verboseBuild }}" -ne 0 ]]; then
verbose_flag=-v
fi
declare -a additional_flags
tar -xvzf artifacts/packaging-tools-${tools_platform}/packaging-tools.tgz -C bin
rm -rf artifacts/packaging-tools-*.tgz
chmod -R u+x bin
# To save space, delete any artifacts that we don't need for packaging.
for pkg in artifacts/firebase-cpp-sdk-*; do
if [[ "${pkg}" != "artifacts/firebase-cpp-sdk-${{ matrix.sdk_platform }}${{ matrix.suffix }}"* ]]; then
echo "Deleting unneeded artifact: ${pkg}"
rm -rf "${pkg}"
fi
done
for pkg in artifacts/firebase-cpp-sdk-${{ matrix.sdk_platform }}${{ matrix.suffix }}*-build/*.tgz; do
# determine the build variant based on the artifact filename
variant=$(sdk-src/build_scripts/desktop/get_variant.sh "${pkg}")
additional_flags=(${verbose_flag})
# Several build targets require explicitly-set binutils format to be passed
# to package.sh (and thus, to merge_libraries), or use LLVM binutils.
if [[ "${{ matrix.sdk_platform }}" == "darwin" ]]; then
# MacOS: use LLVM binutils for both X64 and ARM64
additional_flags+=(-L)
elif [[ "${{ matrix.sdk_platform }}" == "windows" && "${variant}" == *"/x64/"* ]]; then
# Windows x64: force input and output target format
additional_flags+=(-f pe-x86-64,pe-bigobj-x86-64)
elif [[ "${{ matrix.sdk_platform }}" == "windows" && "${variant}" == *"/x86/"* ]]; then
# Windows x86: force input and output target format
additional_flags+=(-f pe-i386,pe-bigobj-i386)
fi
sdk-src/build_scripts/desktop/package.sh -D -b ${pkg} -o firebase-cpp-sdk-${{ matrix.sdk_platform }}${{ matrix.suffix }}-package -p ${{ matrix.sdk_platform }} -t bin -d ${variant} -P python3 -j ${additional_flags[*]}
done
if [[ "${{ matrix.sdk_platform }}" == "darwin" ]]; then
# Darwin has a final step after all the variants are done,
# create universal libraries and frameworks.
sdk-src/build_scripts/other/package.sh sdk-src tmpdir_for_headers
sdk-src/build_scripts/desktop/finish_darwin.sh firebase-cpp-sdk-${{ matrix.sdk_platform }}${{ matrix.suffix }}-package tmpdir_for_headers/include bin
fi
cd firebase-cpp-sdk-${{ matrix.sdk_platform }}${{ matrix.suffix }}-package
tar -czhf ../firebase-cpp-sdk-${{ matrix.sdk_platform }}${{ matrix.suffix }}-package.tgz .
- name: Print package contents
shell: bash
run: |
find firebase-cpp-sdk-*-package -type f
- name: upload SDK zip
uses: actions/upload-artifact@v3
with:
name: firebase-cpp-sdk-${{ matrix.sdk_platform }}${{ matrix.suffix}}-package
path: firebase-cpp-sdk-${{ matrix.sdk_platform }}${{ matrix.suffix}}-package.tgz
- name: Check for unrenamed namespaces
shell: bash
run: |
set +e
if [[ "${{ matrix.sdk_platform }}" == "darwin" ]]; then
nm=bin/llvm-nm
elif [[ "${{ matrix.sdk_platform }}" == "windows" ]]; then
if [[ "${{ matrix.sdk_platform }}" == *"x64"* ]]; then
nm="bin/nm --target pe-x86-64"
else
nm="bin/nm --target pe-i386"
fi
else # sdk_platform = linux
nm=bin/nm
fi
${nm} `find firebase-cpp-sdk-*-package/libs/${{ matrix.sdk_platform }} -type f` | grep '^[0-9a-fA-F][0-9a-fA-F]* ' | cut -d' ' -f3- > raw_symbols.txt
if [[ "${{ matrix.sdk_platform }}" == "windows" ]]; then
cat raw_symbols.txt | sort | uniq | bin/demumble | grep '^[a-zA-Z_]*::' > cpp_symbols.txt
elif [[ "${{ matrix.sdk_platform }}" == "darwin" ]]; then
# remove leading _ on mach-o symbols
cat raw_symbols.txt | sort | uniq | sed 's/^_//' | bin/c++filt | grep '^[a-zA-Z_]*::' > cpp_symbols.txt
else # linux
cat raw_symbols.txt | sort | uniq | bin/c++filt | grep '^[a-zA-Z_]*::' > cpp_symbols.txt
fi
# cpp_symbols.txt contains a list of all C++ symbols, sorted and deduped
# get a list of all top-level namespaces (except system namespaces)
cat cpp_symbols.txt | cut -d: -f1 | sort | uniq > cpp_namespaces.txt
echo "List of all namespaces found:"
cat cpp_namespaces.txt
# Filter out renamed namespaces (f_b_*), standard namespaces, and the public namespace (firebase::).
# These are all specified in the respective scripts, package.sh and merge_libraries.py
echo $(sdk-src/build_scripts/desktop/package.sh -R)'.*' > allow_namespaces.txt
sdk-src/build_scripts/desktop/package.sh -N >> allow_namespaces.txt
python sdk-src/scripts/print_allowed_namespaces.py >> allow_namespaces.txt
allow_namespaces=$(cat allow_namespaces.txt | tr '\n' '|')
cat cpp_namespaces.txt | grep -vE "^(${allow_namespaces})$" > extra_namespaces.txt
# If there are any namespaces in this file, print an error.
if [[ -s extra_namespaces.txt ]]; then
echo '::error ::Unrenamed C++ namespaces in ${{ matrix.sdk_platform }}${{ matrix.suffix }} package:%0A'$(cat extra_namespaces.txt | tr '\n' ' ' | sed 's/ /%0A/g')
exit 1
fi
- name: cleanup build artifacts
if: |
(
(github.event.inputs.preserveIntermediateArtifacts == 0 || github.event.inputs.preserveIntermediateArtifacts == '')
&& github.event.inputs.downloadPublicVersion == ''
&& github.event.inputs.downloadPreviousRun == ''
)
# Remove the build artifacts that were consumed during this step of packaging.
uses: geekyeggo/delete-artifact@v2
with:
name: |
firebase-cpp-sdk-${{ matrix.sdk_platform }}${{ matrix.suffix }}*-build
failOnError: false
useGlob: true
download_sdk_package:
name: download-sdk-package
runs-on: ubuntu-20.04
needs: [log_inputs]
if: ${{ github.event.inputs.downloadPublicVersion != '' || github.event.inputs.downloadPreviousRun != '' }}
steps:
- name: fetch artifact from previous run
uses: dawidd6/action-download-artifact@v2
if: ${{ github.event.inputs.downloadPreviousRun != '' }}
with:
name: 'firebase_cpp_sdk.zip'
workflow: 'cpp-packaging.yml'
run_id: ${{ github.event.inputs.downloadPreviousRun }}
- name: fetch public SDK package from web
if: ${{ github.event.inputs.downloadPublicVersion != '' && github.event.inputs.downloadPreviousRun == '' }}
run: |
if [[ ! "${{ github.event.inputs.downloadPublicVersion }}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
echo Invalid version number: "${{ github.event.inputs.downloadPublicVersion }}"
exit 1
fi
set +e
# Retry up to 10 times because Curl has a tendency to timeout on
# Github runners.
for retry in {1..10} error; do
if [[ $retry == "error" ]]; then exit 5; fi
curl -L https://dl.google.com/firebase/sdk/cpp/firebase_cpp_sdk_${{ github.event.inputs.downloadPublicVersion }}.zip --output firebase_cpp_sdk.zip && break
sleep 300
done
set -e
- name: compute SDK hash
shell: bash
run: |
${{ env.hashCommand }} --tag firebase_cpp_sdk.zip > firebase_cpp_sdk_hash.txt
echo "::warning ::$(cat firebase_cpp_sdk_hash.txt)"
- name: upload hash
uses: actions/upload-artifact@v3
with:
name: firebase_cpp_sdk_hash.txt
path: firebase_cpp_sdk_hash.txt
- name: upload SDK zip
uses: actions/upload-artifact@v3
with:
name: firebase_cpp_sdk.zip
path: firebase_cpp_sdk.zip
merge_packages:
name: final-merge-packages
runs-on: ubuntu-20.04
if: ${{ github.event.inputs.downloadPublicVersion == '' && github.event.inputs.downloadPreviousRun == '' }}
needs: [build_and_package_ios_tvos, build_and_package_android, package_desktop, log_inputs]
steps:
- name: fetch SDK
uses: actions/checkout@v3
with:
path: sdk-src
- name: download artifact
uses: actions/download-artifact@v3
with:
# download-artifact doesn't support wildcards, but by default
# will download all artifacts. Sadly this is what we must do.
path: artifacts
- name: merge SDK packages
shell: bash
run: |
set -ex
mkdir -p firebase-cpp-sdk-final/firebase_cpp_sdk
for pkg in artifacts/firebase-cpp-sdk-*-package/*.tgz; do
tar -xzf "${pkg}" -C firebase-cpp-sdk-final/firebase_cpp_sdk
done
# Add the final files.
sdk-src/build_scripts/other/package.sh sdk-src firebase-cpp-sdk-final/firebase_cpp_sdk
# Zip up the package and grab a hash of the result.
cd firebase-cpp-sdk-final
# Save the hash of every file into the SDK package.
find firebase_cpp_sdk -type f -print0 | xargs -0 ${{ env.hashCommand }} --tag > file_hashes.txt
mv file_hashes.txt firebase_cpp_sdk/
# Zip up the SDK package recursively, preserving symlinks.
zip -9 -r -y ../firebase_cpp_sdk.zip firebase_cpp_sdk
cd ..
- name: compute SDK hash
shell: bash
run: |
${{ env.hashCommand }} --tag firebase_cpp_sdk.zip > firebase_cpp_sdk_hash.txt
echo "::warning ::$(cat firebase_cpp_sdk_hash.txt)"
- name: Print final package contents
shell: bash
run: |
cd firebase-cpp-sdk-final
find firebase_cpp_sdk -type f
- name: upload hash
uses: actions/upload-artifact@v3
with:
name: firebase_cpp_sdk_hash.txt
path: firebase_cpp_sdk_hash.txt
- name: upload SDK zip
uses: actions/upload-artifact@v3
with:
name: firebase_cpp_sdk.zip
path: firebase_cpp_sdk.zip
create_windows_only_package:
name: create-windows-only-package
runs-on: ubuntu-20.04
needs: [merge_packages]
steps:
- name: download SDK zip
uses: actions/download-artifact@v3
with:
name: firebase_cpp_sdk.zip
- name: unzip SDK and remove non-Windows files
run: |
set -x
unzip -q firebase_cpp_sdk.zip -d windows-sdk
cd windows-sdk
# Rename the top-level directory.
mv firebase_cpp_sdk firebase_cpp_sdk_windows
cd firebase_cpp_sdk_windows
# Remove all non-Windows files.
rm -rf xcframeworks frameworks libs/linux libs/ios libs/tvos libs/darwin libs/android Android
cat > readme_windows.md <<EOF
# Firebase C++ SDK - Windows only
This package contains only the subset of the Firebase C++ SDK needed for use on
Windows. For general SDK information, see the accompanying readme.md file.
EOF
cd ..
zip -9 -r -y ../firebase_cpp_sdk_windows.zip firebase_cpp_sdk_windows
cd ..
- name: compute Windows SDK hash
shell: bash
run: |
${{ env.hashCommand }} --tag firebase_cpp_sdk_windows.zip > firebase_cpp_sdk_windows_hash.txt
echo "::warning ::$(cat firebase_cpp_sdk_windows_hash.txt)"
- name: upload Windows hash
uses: actions/upload-artifact@v3
with:
name: firebase_cpp_sdk_windows_hash.txt
path: firebase_cpp_sdk_windows_hash.txt
- name: upload Windows SDK zip
uses: actions/upload-artifact@v3
with:
name: firebase_cpp_sdk_windows.zip
path: firebase_cpp_sdk_windows.zip
cleanup_packaging_artifacts:
# Clean up intermediate artifacts from packaging step.
# This can happen after the final package merge is finished.
name: cleanup-packaging-artifacts
runs-on: ubuntu-20.04
needs: [merge_packages]
if: |
(
(github.event.inputs.preserveIntermediateArtifacts == 0 || github.event.inputs.preserveIntermediateArtifacts == '')
&& github.event.inputs.downloadPublicVersion == ''
&& github.event.inputs.downloadPreviousRun == ''
)
steps:
- uses: geekyeggo/delete-artifact@v2
with:
name: |
packaging-tools-*
firebase-cpp-sdk-*-package
failOnError: false
useGlob: true
trigger_integration_tests:
# Trigger the integration_tests workflow.
needs: [merge_packages, download_sdk_package, create_windows_only_package, cleanup_packaging_artifacts]
if: (github.event.inputs.skipIntegrationTests == 0 || github.event.inputs.skipIntegrationTests == '') && !cancelled() && !failure()
runs-on: ubuntu-20.04
steps:
- name: Checkout repo
uses: actions/checkout@v3
- name: Setup python
uses: actions/setup-python@v4
with:
python-version: '3.8'
- name: Use expanded matrix
if: github.event_name == 'schedule' || github.event.inputs.use_expanded_matrix == '1'
run: |
echo "USE_EXPANDED_MATRIX=1" >> $GITHUB_ENV
- name: Generate token for GitHub API
# This step is necessary because the existing GITHUB_TOKEN cannot be used inside one workflow to trigger another.
#
# Instead, generate a new token here, using our GitHub App's private key and App ID (saved as Secrets).
#
# This method is preferred over the "personal access token" solution, as the GitHub App's scope is limited to just
# the firebase-cpp-sdk repository.
uses: tibdex/github-app-token@v1
id: generate-token
with:
app_id: ${{ secrets.WORKFLOW_TRIGGER_APP_ID }}
private_key: ${{ secrets.WORKFLOW_TRIGGER_APP_PRIVATE_KEY }}
- name: Use GitHub API to start workflow
shell: bash
run: |
pip install -r scripts/gha/python_requirements.txt
if [[ -z ${USE_EXPANDED_MATRIX} ]]; then
USE_EXPANDED_MATRIX=0
fi
verbose_flag=
if [[ -n "${{ github.event.inputs.verboseBuild }}" && "${{ github.event.inputs.verboseBuild }}" -ne 0 ]]; then
verbose_flag=-v
fi
set -e
if [[ "${{ github.event_name }}" == "schedule" ]]; then
# trigger integration tests and generate two reports for nightly workflow run: one for firestore, one for the rest.
python scripts/gha/trigger_workflow.py -t ${{ steps.generate-token.outputs.token }} \
-w integration_tests.yml \
-p test_packaged_sdk ${{ github.run_id }} \
-p use_expanded_matrix ${USE_EXPANDED_MATRIX} \
-p apis "analytics,app_check,auth,database,dynamic_links,functions,installations,messaging,remote_config,storage" \
-p test_pull_request nightly-packaging \
-s 10 \
-A ${verbose_flag}
python scripts/gha/trigger_workflow.py -t ${{ steps.generate-token.outputs.token }} \
-w integration_tests.yml \
-p test_packaged_sdk ${{ github.run_id }} \
-p use_expanded_matrix ${USE_EXPANDED_MATRIX} \
-p apis "firestore" \
-p test_pull_request nightly-packaging \
-s 10 \
-A ${verbose_flag}
else
# trigger integration tests
python scripts/gha/trigger_workflow.py -t ${{ steps.generate-token.outputs.token }} \
-w integration_tests.yml \
-p test_packaged_sdk ${{ github.run_id }} \
-p use_expanded_matrix ${USE_EXPANDED_MATRIX} \
-s 10 \
-A ${verbose_flag}
fi
attempt_retry:
name: "attempt-retry"
needs: [trigger_integration_tests]
runs-on: ubuntu-20.04
if: ${{ failure() && !cancelled() && github.event_name == 'schedule' }}
steps:
- name: Checkout repo
uses: actions/checkout@v3
- name: Setup python
uses: actions/setup-python@v4
with:
python-version: 3.8
- name: Install python deps
run: pip install -r scripts/gha/python_requirements.txt
# The default token can't run workflows, so get an alternate token.
- name: Generate token for GitHub API
uses: tibdex/github-app-token@v1
id: generate-token
with:
app_id: ${{ secrets.WORKFLOW_TRIGGER_APP_ID }}
private_key: ${{ secrets.WORKFLOW_TRIGGER_APP_PRIVATE_KEY }}
- name: Retry failed tests
run: |
echo "::warning ::Attempting to retry failed jobs"
python scripts/gha/trigger_workflow.py -t ${{ steps.generate-token.outputs.token }} \
-w retry-test-failures.yml \
-p run_id ${{ github.run_id }} \
-s 10 \
-A