Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve configure script so ninja can be run without a wrapper #26379

Merged
merged 2 commits into from
May 10, 2023
Merged

Improve configure script so ninja can be run without a wrapper #26379

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
38575f2
Improve configure script so ninja can be run without a wrapper
ksperling-apple May 5, 2023
b8edd5c
Make the build-venv auto-activate fully instead of changing how zap-c…
ksperling-apple May 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
194 changes: 123 additions & 71 deletions scripts/configure
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,53 +33,59 @@
# the build directory, but an external directory can be specified using the
# --build-env-dir option. The build environment directory can be shared by any
# number of build directories, independently of target / tool chain.
#
# Project options can be passed in the usual GNU configure style (--enable-foo,
# --foo-bar=value) and are translated into GN build arguments. By default,
# configure will override the toolchain for the GN build using a 'custom'
# toolchain assembled from the usual environment variables (CC, CXX, AR, CFLAGS,
# CXXFLAGS, ...).

function usage() {
set -o pipefail
shopt -s extglob

function usage() { # status
info "Usage: $0 [OPTIONS] [--project=... [PROJECT OPTIONS]]"
info "Options:"
info " --build-env-dir=DIR Directory to create (host) build environment in"
info " --project=DIR Sub-directory to build, e.g. examples/lighting-app/linux"
exit 0
info " --build-env-dir=DIR Directory to create (host) build environment in"
info " --project=DIR Sub-directory to build, eg examples/lighting-app/linux"
info ""
info "Project options (mapped to GN build args):"
info " --enable-<ARG>[=no] Enables (or disables with '=no') a bool build arg"
info " --<ARG>=<VALUE> Sets a (non-bool) build arg to the given value"
info " GN argument names can be specified with '-' instead of '_' and prefixes"
info " like 'chip_' can be ommitted from names. For the full list of available"
info " build arguments, see the generated args.configured file."
info ""
info " By default, the toolchain for the GN build will be configured from the usual"
info " environment variables (CC, CXX, AR, CFLAGS, CXXFLAGS, ...), falling back to"
info " default tool names (CC=cc, ...). When using this script within an external"
info " build system, toolchain environment variables should be populated."
exit "$1"
}

function main() { # ...
set -o pipefail
CHIP_ROOT=$(cd "$(dirname "$0")/.." && pwd)
BUILD_ENV_DEPS=(
"${CHIP_ROOT}/scripts/setup/requirements.build.txt"
"${CHIP_ROOT}/scripts/setup/constraints.txt"
"${CHIP_ROOT}/scripts/setup/zap.version"
)

if [[ "$PWD" == "$CHIP_ROOT" ]]; then
BUILD_DIR="out/configured"
BUILD_ROOT="${CHIP_ROOT}/${BUILD_DIR}"
BUILD_ENV_DIR=".venv"
info "Configuring in-tree, will build in $BUILD_DIR using environment $BUILD_ENV_DIR"
else
BUILD_DIR="."
BUILD_ROOT="$PWD"
BUILD_ENV_DIR="build-env"
fi
# Parse global options, process VAR=VALUE style arguments, and collect project options
BUILD_ENV_DIR=
PROJECT=

# Parse main options but leave project options in $@
while [[ $# -gt 0 && -z "$PROJECT" ]]; do
PROJECT_ARGS=()
while [[ $# -gt 0 ]]; do
case "$1" in
--help) usage ;;
--help) usage 0 ;;
--build-env-dir=*) BUILD_ENV_DIR="${1#*=}" ;;
--project=*) PROJECT="${1#*=}" ;;
*) fail "Invalid argument: '$1'" ;;
+([A-Z_])=*) export "$1" ;;
*)
[[ -n "$PROJECT" ]] || fail "Invalid argument: '$1'"
PROJECT_ARGS+=("$1")
;;
esac
shift
done

# Ensure we have something to do
[[ -n "$PROJECT" || -n "$BUILD_ENV_DIR" ]] || usage 1

if [[ -n "$PROJECT" ]]; then
local subdir="$(cd "${CHIP_ROOT}/${PROJECT}" 2>/dev/null && pwd)"
[[ -n "$subdir" && -r "${subdir}/.gn" ]] || fail "Invalid project '${PROJECT}'"
Expand All @@ -89,7 +95,28 @@ function main() { # ...

check_binary gn GN
check_binary ninja NINJA
if ! activate_build_env; then

# Work out build and environment directories
if [[ "$PWD" == "$CHIP_ROOT" ]]; then
BUILD_DIR="out/configured"
NINJA_HINT="ninja -C ${BUILD_DIR}"
else
BUILD_DIR="."
NINJA_HINT="ninja"
fi

if [[ -n "$BUILD_ENV_DIR" ]]; then
mkdir -p "$BUILD_ENV_DIR"
BUILD_ENV_PATH="$(cd "$BUILD_ENV_DIR" && pwd)"
[[ -n "$BUILD_ENV_PATH" ]] || fail "Invalid build-env-dir '${BUILD_ENV_DIR}'"
BUILD_ENV_DIR="$BUILD_ENV_PATH" # absolute
else
BUILD_ENV_DIR="build-env" # relative to BUILD_DIR
BUILD_ENV_PATH="${BUILD_DIR}/${BUILD_ENV_DIR}"
fi

# Create the build environment if necessary
if ! check_build_env; then
check_python
configure_python_env
if ! check_binary zap-cli; then
Expand All @@ -98,15 +125,19 @@ function main() { # ...
finalize_build_env
fi

# Configure the project (if requested)
if [[ -z "$PROJECT" ]]; then
info "Build environment created. (Specify --project=DIR to configure a build.)"
return
fi

[[ "$BUILD_DIR" != "." ]] && info "Configuring in-tree, will build in ${BUILD_DIR}"

create_empty_pw_env
gn_generate "$@"
guess_toolchain
gn_generate "${PROJECT_ARGS[@]}"
create_ninja_wrapper
info "You can now run ./ninja-build"
info "You can now run ./ninja-build (or $NINJA_HINT)"
}

function create_empty_pw_env() {
Expand All @@ -123,61 +154,84 @@ function create_empty_pw_env() {
fi
}

function guess_toolchain() {
# There is no widely used standard command for the C++ compiler (analogous to
# `cc` for the C compiler), so if neither CC nor CXX are defined try to guess.
if [[ -z "$CC" && -z "$CXX" ]] && have_binary cc; then
local probe="$(cc -E - <<<'gnu=__GNUC__ clang=__clang__' 2>/dev/null)"
# Check for clang first because it also defines __GNUC__
if [[ "$probe" =~ clang=[1-9] ]] && have_binary clang && have_binary clang++; then
info "Guessing CC=clang CXX=clang++ because cc appears to be clang"
export CC=clang CXX=clang++
elif [[ "$probe" =~ gnu=[1-9] ]] && have_binary gcc && have_binary g++; then
info "Guessing CC=gcc CXX=g++ because cc appears to be gcc"
export CC=gcc CXX=g++
else
info "Unable to guess c++ compiler: $probe"
fi
fi
}

function gn_generate() { # [project options]
mkdir -p "${BUILD_ROOT}"
ensure_no_clobber "${BUILD_ROOT}/args.gn"
(
cd "${CHIP_ROOT}/${PROJECT}" # --root= doesn't work for gn args!

# Run gn gen with an empty args.gn first so we can list all arguments
info "Configuring gn build arguments (see $BUILD_DIR/args.configured for full list)"
echo "# ${CONFIGURE_MARKER}" >"${BUILD_ROOT}/args.gn"
gn -q gen "$BUILD_ROOT"

# Use the argument list to drive the mapping of our command line options to GN args
call_impl process_project_args <(gn args "$BUILD_ROOT" --list --json) "$@" >>"${BUILD_ROOT}/args.gn"
gn args "$BUILD_ROOT" --list >"${BUILD_ROOT}/args.configured"

# Now gn gen with the arguments we have configured.
info "Running gn gen to generate ninja files"
gn -q gen "$BUILD_ROOT"
)
mkdir -p "${BUILD_DIR}"
ensure_no_clobber "${BUILD_DIR}/args.gn"

# Pass --script-executable to all `gn` calls so scripts run in our venv
local gn=(gn --script-executable="${BUILD_ENV_DIR}/bin/python" --root="${CHIP_ROOT}/${PROJECT}")

# Run gn gen with an empty args.gn first so we can list all arguments
info "Configuring gn build arguments (see $BUILD_DIR/args.configured for full list)"
{
echo "# ${CONFIGURE_MARKER}"
echo "# project root: ${PROJECT}"
} >"${BUILD_DIR}/args.gn"
"${gn[@]}" -q gen "$BUILD_DIR"

# Use the argument list to drive the mapping of our command line options to GN args
call_impl process_project_args <("${gn[@]}" args "$BUILD_DIR" --list --json) "$@" >>"${BUILD_DIR}/args.gn"
"${gn[@]}" args "$BUILD_DIR" --list >"${BUILD_DIR}/args.configured"

# Now gn gen with the arguments we have configured.
info "Running gn gen to generate ninja files"
"${gn[@]}" -q gen "$BUILD_DIR"
}

function create_ninja_wrapper() {
# Note: "." != $BUILD_DIR for in-tree builds
local wrapper="ninja-build"
ensure_no_clobber "$wrapper"
cat >"$wrapper" <<END
#!/bin/bash -e
# ${CONFIGURE_MARKER}
cd "\$(dirname "\$0")"
source "${BUILD_ENV_DIR}/bin/activate"
exec "${NINJA}" -C "${BUILD_DIR}" "\$@"
END
{
echo "#!/bin/bash -e"
echo "# ${CONFIGURE_MARKER}"
if [[ "$BUILD_DIR" != "." ]]; then
echo 'args=(-C "$(dirname "$0")/'"${BUILD_DIR}"'")'
else
echo 'args=() dir="$(dirname "$0")"'
echo '[[ "$dir" != "." ]] && args=(-C "$dir")'
fi
echo 'exec ninja "${args[@]}" "$@"'
} >"$wrapper"
chmod a+x "$wrapper"
}

function activate_build_env() {
function check_build_env() {
generate_build_env_cksum # re-used by finalize_build_env
[[ -r "${BUILD_ENV_DIR}/.cksum" ]] || return 1
read -r <"${BUILD_ENV_DIR}/.cksum" || true
[[ -r "${BUILD_ENV_PATH}/.cksum" ]] || return 1
read -r <"${BUILD_ENV_PATH}/.cksum" || true
[[ "$REPLY" == "$CURRENT_ENV_CKSUM" ]] || return 1

[[ -r "${BUILD_ENV_DIR}/bin/activate" ]] || return 1
info "Using existing build environment: ${BUILD_ENV_DIR}"
source "${BUILD_ENV_DIR}/bin/activate"
PYTHON="python"
[[ -r "${BUILD_ENV_PATH}/bin/activate" ]] || return 1
info "Using existing build environment: ${BUILD_ENV_PATH}"
PYTHON="${BUILD_ENV_PATH}/bin/python"
}

function configure_python_env() {
progress "Setting up Python venv"
"$PYTHON" -m venv "$BUILD_ENV_DIR"
info "ok"
"$PYTHON" -m venv "$BUILD_ENV_PATH"
info "$BUILD_ENV_PATH"

progress "Installing Python build dependencies"
"${BUILD_ENV_DIR}/bin/pip" install --require-virtualenv --quiet --upgrade pip wheel
"${BUILD_ENV_DIR}/bin/pip" install --require-virtualenv --quiet \
"${BUILD_ENV_PATH}/bin/pip" install --require-virtualenv --quiet --upgrade pip wheel
"${BUILD_ENV_PATH}/bin/pip" install --require-virtualenv --quiet \
-r "${CHIP_ROOT}/scripts/setup/requirements.build.txt" \
-c "${CHIP_ROOT}/scripts/setup/constraints.txt"
info "ok"
Expand All @@ -190,9 +244,7 @@ function generate_build_env_cksum() {
}

function finalize_build_env() {
echo "$CURRENT_ENV_CKSUM" >"${BUILD_ENV_DIR}/.cksum"
source "${BUILD_ENV_DIR}/bin/activate"
PYTHON="python"
echo "$CURRENT_ENV_CKSUM" >"${BUILD_ENV_PATH}/.cksum"
}

function download_zap() {
Expand All @@ -206,8 +258,8 @@ function download_zap() {
local url="https://github.com/project-chip/zap/releases/download/${version}/zap-${platform}.zip"

progress "Installing zap-cli from $url"
call_impl download_and_extract_zip "$url" "${BUILD_ENV_DIR}/bin" zap-cli
chmod a+x "${BUILD_ENV_DIR}/bin/zap-cli" # ZipFile.extract() does not handle permissions
call_impl download_and_extract_zip "$url" "${BUILD_ENV_PATH}/bin" zap-cli
chmod a+x "${BUILD_ENV_PATH}/bin/zap-cli" # ZipFile.extract() does not handle permissions
info "ok"
}

Expand Down
47 changes: 20 additions & 27 deletions scripts/tools/zap/zap_execution.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,15 @@
#
MIN_ZAP_VERSION = '2023.4.27'

ZAP_INSTALL_BANNER = '*'*80 + """
* You may need to install zap from https://github.com/project-chip/zap/releases
* Please ensure one of these applies (see docs/guides/BUILDING.md for details):
* - `zap-cli` is in your Python venv or $PATH.
ksperling-apple marked this conversation as resolved.
Show resolved Hide resolved
* - `zap-cli` is in $ZAP_INSTALL_PATH
* Use this option if you installed zap but do not want to update $PATH.
* - Point $ZAP_DEVELOPMENT_PATH to your local copy of zap that you develop on.
""" + '*'*80


class ZapTool:
def __init__(self):
Expand All @@ -47,11 +56,15 @@ def __init__(self):
os.environ['ZAP_SKIP_REAL_VERSION'] = '1'
self.check_version = False # cannot check versions without dirtying tree
elif 'ZAP_INSTALL_PATH' in os.environ:
self.zap_start = [os.path.join(
os.environ['ZAP_INSTALL_PATH'], 'zap-cli')]
self.zap_start = [os.path.join(os.environ['ZAP_INSTALL_PATH'], 'zap-cli')]
self.working_directory = None
else:
self.zap_start = ['zap-cli']
# If we're running within a Python venv, check for a zap-cli binary there
ksperling-apple marked this conversation as resolved.
Show resolved Hide resolved
venv_zap_cli = os.path.join(sys.prefix, 'bin', 'zap-cli')
if sys.prefix != sys.base_prefix and os.path.isfile(venv_zap_cli):
self.zap_start = [venv_zap_cli]
else:
self.zap_start = ['zap-cli']
self.working_directory = None

def version_check(self, min_version=None):
Expand Down Expand Up @@ -81,18 +94,8 @@ def version_check(self, min_version=None):
f"Failed to find `Version: ....` line from {self.zap_start} --version")
sys.exit(1)
except FileNotFoundError as e:
print(
f'FAILED TO EXECUTE ZAP GENERATION: {e.strerror} - "{e.filename}"')
print('*'*80)
print('* You may need to install zap. Please ensure one of these applies:')
print(
'* - `zap-cli` is in $PATH. Install from https://github.com/project-chip/zap/releases')
print('* see docs/guides/BUILDING.md for details')
print('* - `zap-cli` is in $ZAP_INSTALL_PATH. Use this option if you')
print('* installed zap but do not want to update $PATH')
print('* - Point $ZAP_DEVELOPMENT_PATH to your local copy of zap that you')
print('* develop on (to use a developer build of zap)')
print('*'*80)
print(f'FAILED TO EXECUTE ZAP GENERATION: {e.strerror} - "{e.filename}"')
print(ZAP_INSTALL_BANNER)
sys.exit(1)

def parse_version_string(str):
Expand All @@ -109,16 +112,6 @@ def run(self, cmd: str, *extra_args: Tuple[str]):
subprocess.check_call(
self.zap_start + [cmd] + list(extra_args), cwd=self.working_directory)
except FileNotFoundError as e:
print(
f'FAILED TO EXECUTE ZAP GENERATION: {e.strerror} - "{e.filename}"')
print('*'*80)
print('* You may need to install zap. Please ensure one of these applies:')
print(
'* - `zap-cli` is in $PATH. Install from https://github.com/project-chip/zap/releases')
print('* see docs/guides/BUILDING.md for details')
print('* - `zap-cli` is in $ZAP_INSTALL_PATH. Use this option if you')
print('* installed zap but do not want to update $PATH')
print('* - Point $ZAP_DEVELOPMENT_PATH to your local copy of zap that you')
print('* develop on (to use a developer build of zap)')
print('*'*80)
print(f'FAILED TO EXECUTE ZAP GENERATION: {e.strerror} - "{e.filename}"')
print(ZAP_INSTALL_BANNER)
sys.exit(1)