Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ci(refactor): Refactor workflows and skip files #9696

Merged
merged 10 commits into from
Jun 5, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
8 changes: 6 additions & 2 deletions .github/scripts/install-platformio-esp32.sh
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,9 @@ function count_sketches(){ # count_sketches <examples-path>
if [[ "${sketchdirname}.ino" != "$sketchname" ]]; then
continue
fi
if [[ -f "$sketchdir/.test.skip" ]]; then
is_target=$(jq -r --arg target $target '.targets[$target]' $sketchdir/ci.json)
# If the target is listed as false, skip the sketch. Otherwise, include it.
if [[ "$is_target" == "false" ]]; then
continue
fi
echo $sketch >> sketches.txt
Expand Down Expand Up @@ -161,8 +163,10 @@ function build_pio_sketches(){ # build_pio_sketches <board> <options> <examples-
local sketchdir=$(dirname $sketch)
local sketchdirname=$(basename $sketchdir)
local sketchname=$(basename $sketch)
is_target=$(jq -r --arg target $target '.targets[$target]' $sketchdir/ci.json)
# If the target is listed as false, skip the sketch. Otherwise, include it.
if [ "${sketchdirname}.ino" != "$sketchname" ] \
|| [ -f "$sketchdir/.test.skip" ]; then
|| [[ "$is_target" == "false" ]]; then
continue
fi
sketchnum=$(($sketchnum + 1))
Expand Down
20 changes: 12 additions & 8 deletions .github/scripts/on-push.sh
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ function build(){
local chunk_index=$3
local chunks_cnt=$4
local build_log=$5
shift; shift; shift; shift; shift;
local sketches_file=$6
shift; shift; shift; shift; shift; shift;
local sketches=$*

local BUILD_SKETCH="${SCRIPTS_DIR}/sketch_utils.sh build"
Expand All @@ -23,6 +24,9 @@ function build(){
if [ "$OS_IS_LINUX" == "1" ]; then
args+=" -p $ARDUINO_ESP32_PATH/libraries"
args+=" -i $chunk_index -m $chunks_cnt"
if [ -n "$sketches_file" ]; then
args+=" -f $sketches_file"
fi
if [ $build_log -eq 1 ]; then
args+=" -l $build_log"
fi
Expand Down Expand Up @@ -50,6 +54,7 @@ fi
CHUNK_INDEX=$1
CHUNKS_CNT=$2
BUILD_LOG=$3
SKETCHES_FILE=$4
BUILD_PIO=0
if [ "$#" -lt 2 ] || [ "$CHUNKS_CNT" -le 0 ]; then
CHUNK_INDEX=0
Expand All @@ -69,7 +74,6 @@ fi

SCRIPTS_DIR="./.github/scripts"
if [ "$BUILD_PIO" -eq 0 ]; then
#source ${SCRIPTS_DIR}/install-arduino-ide.sh
source ${SCRIPTS_DIR}/install-arduino-cli.sh
source ${SCRIPTS_DIR}/install-arduino-core-esp32.sh

Expand All @@ -95,12 +99,12 @@ if [ "$BUILD_PIO" -eq 0 ]; then
fi

#build sketches for different targets
build "esp32s3" $FQBN_ESP32S3 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
build "esp32s2" $FQBN_ESP32S2 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
build "esp32c3" $FQBN_ESP32C3 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
build "esp32c6" $FQBN_ESP32C6 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
build "esp32h2" $FQBN_ESP32H2 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
build "esp32" $FQBN_ESP32 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
build "esp32s3" $FQBN_ESP32S3 "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
build "esp32s2" $FQBN_ESP32S2 "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
build "esp32c3" $FQBN_ESP32C3 "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
build "esp32c6" $FQBN_ESP32C6 "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
build "esp32h2" $FQBN_ESP32H2 "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"
build "esp32" $FQBN_ESP32 "$CHUNK_INDEX" "$CHUNKS_CNT" "$BUILD_LOG" "$SKETCHES_FILE" "$SKETCHES_ESP32"

if [ "$BUILD_LOG" -eq 1 ]; then
#remove last comma from the last JSON object
Expand Down
82 changes: 57 additions & 25 deletions .github/scripts/sketch_utils.sh
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
done

xtra_opts=$*
len=0

if [ -z $sketchdir ]; then
echo "ERROR: Sketch directory not provided"
Expand All @@ -64,26 +65,30 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
# precedence. Note that the following logic also falls to the default
# parameters if no arguments were passed and no file was found.

if [ -z $options ] && [ -f $sketchdir/cfg.json ]; then
if [ -z $options ] && [ -f $sketchdir/ci.json ]; then
# The config file could contain multiple FQBNs for one chip. If
# that's the case we build one time for every FQBN.

len=`jq -r --arg chip $target '.targets[] | select(.name==$chip) | .fqbn | length' $sketchdir/cfg.json`
fqbn=`jq -r --arg chip $target '.targets[] | select(.name==$chip) | .fqbn' $sketchdir/cfg.json`
else
len=`jq -r --arg target $target '.fqbn[$target] | length' $sketchdir/ci.json`
if [ $len -gt 0 ]; then
fqbn=`jq -r --arg target $target '.fqbn[$target] | sort' $sketchdir/ci.json`
fi
fi

if [ ! -z $options ] || [ $len -eq 0 ]; then
# Since we are passing options, we will end up with only one FQBN to
# build.

len=1

# Default FQBN options if none were passed in the command line.

esp32_opts="FlashMode=dio,PSRAM=enabled,PartitionScheme=huge_app"
esp32s2_opts="PSRAM=enabled,PartitionScheme=huge_app"
esp32s3_opts="PSRAM=opi,USBMode=default,PartitionScheme=huge_app"
esp32c3_opts="FlashMode=dio,PartitionScheme=huge_app"
esp32c6_opts="PartitionScheme=huge_app"
esp32h2_opts="PartitionScheme=huge_app"
esp32_opts="PSRAM=enabled,PartitionScheme=huge_app,FlashMode=dio"
esp32s2_opts="PSRAM=enabled,PartitionScheme=huge_app,FlashMode=dio"
esp32s3_opts="PSRAM=opi,USBMode=default,PartitionScheme=huge_app,FlashMode=dio"
esp32c3_opts="PartitionScheme=huge_app,FlashMode=dio"
esp32c6_opts="PartitionScheme=huge_app,FlashMode=dio"
esp32h2_opts="PartitionScheme=huge_app,FlashMode=dio"

# Select the common part of the FQBN based on the target. The rest will be
# appended depending on the passed options.
Expand Down Expand Up @@ -135,7 +140,14 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex

sketchname=$(basename $sketchdir)

if [[ -n $target ]] && [[ -f "$sketchdir/.skip.$target" ]]; then
# If the target is listed as false, skip the sketch. Otherwise, include it.
if [ -f $sketchdir/ci.json ]; then
is_target=$(jq -r --arg target $target '.targets[$target]' $sketchdir/ci.json)
else
is_target="true"
fi

if [[ "$is_target" == "false" ]]; then
echo "Skipping $sketchname for target $target"
exit 0
fi
Expand Down Expand Up @@ -247,35 +259,48 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
unset options
}

function count_sketches(){ # count_sketches <path> [target]
function count_sketches(){ # count_sketches <path> [target] [file]
local path=$1
local target=$2
local file=$3

if [ $# -lt 1 ]; then
echo "ERROR: Illegal number of parameters"
echo "USAGE: ${0} count <path> [target]"
fi

rm -rf sketches.txt
touch sketches.txt
if [ ! -d "$path" ]; then
touch sketches.txt
return 0
fi

local sketches=$(find $path -name *.ino | sort)
if [ -n "$file" ]; then
local sketches=$(cat $file)
else
local sketches=$(find $path -name *.ino | sort)
fi

local sketchnum=0
for sketch in $sketches; do
local sketchdir=$(dirname $sketch)
local sketchdirname=$(basename $sketchdir)
local sketchname=$(basename $sketch)
if [[ "$sketchdirname.ino" != "$sketchname" ]]; then
continue
elif [[ -n $target ]] && [[ -f "$sketchdir/.skip.$target" ]]; then
continue
else
echo $sketch >> sketches.txt
sketchnum=$(($sketchnum + 1))
elif [[ -n $target ]]; then
# If the target is listed as false, skip the sketch. Otherwise, include it.
if [ -f $sketchdir/ci.json ]; then
is_target=$(jq -r --arg target $target '.targets[$target]' $sketchdir/ci.json)
else
is_target="true"
fi
if [[ "$is_target" == "false" ]]; then
continue
fi
fi
echo $sketch >> sketches.txt
sketchnum=$(($sketchnum + 1))
done
return $sketchnum
}
Expand Down Expand Up @@ -319,6 +344,10 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
shift
log_compilation=$1
;;
-f )
shift
sketches_file=$1
;;
* )
break
;;
Expand All @@ -328,7 +357,7 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat

local xtra_opts=$*

if [ -z $chunk_index ] || [ -z $chunk_max ]; then
if [ -z "$chunk_index" ] || [ -z "$chunk_max" ]; then
echo "ERROR: Invalid chunk paramters"
echo "$USAGE"
exit 1
Expand All @@ -339,13 +368,18 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
return 1
fi

if [ "$chunk_index" -gt "$chunk_max" ] && [ "$chunk_max" -ge 2 ]; then
if [ "$chunk_index" -gt "$chunk_max" ] && [ "$chunk_max" -ge 2 ]; then
chunk_index=$chunk_max
fi

set +e
count_sketches "$path" "$target"
local sketchcount=$?
if [ -n "$sketches_file" ]; then
count_sketches "$path" "$target" "$sketches_file"
local sketchcount=$?
else
count_sketches "$path" "$target"
local sketchcount=$?
fi
set -e
local sketches=$(cat sketches.txt)
rm -rf sketches.txt
Expand All @@ -364,8 +398,6 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
else
start_index=$(( $chunk_index * $chunk_size ))
if [ "$sketchcount" -le "$start_index" ]; then
echo "Skipping job"
touch ~/.build_skipped
return 0
fi

Expand Down
3 changes: 1 addition & 2 deletions .github/scripts/tests_build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@ while [ ! -z "$1" ]; do
shift
done

#source ${SCRIPTS_DIR}/install-arduino-ide.sh
source ${SCRIPTS_DIR}/install-arduino-cli.sh
source ${SCRIPTS_DIR}/install-arduino-core-esp32.sh

Expand All @@ -72,7 +71,7 @@ fi

if [ $chunk_build -eq 1 ]; then
BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh chunk_build"
args+=" -p $test_folder"
args+=" -p $test_folder -i 0 -m 1"
else
BUILD_CMD="${SCRIPTS_DIR}/sketch_utils.sh build"
args+=" -s $test_folder/$sketch"
Expand Down
Loading
Loading