Skip to content

Inference with metrics and timing evaluation #2

Inference with metrics and timing evaluation

Inference with metrics and timing evaluation #2

name: Inference with metrics and timing evaluation
on:
push:
paths:
- "**/**/*.onnx"
workflow_dispatch:
permissions:
contents: write
pull-requests: write
jobs:
select-runner:
runs-on: ubuntu-22.04
outputs:
runner: ${{ steps.select_runner.outputs.runner }}
steps:
- uses: actions/checkout@v4
- name: Select runner
id: select_runner
run: |
if [ "${{ github.repository }}" = "NeuroDesk/cvpr-sam-on-laptop-2024" ]; then
echo "runner=self-hosted" >> $GITHUB_OUTPUT
else
echo "runner=ubuntu-22.04" >> $GITHUB_OUTPUT
fi
list_models:
needs: [select-runner]
runs-on: ${{ needs.select-runner.outputs.runner }}
outputs:
model_list: ${{ steps.set_model_list.outputs.model_list }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.10'
- name: Set up environment and data from osf
id: set_model_list
run: |
python -m pip install --upgrade pip
python -m pip install osfclient
osf --project u8tny fetch checkpoints
model_dir=$(find checkpoints/ -type f -name *.onnx | cut -d/ -f2)
all_models=$(echo $model_dir | tr ' ' '\n' | cut -d. -f1 | sort | uniq)
model_list='['
for MODEL in $(echo "${all_models}"); do
model_list+="\"${MODEL}\","
done
model_list=$(sed '$s/,$//' <<< $model_list)
model_list+=']'
echo "model_list=${model_list}"
echo "model_list=${model_list}" >> $GITHUB_OUTPUT
infer_new_model:
needs: [select-runner, list_models]
if: ${{ needs.list_models.outputs.model_list != '[]' }}
runs-on: ${{ needs.select-runner.outputs.runner }}
strategy:
fail-fast: false
matrix:
models: ${{ fromJson(needs.list_models.outputs.model_list) }}
steps:
- name: Cleanup
shell: bash
run: |
pwd
ls -l
echo ${{ secrets.USERPWD }} | sudo -S rm -rf test_demo
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.10'
- name: Set up environment and data from osf for ${{ matrix.models }}
id: set_model_list
run: |
python -m pip install --upgrade pip
python -m pip install osfclient numpy nibabel pandas tqdm matplotlib scipy
osf --project u8tny fetch test_demo.tar.xz
osf --project u8tny fetch checkpoints/${{ matrix.models }}
tar -xvf test_demo.tar.xz
rm test_demo.tar.xz
mv checkpoints/${{ matrix.models }} work_dir/
mv imgs gts test_demo
- name: Run inference on ${{ matrix.models }}
shell: bash
run: |
mkdir segs && chmod -R 777 ./*
export DOCKER_IMAGE_NAME=hawken50
echo ${{ secrets.USERPWD }} | sudo -S docker build -f Dockerfile -t "$DOCKER_IMAGE_NAME" .
echo ${{ secrets.USERPWD }} | sudo -S docker container run -m 8G --name "$DOCKER_IMAGE_NAME" --rm "$DOCKER_IMAGE_NAME":latest /bin/bash -c "ls"
echo ${{ secrets.USERPWD }} | sudo -S docker container run -m 8G --name "$DOCKER_IMAGE_NAME" --rm "$DOCKER_IMAGE_NAME":latest /bin/bash -c "ls work_dir"
echo ${{ secrets.USERPWD }} | ./evaluate.sh .
- name: Display accuracy, efficiency, and running time
run: |
printf "Metrics:\n" >> $GITHUB_STEP_SUMMARY
cat ./metrics.csv >> $GITHUB_STEP_SUMMARY
printf "\n\n" >> $GITHUB_STEP_SUMMARY
printf "Running time:\n" >> $GITHUB_STEP_SUMMARY
cat ./running_time.csv >> $GITHUB_STEP_SUMMARY