Skip to content
Inference with metrics and timing evaluation

Inference with metrics and timing evaluation #38

Sign in to view logs

Inference with metrics and timing evaluation

Inference with metrics and timing evaluation #38

Workflow file for this run

.github/workflows/model-inference-with-timing.yml at e749205
name: Inference with metrics and timing evaluation
on:
push:
paths:
- "**/**/*.onnx"
workflow_dispatch:
permissions:
contents: write
pull-requests: write
jobs:
select-runner:
runs-on: ubuntu-22.04
outputs:
runner: ${{ steps.select_runner.outputs.runner }}
steps:
- uses: actions/checkout@v4
- name: Select runner
id: select_runner
run: |
if [ "${{ github.repository }}" = "NeuroDesk/cvpr-sam-on-laptop-2024" ]; then
echo "runner=self-hosted" >> $GITHUB_OUTPUT
else
echo "runner=ubuntu-22.04" >> $GITHUB_OUTPUT
fi
list_models:
needs: [select-runner]
runs-on: ${{ needs.select-runner.outputs.runner }}
outputs:
model_list: ${{ steps.set_model_list.outputs.model_list }}
all_models: ${{ steps.set_model_list.outputs.all_models }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Set up environment and data from osf
id: set_model_list
run: |
python -m pip install --upgrade pip
python -m pip install osfclient
model_dir=$( osf --project u8tny list)
all_models=$( echo "$model_dir" | awk -F '/' '/osfstorage\/checkpoints/ {print $3}' | sort -u | tr '\n' ' ')
echo "all_models=${all_models}"
model_list='['
for MODEL in $(echo "${all_models}"); do
echo "MODEL=${MODEL}"
model_list+="\"${MODEL}\","
done
model_list=$(sed '$s/,$//' <<< $model_list)
model_list+=']'
echo "model_list=${model_list}"
echo "model_list=${model_list}" >> $GITHUB_OUTPUT
echo "all_models=${all_models}" >> $GITHUB_OUTPUT
infer_new_model:
needs: [select-runner, list_models]
if: ${{ needs.list_models.outputs.model_list != '[]' }}
runs-on: ${{ needs.select-runner.outputs.runner }}
strategy:
fail-fast: false
matrix:
models: ${{ fromJson(needs.list_models.outputs.model_list) }}
steps:
- name: Cleanup
shell: bash
run: |
pwd
ls -l
echo ${{ secrets.USERPWD }} | sudo -S rm -rf test_demo
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Set up environment and data from osf for ${{ matrix.models }}
id: set_model_list
run: |
python -m pip install --upgrade pip
python -m pip install osfclient numpy nibabel pandas tqdm matplotlib scipy
osf --project u8tny fetch test_demo.tar.xz
osf --project u8tny fetch checkpoints/${{ matrix.models }}/${{ matrix.models }}_encoder.onnx
osf --project u8tny fetch checkpoints/${{ matrix.models }}/${{ matrix.models }}_decoder.onnx
tar -xvf test_demo.tar.xz
rm test_demo.tar.xz
mv ${{ matrix.models }}_encoder.onnx ${{ matrix.models }}_decoder.onnx work_dir/
mv imgs gts test_demo
- name: Run inference on ${{ matrix.models }}
shell: bash
run: |
mkdir segs && chmod -R 777 ./*
export DOCKER_IMAGE_NAME=hawken50
echo ${{ secrets.USERPWD }} | sudo -S docker build -f Dockerfile -t "$DOCKER_IMAGE_NAME" .
echo ${{ secrets.USERPWD }} | sudo -S docker container run -m 8G --name "$DOCKER_IMAGE_NAME" --rm "$DOCKER_IMAGE_NAME":latest /bin/bash -c "ls"
echo ${{ secrets.USERPWD }} | sudo -S docker container run -m 8G --name "$DOCKER_IMAGE_NAME" --rm "$DOCKER_IMAGE_NAME":latest /bin/bash -c "ls work_dir"
echo ${{ secrets.USERPWD }} | ./evaluate.sh .
- name: Display accuracy, efficiency, and running time
run: |
printf "Metrics:\n" >> $GITHUB_STEP_SUMMARY
cat ./metrics.csv >> $GITHUB_STEP_SUMMARY
printf "\n\n" >> $GITHUB_STEP_SUMMARY
printf "Running time:\n" >> $GITHUB_STEP_SUMMARY
cat ./running_time.csv >> $GITHUB_STEP_SUMMARY
mkdir ${{ matrix.models }}_results
mv metrics.csv running_time.csv ${{ matrix.models }}_results/
- name: Archive metrics and running time
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.models }}_results
path: |
${{ matrix.models }}_results
plot_results:
needs: [list_models, infer_new_model]
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Set up environment and data from osf
run: |
python -m pip install --upgrade pip
python -m pip install numpy tqdm matplotlib scipy pandas
- name: Download result artifacts
uses: actions/download-artifact@v4
- name: Plot results
env:
IMAGE_BASE64: ""
run: |
python plot_results.py -models ${{ needs.list_models.outputs.all_models }}
IMAGE_BASE64="data:image/png;base64,$(base64 -w 0 pareto_front.png)"
echo $IMAGE_BASE64
echo '<img src="'"${{ env.IMAGE_BASE64 }}"'" alt="Image" width="300">'
echo '<img src="'"${{ env.IMAGE_BASE64 }}"'" alt="Image" width="300">' >> "$GITHUB_STEP_SUMMARY"