Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: change ml models location #938

Merged
merged 11 commits into from
May 27, 2022
Merged

chore: change ml models location #938

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
c767204
chore: change ml models location
eratostennis May 20, 2022
950bb92
refactor: model download method
eratostennis May 25, 2022
e0d8325
refactor: use message instead of echo
eratostennis May 26, 2022
0a68888
refactor: remove unused code and set timeout for downloading
eratostennis May 26, 2022
f620f95
chore: change ml models location
eratostennis May 20, 2022
fed403c
refactor: model download method
eratostennis May 25, 2022
b1afe8e
refactor: use message instead of echo
eratostennis May 26, 2022
be654fd
refactor: remove unused code and set timeout for downloading
eratostennis May 26, 2022
fc41f1e
Merge branch 'feature/moving-ml-model-storage' of github.com:eratoste…
eratostennis May 27, 2022
5a3bb7e
chore: adjust time out prediod
eratostennis May 27, 2022
8ab283c
Merge branch 'main' into feature/moving-ml-model-storage
wep21 May 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions perception/lidar_apollo_instance_segmentation/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,9 @@ else()
set(CUDNN_AVAIL OFF)
endif()

find_program(GDOWN_AVAIL "gdown")
if(NOT GDOWN_AVAIL)
message("gdown: command not found. External files could not be downloaded.")
find_program(CURL_AVAIL "curl")
if(NOT CURL_AVAIL)
message("curl: command not found. External files could not be downloaded.")
endif()
if(TRT_AVAIL AND CUDA_AVAIL AND CUDNN_AVAIL)
# download weight files
Expand All @@ -79,19 +79,19 @@ if(TRT_AVAIL AND CUDA_AVAIL AND CUDNN_AVAIL)
message(STATUS "Checking and downloading vlp-16.caffemodel")
if(NOT EXISTS "${FILE}")
message(STATUS "... file does not exist. Downloading now ...")
execute_process(COMMAND gdown "https://drive.google.com/uc?id=1bbO_eeGG5HUqyUiAYjOd6hUn-Zma0mMJ" -O ${PATH}/vlp-16.caffemodel)
execute_process(COMMAND curl -s "https://awf.ml.dev.web.auto/perception/models/vlp-16.caffemodel" -o ${PATH}/vlp-16.caffemodel)
endif()
set(FILE "${PATH}/hdl-64.caffemodel")
message(STATUS "Checking and downloading hdl-64.caffemodel")
if(NOT EXISTS "${FILE}")
message(STATUS "... file does not exist. Downloading now ...")
execute_process(COMMAND gdown "https://drive.google.com/uc?id=1ZdB6V3jua3GmtYuY9NR1nc9QZe_ChjkP" -O ${PATH}/hdl-64.caffemodel)
execute_process(COMMAND curl -s "https://awf.ml.dev.web.auto/perception/models/hdl-64.caffemodel" -o ${PATH}/hdl-64.caffemodel)
endif()
set(FILE "${PATH}/vls-128.caffemodel")
message(STATUS "Checking and downloading vls-128.caffemodel")
if(NOT EXISTS "${FILE}")
message(STATUS "... file does not exist. Downloading now ...")
execute_process(COMMAND gdown "https://drive.google.com/uc?id=1izpNotNxS6mrYIzJwHQ_EyX_IPDU-MBr" -O ${PATH}/vls-128.caffemodel)
execute_process(COMMAND curl -s "https://awf.ml.dev.web.auto/perception/models/vls-128.caffemodel" -o ${PATH}/vls-128.caffemodel)
endif()

find_package(PCL REQUIRED)
Expand Down
1 change: 1 addition & 0 deletions perception/lidar_apollo_instance_segmentation/package.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
<author email="yukky.saito@gmail.com">Yukihiro Saito</author>

<buildtool_depend>ament_cmake_auto</buildtool_depend>
<buildtool_depend>curl</buildtool_depend>
wep21 marked this conversation as resolved.
Show resolved Hide resolved

<build_depend>autoware_cmake</build_depend>

Expand Down
27 changes: 16 additions & 11 deletions perception/lidar_centerpoint/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,45 +66,50 @@ else()
set(CUDNN_AVAIL OFF)
endif()

execute_process(COMMAND echo "start to download" 1>&2)
if(TRT_AVAIL AND CUDA_AVAIL AND CUDNN_AVAIL)
# Download trained models
find_program(GDOWN_AVAIL "gdown")
if(NOT GDOWN_AVAIL)
message("gdown: command not found. External files could not be downloaded.")
find_program(CURL_AVAIL "curl")
if(NOT CURL_AVAIL)
message("curl: command not found. External files could not be downloaded.")
endif()

set(DATA_PATH ${CMAKE_CURRENT_SOURCE_DIR}/data)
execute_process(COMMAND mkdir -p ${DATA_PATH})

function(download FILE_NAME GFILE_ID FILE_HASH)
# https://drive.google.com/file/d/GFILE_ID/view
function(download FILE_NAME FILE_HASH)
message(STATUS "Checking and downloading ${FILE_NAME}")
set(FILE_PATH ${DATA_PATH}/${FILE_NAME})
execute_process(COMMAND echo "start ${FILE_NAME}" 1>&2)
wep21 marked this conversation as resolved.
Show resolved Hide resolved
if(EXISTS ${FILE_PATH})
execute_process(COMMAND echo "found ${FILE_NAME}")
file(MD5 ${FILE_PATH} EXISTING_FILE_HASH)
if(${FILE_HASH} EQUAL ${EXISTING_FILE_HASH})
execute_process(COMMAND echo "same ${FILE_NAME}")
message(STATUS "File already exists.")
else()
execute_process(COMMAND echo "diff ${FILE_NAME}")
message(STATUS "File hash changes. Downloading now ...")
execute_process(COMMAND gdown --quiet https://drive.google.com/uc?id=${GFILE_ID} -O ${FILE_PATH})
execute_process(COMMAND curl -s https://awf.ml.dev.web.auto/perception/models/${FILE_NAME} -o ${FILE_PATH})
# file(MD5 ${FILE_PATH} DOWNLOADED_FILE_HASH) # disable to pass ci
message(STATUS "Downloaded file hash: ${DOWNLOADED_FILE_HASH}")
endif()
else()
execute_process(COMMAND echo "not found ${FILE_NAME}")
message(STATUS "File doesn't exists. Downloading now ...")
execute_process(COMMAND gdown --quiet https://drive.google.com/uc?id=${GFILE_ID} -O ${FILE_PATH})
execute_process(COMMAND curl -s https://awf.ml.dev.web.auto/perception/models/${FILE_NAME} -o ${FILE_PATH})
# file(MD5 ${FILE_PATH} DOWNLOADED_FILE_HASH) # disable to pass ci
message(STATUS "Downloaded file hash: ${DOWNLOADED_FILE_HASH}")
endif()
endfunction()

# default model
download(pts_voxel_encoder_default.onnx 1KFhmA4oFT6CtZx5806QeMzn5H2tKa3oD 410f730c537968cb27fbd70c941849a8)
download(pts_backbone_neck_head_default.onnx 1iyk5VoQ4uNBGPZwypVZIMjSuSYAI1RxP e97c165c7877222c0e27e44409a07517)
download(pts_voxel_encoder_default.onnx 410f730c537968cb27fbd70c941849a8)
download(pts_backbone_neck_head_default.onnx e97c165c7877222c0e27e44409a07517)

# aip_x2 model
download(pts_voxel_encoder_aip_x2.onnx 13aYPRHx17Ge4BqxzW9drAUSWTppjtUV5 3ae5e9efd7b2ed12115e6f0b28cac58d)
download(pts_backbone_neck_head_aip_x2.onnx 14PJ_L3Jpz6Yi8GzoctVOEbGWcaCLArGp 6a406a19e05660677c162486ab332de8)
download(pts_voxel_encoder_aip_x2.onnx 3ae5e9efd7b2ed12115e6f0b28cac58d)
download(pts_backbone_neck_head_aip_x2.onnx 6a406a19e05660677c162486ab332de8)

find_package(ament_cmake_auto REQUIRED)
ament_auto_find_build_dependencies()
Expand Down
1 change: 1 addition & 0 deletions perception/lidar_centerpoint/package.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
<license>Apache License 2.0</license>

<buildtool_depend>ament_cmake_auto</buildtool_depend>
<buildtool_depend>curl</buildtool_depend>
wep21 marked this conversation as resolved.
Show resolved Hide resolved

<build_depend>autoware_cmake</build_depend>

Expand Down
22 changes: 11 additions & 11 deletions perception/tensorrt_yolo/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,9 @@ else()
endif()

# Download onnx
find_program(GDOWN_AVAIL "gdown")
if(NOT GDOWN_AVAIL)
message("gdown: command not found. External files could not be downloaded.")
find_program(CURL_AVAIL "curl")
if(NOT CURL_AVAIL)
message("curl: command not found. External files could not be downloaded.")
endif()
set(PATH "${CMAKE_CURRENT_SOURCE_DIR}/data")
if(NOT EXISTS "${PATH}")
Expand All @@ -82,7 +82,7 @@ message(STATUS "Checking and downloading yolov3.onnx")
if(NOT EXISTS "${FILE}")
message(STATUS "... file does not exist. Downloading now ...")
execute_process(
COMMAND gdown "https://drive.google.com/uc?id=1ZYoBqVynmO5kKntyN56GEbELRpuXG8Ym" -O ${PATH}/yolov3.onnx
COMMAND curl -s "https://awf.ml.dev.web.auto/perception/models/yolov3.onnx" -o ${PATH}/yolov3.onnx
ERROR_QUIET
)
endif()
Expand All @@ -93,7 +93,7 @@ message(STATUS "Checking and downloading yolov4.onnx")
if(NOT EXISTS "${FILE}")
message(STATUS "... file does not exist. Downloading now ...")
execute_process(
COMMAND gdown "https://drive.google.com/uc?id=1vkNmSwcIpTkJ_-BrKhxtit0PBJeJYTVX" -O ${PATH}/yolov4.onnx
COMMAND curl -s "https://awf.ml.dev.web.auto/perception/models/yolov4.onnx" -o ${PATH}/yolov4.onnx
ERROR_QUIET
)
endif()
Expand All @@ -104,7 +104,7 @@ message(STATUS "Checking and downloading yolov4.onnx")
if(NOT EXISTS "${FILE}")
message(STATUS "... file does not exist. Downloading now ...")
execute_process(
COMMAND gdown "https://drive.google.com/uc?id=1rUHjV_JfkmlVFgb20XXrOMWo_HZAjrFh" -O ${PATH}/yolov4-tiny.onnx
COMMAND curl -s "https://awf.ml.dev.web.auto/perception/models/yolov4-tiny.onnx" -o ${PATH}/yolov4-tiny.onnx
ERROR_QUIET
)
endif()
Expand All @@ -115,7 +115,7 @@ message(STATUS "Checking and downloading yolov5s.onnx")
if(NOT EXISTS "${FILE}")
message(STATUS "... file does not exist. Downloading now ...")
execute_process(
COMMAND gdown "https://drive.google.com/uc?id=1CF21nQWigwCPTr5psQZXg6cBQIOYKbad" -O ${PATH}/yolov5s.onnx
COMMAND curl -s "https://awf.ml.dev.web.auto/perception/models/yolov5s.onnx" -o ${PATH}/yolov5s.onnx
ERROR_QUIET
)
endif()
Expand All @@ -126,7 +126,7 @@ message(STATUS "Checking and downloading yolov5m.onnx")
if(NOT EXISTS "${FILE}")
message(STATUS "... file does not exist. Downloading now ...")
execute_process(
COMMAND gdown "https://drive.google.com/uc?id=1a1h50KJH6slwmjKZpPlS-errukF-BrgG" -O ${PATH}/yolov5m.onnx
COMMAND curl -s "https://awf.ml.dev.web.auto/perception/models/yolov5m.onnx" -o ${PATH}/yolov5m.onnx
ERROR_QUIET
)
endif()
Expand All @@ -137,7 +137,7 @@ message(STATUS "Checking and downloading yolov5l.onnx")
if(NOT EXISTS "${FILE}")
message(STATUS "... file does not exist. Downloading now ...")
execute_process(
COMMAND gdown "https://drive.google.com/uc?id=1xO8S92Cq7qrmx93UHHyA7Cd7-dJsBDP8" -O ${PATH}/yolov5l.onnx
COMMAND curl -s "https://awf.ml.dev.web.auto/perception/models/yolov5l.onnx" -o ${PATH}/yolov5l.onnx
ERROR_QUIET
)
endif()
Expand All @@ -148,7 +148,7 @@ message(STATUS "Checking and downloading yolov5x.onnx")
if(NOT EXISTS "${FILE}")
message(STATUS "... file does not exist. Downloading now ...")
execute_process(
COMMAND gdown "https://drive.google.com/uc?id=1kAHuNJUCxpD-yWrS6t95H3zbAPfClLxI" -O ${PATH}/yolov5x.onnx
COMMAND curl -s "https://awf.ml.dev.web.auto/perception/models/yolov5x.onnx" -o ${PATH}/yolov5x.onnx
ERROR_QUIET
)
endif()
Expand All @@ -158,7 +158,7 @@ message(STATUS "Checking and downloading coco.names")
if(NOT EXISTS "${FILE}")
message(STATUS "... file does not exist. Downloading now ...")
execute_process(
COMMAND gdown "https://drive.google.com/uc?id=19wXD23PE16kJDkZ7j2W3Ijvx5I1kO9kJ" -O ${PATH}/coco.names
COMMAND curl -s "https://awf.ml.dev.web.auto/perception/models/coco.names" -o ${PATH}/coco.names
ERROR_QUIET
)
endif()
Expand Down
1 change: 1 addition & 0 deletions perception/tensorrt_yolo/package.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
<license>Apache License 2.0</license>

<buildtool_depend>ament_cmake_auto</buildtool_depend>
<buildtool_depend>curl</buildtool_depend>
wep21 marked this conversation as resolved.
Show resolved Hide resolved

<build_depend>autoware_cmake</build_depend>

Expand Down
18 changes: 9 additions & 9 deletions perception/traffic_light_classifier/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,14 +66,14 @@ else()
endif()

# Download caffemodel and prototxt
set(PRETRAINED_MODEL_LINK "https://drive.google.com/uc?id=15CQceCn9TZDU6huKJacQvUnDiLHionb3")
set(PRETRAINED_MODEL_LINK "https://awf.ml.dev.web.auto/perception/models/traffic_light_classifier_mobilenetv2.onnx")
set(PRETRAINED_MODEL_HASH 7dc31c696b0400ddfc2cc5521586fa51)
set(LAMP_LABEL_LINK "https://drive.google.com/uc?id=1D7n3oGSWLkWgxET6PcWqEzOhmmPcqM52")
set(LAMP_LABEL_LINK "https://awf.ml.dev.web.auto/perception/models/lamp_labels.txt")
set(LAMP_LABEL_HASH 20167c8e9a1f9d2ec7b0b0088c4100f0)

find_program(GDOWN_AVAIL "gdown")
if(NOT GDOWN_AVAIL)
message(STATUS "gdown: command not found. External files could not be downloaded.")
find_program(CURL_AVAIL "curl")
if(NOT CURL_AVAIL)
message(STATUS "curl: command not found. External files could not be downloaded.")
endif()
set(PATH "${CMAKE_CURRENT_SOURCE_DIR}/data")
if(NOT EXISTS "${PATH}")
Expand All @@ -85,11 +85,11 @@ if(EXISTS "${FILE}")
file(MD5 "${FILE}" EXISTING_FILE_HASH)
if(NOT "${PRETRAINED_MODEL_HASH}" EQUAL "${EXISTING_FILE_HASH}")
message(STATUS "... file hash changed. Downloading now ...")
execute_process(COMMAND gdown --quiet "${PRETRAINED_MODEL_LINK}" -O ${PATH}/traffic_light_classifier_mobilenetv2.onnx)
execute_process(COMMAND curl -s "${PRETRAINED_MODEL_LINK}" -o ${PATH}/traffic_light_classifier_mobilenetv2.onnx)
endif()
else()
message(STATUS "... file does not exist. Downloading now ...")
execute_process(COMMAND gdown --quiet "${PRETRAINED_MODEL_LINK}" -O ${PATH}/traffic_light_classifier_mobilenetv2.onnx)
execute_process(COMMAND curl -s "${PRETRAINED_MODEL_LINK}" -o ${PATH}/traffic_light_classifier_mobilenetv2.onnx)
endif()

set(FILE "${PATH}/lamp_labels.txt")
Expand All @@ -98,11 +98,11 @@ if(EXISTS "${FILE}")
file(MD5 "${FILE}" EXISTING_FILE_HASH)
if(NOT "${LAMP_LABEL_HASH}" EQUAL "${EXISTING_FILE_HASH}")
message(STATUS "... file does not exist. Downloading now ...")
execute_process(COMMAND gdown --quiet "${LAMP_LABEL_LINK}" -O ${PATH}/lamp_labels.txt)
execute_process(COMMAND curl -s "${LAMP_LABEL_LINK}" -o ${PATH}/lamp_labels.txt)
endif()
else()
message(STATUS "... file does not exist. Downloading now ...")
execute_process(COMMAND gdown --quiet "${LAMP_LABEL_LINK}" -O ${PATH}/lamp_labels.txt)
execute_process(COMMAND curl -s "${LAMP_LABEL_LINK}" -o ${PATH}/lamp_labels.txt)
endif()

find_package(ament_cmake_auto REQUIRED)
Expand Down
1 change: 1 addition & 0 deletions perception/traffic_light_classifier/package.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
<license>Apache License 2.0</license>

<buildtool_depend>ament_cmake_auto</buildtool_depend>
<buildtool_depend>curl</buildtool_depend>
wep21 marked this conversation as resolved.
Show resolved Hide resolved
<buildtool_depend>wget</buildtool_depend>

<build_depend>autoware_cmake</build_depend>
Expand Down
18 changes: 9 additions & 9 deletions perception/traffic_light_ssd_fine_detector/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,14 +68,14 @@ else()
endif()

# Download caffemodel and prototxt
set(PRETRAINED_MODEL_LINK "https://drive.google.com/uc?id=1USFDPRH9JrVdGoqt27qHjRgittwc0kcO")
set(PRETRAINED_MODEL_LINK "https://awf.ml.dev.web.auto/perception/models/mb2-ssd-lite-tlr.onnx")
set(PRETRAINED_MODEL_HASH 34ce7f2cbacbf6da8bc35769f027b73f)
set(LAMP_LABEL_LINK "https://drive.google.com/uc?id=1hPcKvKgKz0fqEo0cNAXH7roEletqZErL")
set(LAMP_LABEL_LINK "https://awf.ml.dev.web.auto/perception/models/voc_labels_tl.txt")
set(LAMP_LABEL_HASH e9f45efb02f2a9aa8ac27b3d5c164905)

find_program(GDOWN_AVAIL "gdown")
if(NOT GDOWN_AVAIL)
message(STATUS "gdown: command not found. External files could not be downloaded.")
find_program(CURL_AVAIL "curl")
if(NOT CURL_AVAIL)
message(STATUS "curl: command not found. External files could not be downloaded.")
endif()
set(PATH "${CMAKE_CURRENT_SOURCE_DIR}/data")
if(NOT EXISTS "${PATH}")
Expand All @@ -88,11 +88,11 @@ if(EXISTS "${FILE}")
file(MD5 "${FILE}" EXISTING_FILE_HASH)
if(NOT "${PRETRAINED_MODEL_HASH}" EQUAL "${EXISTING_FILE_HASH}")
message(STATUS "... file hash changed. Downloading now ...")
execute_process(COMMAND gdown --quiet "${PRETRAINED_MODEL_LINK}" -O ${PATH}/mb2-ssd-lite-tlr.onnx)
execute_process(COMMAND curl -s "${PRETRAINED_MODEL_LINK}" -o ${PATH}/mb2-ssd-lite-tlr.onnx)
endif()
else()
message(STATUS "... file does not exist. Downloading now ...")
execute_process(COMMAND gdown --quiet "${PRETRAINED_MODEL_LINK}" -O ${PATH}/mb2-ssd-lite-tlr.onnx)
execute_process(COMMAND curl -s "${PRETRAINED_MODEL_LINK}" -o ${PATH}/mb2-ssd-lite-tlr.onnx)
endif()

set(FILE "${PATH}/voc_labels_tl.txt")
Expand All @@ -101,11 +101,11 @@ if(EXISTS "${FILE}")
file(MD5 "${FILE}" EXISTING_FILE_HASH)
if(NOT "${LAMP_LABEL_HASH}" EQUAL "${EXISTING_FILE_HASH}")
message(STATUS "... file does not exist. Downloading now ...")
execute_process(COMMAND gdown --quiet "${LAMP_LABEL_LINK}" -O ${PATH}/voc_labels_tl.txt)
execute_process(COMMAND curl -s "${LAMP_LABEL_LINK}" -o ${PATH}/voc_labels_tl.txt)
endif()
else()
message(STATUS "... file does not exist. Downloading now ...")
execute_process(COMMAND gdown --quiet "${LAMP_LABEL_LINK}" -O ${PATH}/voc_labels_tl.txt)
execute_process(COMMAND curl -s "${LAMP_LABEL_LINK}" -o ${PATH}/voc_labels_tl.txt)
endif()

if(TRT_AVAIL AND CUDA_AVAIL AND CUDNN_AVAIL)
Expand Down
1 change: 1 addition & 0 deletions perception/traffic_light_ssd_fine_detector/package.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
<license>Apache License 2.0</license>

<buildtool_depend>ament_cmake_auto</buildtool_depend>
<buildtool_depend>curl</buildtool_depend>
wep21 marked this conversation as resolved.
Show resolved Hide resolved

<build_depend>autoware_cmake</build_depend>

Expand Down