Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: change ml models location #938

Merged
merged 11 commits into from
May 27, 2022
Merged

chore: change ml models location #938

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
c767204
chore: change ml models location
eratostennis May 20, 2022
950bb92
refactor: model download method
eratostennis May 25, 2022
e0d8325
refactor: use message instead of echo
eratostennis May 26, 2022
0a68888
refactor: remove unused code and set timeout for downloading
eratostennis May 26, 2022
f620f95
chore: change ml models location
eratostennis May 20, 2022
fed403c
refactor: model download method
eratostennis May 25, 2022
b1afe8e
refactor: use message instead of echo
eratostennis May 26, 2022
be654fd
refactor: remove unused code and set timeout for downloading
eratostennis May 26, 2022
fc41f1e
Merge branch 'feature/moving-ml-model-storage' of github.com:eratoste…
eratostennis May 27, 2022
5a3bb7e
chore: adjust time out prediod
eratostennis May 27, 2022
8ab283c
Merge branch 'main' into feature/moving-ml-model-storage
wep21 May 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 39 additions & 25 deletions perception/lidar_apollo_instance_segmentation/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,35 +65,49 @@ else()
set(CUDNN_AVAIL OFF)
endif()

find_program(GDOWN_AVAIL "gdown")
if(NOT GDOWN_AVAIL)
message("gdown: command not found. External files could not be downloaded.")
endif()
if(TRT_AVAIL AND CUDA_AVAIL AND CUDNN_AVAIL)
# download weight files
set(PATH "${CMAKE_CURRENT_SOURCE_DIR}/data")
if(NOT EXISTS "${PATH}")
execute_process(COMMAND mkdir -p ${PATH})
endif()
set(FILE "${PATH}/vlp-16.caffemodel")
message(STATUS "Checking and downloading vlp-16.caffemodel")
if(NOT EXISTS "${FILE}")
message(STATUS "... file does not exist. Downloading now ...")
execute_process(COMMAND gdown "https://drive.google.com/uc?id=1bbO_eeGG5HUqyUiAYjOd6hUn-Zma0mMJ" -O ${PATH}/vlp-16.caffemodel)
endif()
set(FILE "${PATH}/hdl-64.caffemodel")
message(STATUS "Checking and downloading hdl-64.caffemodel")
if(NOT EXISTS "${FILE}")
message(STATUS "... file does not exist. Downloading now ...")
execute_process(COMMAND gdown "https://drive.google.com/uc?id=1ZdB6V3jua3GmtYuY9NR1nc9QZe_ChjkP" -O ${PATH}/hdl-64.caffemodel)
endif()
set(FILE "${PATH}/vls-128.caffemodel")
message(STATUS "Checking and downloading vls-128.caffemodel")
if(NOT EXISTS "${FILE}")
message(STATUS "... file does not exist. Downloading now ...")
execute_process(COMMAND gdown "https://drive.google.com/uc?id=1izpNotNxS6mrYIzJwHQ_EyX_IPDU-MBr" -O ${PATH}/vls-128.caffemodel)
set(DATA_PATH "${CMAKE_CURRENT_SOURCE_DIR}/data")
if(NOT EXISTS "${DATA_PATH}")
execute_process(COMMAND mkdir -p ${DATA_PATH})
endif()

function(download FILE_NAME FILE_HASH)
message(STATUS "Checking and downloading ${FILE_NAME}")
set(FILE_PATH ${DATA_PATH}/${FILE_NAME})
set(STATUS_CODE 0)
message(STATUS "start ${FILE_NAME}")
if(EXISTS ${FILE_PATH})
message(STATUS "found ${FILE_NAME}")
file(MD5 ${FILE_PATH} EXISTING_FILE_HASH)
if(${FILE_HASH} STREQUAL ${EXISTING_FILE_HASH})
message(STATUS "same ${FILE_NAME}")
message(STATUS "File already exists.")
else()
message(STATUS "diff ${FILE_NAME}")
message(STATUS "File hash changes. Downloading now ...")
file(DOWNLOAD https://awf.ml.dev.web.auto/perception/models/${FILE_NAME} ${FILE_PATH} STATUS DOWNLOAD_STATUS TIMEOUT 180)
list(GET DOWNLOAD_STATUS 0 STATUS_CODE)
list(GET DOWNLOAD_STATUS 1 ERROR_MESSAGE)
endif()
else()
message(STATUS "not found ${FILE_NAME}")
message(STATUS "File doesn't exists. Downloading now ...")
file(DOWNLOAD https://awf.ml.dev.web.auto/perception/models/${FILE_NAME} ${FILE_PATH} STATUS DOWNLOAD_STATUS TIMEOUT 180)
list(GET DOWNLOAD_STATUS 0 STATUS_CODE)
list(GET DOWNLOAD_STATUS 1 ERROR_MESSAGE)
endif()
if(${STATUS_CODE} EQUAL 0)
message(STATUS "Download completed successfully!")
else()
message(FATAL_ERROR "Error occurred during download: ${ERROR_MESSAGE}")
endif()
endfunction()

download(vlp-16.caffemodel f79f56a835893eb5289182dd06ce9905)
download(hdl-64.caffemodel f79f56a835893eb5289182dd06ce9905)
download(vls-128.caffemodel e5864f65c42d7d69a58fa7b01970d078)

find_package(PCL REQUIRED)

set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_MWAITXINTRIN_H_INCLUDED")
Expand Down
42 changes: 24 additions & 18 deletions perception/lidar_centerpoint/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,45 +66,51 @@ else()
set(CUDNN_AVAIL OFF)
endif()

message(STATUS "start to download")
if(TRT_AVAIL AND CUDA_AVAIL AND CUDNN_AVAIL)
# Download trained models
find_program(GDOWN_AVAIL "gdown")
if(NOT GDOWN_AVAIL)
message("gdown: command not found. External files could not be downloaded.")
endif()

set(DATA_PATH ${CMAKE_CURRENT_SOURCE_DIR}/data)
execute_process(COMMAND mkdir -p ${DATA_PATH})

function(download FILE_NAME GFILE_ID FILE_HASH)
# https://drive.google.com/file/d/GFILE_ID/view
function(download FILE_NAME FILE_HASH)
message(STATUS "Checking and downloading ${FILE_NAME}")
set(FILE_PATH ${DATA_PATH}/${FILE_NAME})
set(STATUS_CODE 0)
message(STATUS "start ${FILE_NAME}")
if(EXISTS ${FILE_PATH})
message(STATUS "found ${FILE_NAME}")
file(MD5 ${FILE_PATH} EXISTING_FILE_HASH)
if(${FILE_HASH} EQUAL ${EXISTING_FILE_HASH})
if("${FILE_HASH}" STREQUAL "${EXISTING_FILE_HASH}")
message(STATUS "same ${FILE_NAME}")
message(STATUS "File already exists.")
else()
message(STATUS "diff ${FILE_NAME}")
message(STATUS "File hash changes. Downloading now ...")
execute_process(COMMAND gdown --quiet https://drive.google.com/uc?id=${GFILE_ID} -O ${FILE_PATH})
# file(MD5 ${FILE_PATH} DOWNLOADED_FILE_HASH) # disable to pass ci
message(STATUS "Downloaded file hash: ${DOWNLOADED_FILE_HASH}")
file(DOWNLOAD https://awf.ml.dev.web.auto/perception/models/${FILE_NAME} ${FILE_PATH} STATUS DOWNLOAD_STATUS TIMEOUT 180)
list(GET DOWNLOAD_STATUS 0 STATUS_CODE)
list(GET DOWNLOAD_STATUS 1 ERROR_MESSAGE)
endif()
else()
message(STATUS "not found ${FILE_NAME}")
message(STATUS "File doesn't exists. Downloading now ...")
execute_process(COMMAND gdown --quiet https://drive.google.com/uc?id=${GFILE_ID} -O ${FILE_PATH})
# file(MD5 ${FILE_PATH} DOWNLOADED_FILE_HASH) # disable to pass ci
message(STATUS "Downloaded file hash: ${DOWNLOADED_FILE_HASH}")
file(DOWNLOAD https://awf.ml.dev.web.auto/perception/models/${FILE_NAME} ${FILE_PATH} STATUS DOWNLOAD_STATUS TIMEOUT 180)
list(GET DOWNLOAD_STATUS 0 STATUS_CODE)
list(GET DOWNLOAD_STATUS 1 ERROR_MESSAGE)
endif()
if(${STATUS_CODE} EQUAL 0)
message(STATUS "Download completed successfully!")
else()
message(FATAL_ERROR "Error occurred during download: ${ERROR_MESSAGE}")
endif()
endfunction()

# default model
download(pts_voxel_encoder_default.onnx 1KFhmA4oFT6CtZx5806QeMzn5H2tKa3oD 410f730c537968cb27fbd70c941849a8)
download(pts_backbone_neck_head_default.onnx 1iyk5VoQ4uNBGPZwypVZIMjSuSYAI1RxP e97c165c7877222c0e27e44409a07517)
download(pts_voxel_encoder_default.onnx 410f730c537968cb27fbd70c941849a8)
download(pts_backbone_neck_head_default.onnx e97c165c7877222c0e27e44409a07517)

# aip_x2 model
download(pts_voxel_encoder_aip_x2.onnx 13aYPRHx17Ge4BqxzW9drAUSWTppjtUV5 3ae5e9efd7b2ed12115e6f0b28cac58d)
download(pts_backbone_neck_head_aip_x2.onnx 14PJ_L3Jpz6Yi8GzoctVOEbGWcaCLArGp 6a406a19e05660677c162486ab332de8)
download(pts_voxel_encoder_aip_x2.onnx 3ae5e9efd7b2ed12115e6f0b28cac58d)
download(pts_backbone_neck_head_aip_x2.onnx 6a406a19e05660677c162486ab332de8)

find_package(ament_cmake_auto REQUIRED)
ament_auto_find_build_dependencies()
Expand Down
136 changes: 43 additions & 93 deletions perception/tensorrt_yolo/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,100 +68,50 @@ else()
endif()

# Download onnx
find_program(GDOWN_AVAIL "gdown")
if(NOT GDOWN_AVAIL)
message("gdown: command not found. External files could not be downloaded.")
endif()
set(PATH "${CMAKE_CURRENT_SOURCE_DIR}/data")
if(NOT EXISTS "${PATH}")
execute_process(COMMAND mkdir -p ${PATH})
endif()
# yolov3
set(FILE "${PATH}/yolov3.onnx")
message(STATUS "Checking and downloading yolov3.onnx")
if(NOT EXISTS "${FILE}")
message(STATUS "... file does not exist. Downloading now ...")
execute_process(
COMMAND gdown "https://drive.google.com/uc?id=1ZYoBqVynmO5kKntyN56GEbELRpuXG8Ym" -O ${PATH}/yolov3.onnx
ERROR_QUIET
)
endif()

# yolov4
set(FILE "${PATH}/yolov4.onnx")
message(STATUS "Checking and downloading yolov4.onnx")
if(NOT EXISTS "${FILE}")
message(STATUS "... file does not exist. Downloading now ...")
execute_process(
COMMAND gdown "https://drive.google.com/uc?id=1vkNmSwcIpTkJ_-BrKhxtit0PBJeJYTVX" -O ${PATH}/yolov4.onnx
ERROR_QUIET
)
endif()

# yolov4-tiny
set(FILE "${PATH}/yolov4-tiny.onnx")
message(STATUS "Checking and downloading yolov4.onnx")
if(NOT EXISTS "${FILE}")
message(STATUS "... file does not exist. Downloading now ...")
execute_process(
COMMAND gdown "https://drive.google.com/uc?id=1rUHjV_JfkmlVFgb20XXrOMWo_HZAjrFh" -O ${PATH}/yolov4-tiny.onnx
ERROR_QUIET
)
endif()

# yolov5s
set(FILE "${PATH}/yolov5s.onnx")
message(STATUS "Checking and downloading yolov5s.onnx")
if(NOT EXISTS "${FILE}")
message(STATUS "... file does not exist. Downloading now ...")
execute_process(
COMMAND gdown "https://drive.google.com/uc?id=1CF21nQWigwCPTr5psQZXg6cBQIOYKbad" -O ${PATH}/yolov5s.onnx
ERROR_QUIET
)
endif()

# yolov5m
set(FILE "${PATH}/yolov5m.onnx")
message(STATUS "Checking and downloading yolov5m.onnx")
if(NOT EXISTS "${FILE}")
message(STATUS "... file does not exist. Downloading now ...")
execute_process(
COMMAND gdown "https://drive.google.com/uc?id=1a1h50KJH6slwmjKZpPlS-errukF-BrgG" -O ${PATH}/yolov5m.onnx
ERROR_QUIET
)
endif()

# yolov5l
set(FILE "${PATH}/yolov5l.onnx")
message(STATUS "Checking and downloading yolov5l.onnx")
if(NOT EXISTS "${FILE}")
message(STATUS "... file does not exist. Downloading now ...")
execute_process(
COMMAND gdown "https://drive.google.com/uc?id=1xO8S92Cq7qrmx93UHHyA7Cd7-dJsBDP8" -O ${PATH}/yolov5l.onnx
ERROR_QUIET
)
endif()

# yolov5x
set(FILE "${PATH}/yolov5x.onnx")
message(STATUS "Checking and downloading yolov5x.onnx")
if(NOT EXISTS "${FILE}")
message(STATUS "... file does not exist. Downloading now ...")
execute_process(
COMMAND gdown "https://drive.google.com/uc?id=1kAHuNJUCxpD-yWrS6t95H3zbAPfClLxI" -O ${PATH}/yolov5x.onnx
ERROR_QUIET
)
endif()

set(FILE "${PATH}/coco.names")
message(STATUS "Checking and downloading coco.names")
if(NOT EXISTS "${FILE}")
message(STATUS "... file does not exist. Downloading now ...")
execute_process(
COMMAND gdown "https://drive.google.com/uc?id=19wXD23PE16kJDkZ7j2W3Ijvx5I1kO9kJ" -O ${PATH}/coco.names
ERROR_QUIET
)
set(DATA_PATH "${CMAKE_CURRENT_SOURCE_DIR}/data")
if(NOT EXISTS "${DATA_PATH}")
execute_process(COMMAND mkdir -p ${DATA_PATH})
endif()
function(download FILE_NAME FILE_HASH)
message(STATUS "Checking and downloading ${FILE_NAME}")
set(FILE_PATH ${DATA_PATH}/${FILE_NAME})
set(STATUS_CODE 0)
message(STATUS "start ${FILE_NAME}")
if(EXISTS ${FILE_PATH})
message(STATUS "found ${FILE_NAME}")
file(MD5 ${FILE_PATH} EXISTING_FILE_HASH)
if(${FILE_HASH} STREQUAL ${EXISTING_FILE_HASH})
message(STATUS "same ${FILE_NAME}")
message(STATUS "File already exists.")
else()
message(STATUS "diff ${FILE_NAME}")
message(STATUS "File hash changes. Downloading now ...")
file(DOWNLOAD https://awf.ml.dev.web.auto/perception/models/${FILE_NAME} ${FILE_PATH} STATUS DOWNLOAD_STATUS TIMEOUT 180)
list(GET DOWNLOAD_STATUS 0 STATUS_CODE)
list(GET DOWNLOAD_STATUS 1 ERROR_MESSAGE)
endif()
else()
message(STATUS "not found ${FILE_NAME}")
message(STATUS "File doesn't exists. Downloading now ...")
file(DOWNLOAD https://awf.ml.dev.web.auto/perception/models/${FILE_NAME} ${FILE_PATH} STATUS DOWNLOAD_STATUS TIMEOUT 180)
list(GET DOWNLOAD_STATUS 0 STATUS_CODE)
list(GET DOWNLOAD_STATUS 1 ERROR_MESSAGE)
endif()
if(${STATUS_CODE} EQUAL 0)
message(STATUS "Download completed successfully!")
else()
message(FATAL_ERROR "Error occurred during download: ${ERROR_MESSAGE}")
endif()
endfunction()

download(yolov3.onnx bde3b120f51db251d9a89cd25205e062)
download(yolov4.onnx 99ed8c5d18acad2ebb5ee80860d9cbaf)
download(yolov4-tiny.onnx d16dfd56dddfce75f25d5aaf4c8f4c40)
download(yolov5s.onnx 646b295db6089597e79163446d6eedca)
download(yolov5m.onnx 8fc91d70599ac59ae92c6490001fd84b)
download(yolov5l.onnx 99ff1fc966bda8efbc45c3b8bf00eea2)
download(yolov5x.onnx c51beebed276735cb16a18ca32959e9f)
download(coco.names 8fc50561361f8bcf96b0177086e7616c)

# create calib image directory for int8 calibration
set(CALIB_IMAGE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/calib_image")
Expand Down
69 changes: 34 additions & 35 deletions perception/traffic_light_classifier/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,44 +66,43 @@ else()
endif()

# Download caffemodel and prototxt
set(PRETRAINED_MODEL_LINK "https://drive.google.com/uc?id=15CQceCn9TZDU6huKJacQvUnDiLHionb3")
set(PRETRAINED_MODEL_HASH 7dc31c696b0400ddfc2cc5521586fa51)
set(LAMP_LABEL_LINK "https://drive.google.com/uc?id=1D7n3oGSWLkWgxET6PcWqEzOhmmPcqM52")
set(LAMP_LABEL_HASH 20167c8e9a1f9d2ec7b0b0088c4100f0)

find_program(GDOWN_AVAIL "gdown")
if(NOT GDOWN_AVAIL)
message(STATUS "gdown: command not found. External files could not be downloaded.")
set(DATA_PATH "${CMAKE_CURRENT_SOURCE_DIR}/data")
if(NOT EXISTS "${DATA_PATH}")
execute_process(COMMAND mkdir -p ${DATA_PATH})
endif()
set(PATH "${CMAKE_CURRENT_SOURCE_DIR}/data")
if(NOT EXISTS "${PATH}")
execute_process(COMMAND mkdir -p ${PATH})
endif()
set(FILE "${PATH}/traffic_light_classifier_mobilenetv2.onnx")
message(STATUS "Checking and downloading traffic_light_classifier_mobilenetv2.onnx")
if(EXISTS "${FILE}")
file(MD5 "${FILE}" EXISTING_FILE_HASH)
if(NOT "${PRETRAINED_MODEL_HASH}" EQUAL "${EXISTING_FILE_HASH}")
message(STATUS "... file hash changed. Downloading now ...")
execute_process(COMMAND gdown --quiet "${PRETRAINED_MODEL_LINK}" -O ${PATH}/traffic_light_classifier_mobilenetv2.onnx)
function(download FILE_NAME FILE_HASH)
message(STATUS "Checking and downloading ${FILE_NAME}")
set(FILE_PATH ${DATA_PATH}/${FILE_NAME})
set(STATUS_CODE 0)
message(STATUS "start ${FILE_NAME}")
if(EXISTS ${FILE_PATH})
message(STATUS "found ${FILE_NAME}")
file(MD5 ${FILE_PATH} EXISTING_FILE_HASH)
if(${FILE_HASH} STREQUAL ${EXISTING_FILE_HASH})
message(STATUS "same ${FILE_NAME}")
message(STATUS "File already exists.")
else()
message(STATUS "diff ${FILE_NAME}")
message(STATUS "File hash changes. Downloading now ...")
file(DOWNLOAD https://awf.ml.dev.web.auto/perception/models/${FILE_NAME} ${FILE_PATH} STATUS DOWNLOAD_STATUS TIMEOUT 180)
list(GET DOWNLOAD_STATUS 0 STATUS_CODE)
list(GET DOWNLOAD_STATUS 1 ERROR_MESSAGE)
endif()
else()
message(STATUS "not found ${FILE_NAME}")
message(STATUS "File doesn't exists. Downloading now ...")
file(DOWNLOAD https://awf.ml.dev.web.auto/perception/models/${FILE_NAME} ${FILE_PATH} STATUS DOWNLOAD_STATUS TIMEOUT 180)
list(GET DOWNLOAD_STATUS 0 STATUS_CODE)
list(GET DOWNLOAD_STATUS 1 ERROR_MESSAGE)
endif()
else()
message(STATUS "... file does not exist. Downloading now ...")
execute_process(COMMAND gdown --quiet "${PRETRAINED_MODEL_LINK}" -O ${PATH}/traffic_light_classifier_mobilenetv2.onnx)
endif()

set(FILE "${PATH}/lamp_labels.txt")
message(STATUS "Checking and downloading lamp_labels.txt")
if(EXISTS "${FILE}")
file(MD5 "${FILE}" EXISTING_FILE_HASH)
if(NOT "${LAMP_LABEL_HASH}" EQUAL "${EXISTING_FILE_HASH}")
message(STATUS "... file does not exist. Downloading now ...")
execute_process(COMMAND gdown --quiet "${LAMP_LABEL_LINK}" -O ${PATH}/lamp_labels.txt)
if(${STATUS_CODE} EQUAL 0)
message(STATUS "Download completed successfully!")
else()
message(FATAL_ERROR "Error occurred during download: ${ERROR_MESSAGE}")
endif()
else()
message(STATUS "... file does not exist. Downloading now ...")
execute_process(COMMAND gdown --quiet "${LAMP_LABEL_LINK}" -O ${PATH}/lamp_labels.txt)
endif()
endfunction()
download(traffic_light_classifier_mobilenetv2.onnx 7dc31c696b0400ddfc2cc5521586fa51)
download(lamp_labels.txt 20167c8e9a1f9d2ec7b0b0088c4100f0)

find_package(ament_cmake_auto REQUIRED)
ament_auto_find_build_dependencies()
Expand Down
Loading