From 4c6c73075abd7e782c4ce44937aac05e126fae91 Mon Sep 17 00:00:00 2001 From: Benedikt Kuehne Date: Wed, 15 Nov 2023 09:51:33 +0100 Subject: [PATCH 1/6] update cvexplore version --- installer/IF20_cve_search.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/IF20_cve_search.sh b/installer/IF20_cve_search.sh index 94040cc85..b933ef82d 100755 --- a/installer/IF20_cve_search.sh +++ b/installer/IF20_cve_search.sh @@ -78,7 +78,7 @@ IF20_cve_search() { pip_install "dnspython==2.2.1" pip_install "Werkzeug" pip_install "python-dateutil" - pip_install "CveXplore==0.3.14" + pip_install "CveXplore==0.3.15" REDIS_PW="$(tr -dc A-Za-z0-9 Date: Wed, 15 Nov 2023 10:40:37 +0100 Subject: [PATCH 2/6] switch to cvexplore directly for updating --- installer/IF20_cve_search.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/installer/IF20_cve_search.sh b/installer/IF20_cve_search.sh index b933ef82d..ad5d816b7 100755 --- a/installer/IF20_cve_search.sh +++ b/installer/IF20_cve_search.sh @@ -179,9 +179,10 @@ IF20_cve_search() { /etc/init.d/redis-server restart CNT=0 while [[ "${CVE_INST}" -eq 1 ]]; do - ./sbin/db_mgmt_cpe_dictionary.py -p || true - ./sbin/db_mgmt_json.py -p || true - ./sbin/db_updater.py -f || true + MONGODB_HOST="172.36.0.1" cvexplore database update + # ./sbin/db_mgmt_cpe_dictionary.py -p || true + # ./sbin/db_mgmt_json.py -p || true + # ./sbin/db_updater.py -f || true if [[ $(./bin/search.py -p busybox 2>/dev/null | grep -c ":\ CVE-") -gt 18 ]]; then break fi @@ -194,6 +195,7 @@ IF20_cve_search() { echo -e "\\n""${GREEN}""${BOLD}""CVE database is up and running. No installation process performed!""${NC}" fi cd "${HOME_PATH}" || ( echo "Could not install EMBA component cve-search" && exit 1 ) + # TODO change the cron too sed -e "s#EMBA_INSTALL_PATH#$(pwd)#" config/emba_updater.init > config/emba_updater sed -e "s#EMBA_INSTALL_PATH#$(pwd)#" config/emba_updater_data.init > config/emba_updater_data chmod +x config/emba_updater From c18de8f149b999a0aca2dffac31054d29522e26c Mon Sep 17 00:00:00 2001 From: Benedikt Kuehne Date: Wed, 15 Nov 2023 10:42:59 +0100 Subject: [PATCH 3/6] update --- installer/IF20_cve_search.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/IF20_cve_search.sh b/installer/IF20_cve_search.sh index ad5d816b7..85958dbd0 100755 --- a/installer/IF20_cve_search.sh +++ b/installer/IF20_cve_search.sh @@ -179,7 +179,7 @@ IF20_cve_search() { /etc/init.d/redis-server restart CNT=0 while [[ "${CVE_INST}" -eq 1 ]]; do - MONGODB_HOST="172.36.0.1" cvexplore database update + MONGODB_HOST="172.36.0.1" cvexplore database populate # ./sbin/db_mgmt_cpe_dictionary.py -p || true # ./sbin/db_mgmt_json.py -p || true # ./sbin/db_updater.py -f || true From 3d95ee49e7fdee003da8ba03212101efc1914394 Mon Sep 17 00:00:00 2001 From: Benedikt Kuehne Date: Wed, 15 Nov 2023 11:23:36 +0100 Subject: [PATCH 4/6] make mongodb IP global var --- config/emba_updater.init | 3 +-- helpers/helpers_emba_defaults.sh | 4 ++++ helpers/helpers_emba_dependency_check.sh | 10 +++++----- installer/IF20_cve_search.sh | 7 +++---- installer/R00_emba_remove.sh | 2 +- 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/config/emba_updater.init b/config/emba_updater.init index 09abe30ce..a660f9f06 100644 --- a/config/emba_updater.init +++ b/config/emba_updater.init @@ -6,7 +6,6 @@ BASE_PATH="$(pwd)" LOG_DIR="/var/log" [ -d EMBA_INSTALL_PATH ] || exit 0 -[ -x EMBA_INSTALL_PATH/external/cve-search/sbin/db_updater.py ] || exit 0 [ -x /etc/init.d/redis-server ] || exit 0 [ -d "$LOG_DIR" ] || exit 0 @@ -28,7 +27,7 @@ echo "[*] EMBA update - cve-search update" | tee -a "$LOG_DIR"/emba_update.log service mongod start | tee -a "$LOG_DIR"/emba_update.log /etc/init.d/redis-server start | tee -a "$LOG_DIR"/emba_update.log -EMBA_INSTALL_PATH/external/cve-search/sbin/db_updater.py -v | tee -a "$LOG_DIR"/emba_update.log +cvexplore database update | tee -a "$LOG_DIR"/emba_update.log echo "[*] EMBA update - update local docker image" | tee -a "$LOG_DIR"/emba_update.log docker pull embeddedanalyzer/emba | tee -a "$LOG_DIR"/emba_update.log diff --git a/helpers/helpers_emba_defaults.sh b/helpers/helpers_emba_defaults.sh index d8d854607..1c4d066fb 100755 --- a/helpers/helpers_emba_defaults.sh +++ b/helpers/helpers_emba_defaults.sh @@ -121,8 +121,12 @@ set_defaults() { export UNBLOB=1 # additional extraction with unblob - https://github.com/onekey-sec/unblob # currently the extracted results are not further used. The current implementation # is for evaluation purposes + export CVE_BLACKLIST="${CONFIG_DIR}"/cve-blacklist.txt # include the blacklisted CVE values to this file export CVE_WHITELIST="${CONFIG_DIR}"/cve-whitelist.txt # include the whitelisted CVE values to this file + export MONGODB_HOST="172.36.0.1" # cveXplore mondodb host + # export MONGODB_PORT=27017 # cveXplore mondodb port + export MODULE_BLACKLIST=() if [[ -f "${CONFIG_DIR}"/module_blacklist.txt ]]; then readarray -t MODULE_BLACKLIST < "${CONFIG_DIR}"/module_blacklist.txt diff --git a/helpers/helpers_emba_dependency_check.sh b/helpers/helpers_emba_dependency_check.sh index 88bdbf482..6c57d4bf2 100755 --- a/helpers/helpers_emba_dependency_check.sh +++ b/helpers/helpers_emba_dependency_check.sh @@ -71,17 +71,17 @@ check_dep_port() check_docker_env() { TOOL_NAME="MongoDB" print_output " ""${TOOL_NAME}"" - \\c" "no_log" - if ! grep -q "bindIp: 172.36.0.1" /etc/mongod.conf; then + if ! grep -q "bindIp: ${MONGODB_HOST}" /etc/mongod.conf; then echo -e "${RED}""not ok""${NC}" echo -e "${RED}"" Wrong ""mongodb config"" - check your installation""${NC}" - echo -e "${RED}"" RE-run installation - bindIp should be set to 172.36.0.1""${NC}" + echo -e "${RED}"" RE-run installation - bindIp should be set to ${MONGODB_HOST}""${NC}" DEP_ERROR=1 else echo -e "${GREEN}""ok""${NC}" fi TOOL_NAME="Docker Interface" print_output " ""${TOOL_NAME}"" -""${RED}"" \\c" "no_log" - if ! ip a show emba_runs | grep -q "172.36.0.1" ; then + if ! ip a show emba_runs | grep -q "${MONGODB_HOST}" ; then echo -e "${RED}"" Missing ""Docker-Interface"" - check your installation""${NC}" if [[ "${WSL}" -eq 1 ]]; then echo -e "${RED}"" Is dockerd running (e.g., sudo dockerd --iptables=false &)""${NC}" @@ -91,7 +91,7 @@ check_docker_env() { echo -e "${ORANGE}"" Trying to auto-maintain the docker interface ...""${NC}" systemctl restart NetworkManager docker fi - if ! ip a show emba_runs | grep -q "172.36.0.1" ; then + if ! ip a show emba_runs | grep -q "${MONGODB_HOST}" ; then echo -e "${RED}"" Use \$systemctl restart NetworkManager docker or reset the docker interface manually (\$ docker network rm emba_runs)""${NC}" DEP_ERROR=1 else @@ -105,7 +105,7 @@ check_docker_env() { } check_nw_interface() { - if ! ip a show emba_runs | grep -q "172.36.0.1" ; then + if ! ip a show emba_runs | grep -q "${MONGODB_HOST}" ; then echo -e "${RED}"" Network interface not available"" - trying to restart now""${NC}" systemctl restart NetworkManager docker echo -e "${GREEN}"" docker-networks restarted""${NC}" diff --git a/installer/IF20_cve_search.sh b/installer/IF20_cve_search.sh index 85958dbd0..680eadabd 100755 --- a/installer/IF20_cve_search.sh +++ b/installer/IF20_cve_search.sh @@ -83,7 +83,7 @@ IF20_cve_search() { REDIS_PW="$(tr -dc A-Za-z0-9 /dev/null + sed -zE "s/localhost([^\n]*\n[^\n]*27017)/${MONGODB_HOST}\1/" ./etc/configuration.ini.sample | tee ./etc/configuration.ini &>/dev/null # we do not use the web server. In case someone enables it we have a good default configuration in place: sed -i "s/^Debug:\ True/Debug:\ False/g" ./etc/configuration.ini sed -i "s/^LoginRequired:\ False/LoginRequired:\ True/g" ./etc/configuration.ini @@ -150,7 +150,7 @@ IF20_cve_search() { if ! [[ -f /etc/mongod.conf ]]; then echo "Could not install EMBA component mongod - missing mongod.conf file" && exit 1 fi - sed -i 's/bindIp\:\ 127.0.0.1/bindIp\:\ 172.36.0.1/g' /etc/mongod.conf + sed -i "s/bindIp\:\ 127.0.0.1/bindIp\:\ ${MONGODB_HOST}/g" /etc/mongod.conf if [[ "${WSL}" -eq 0 ]]; then systemctl daemon-reload @@ -179,7 +179,7 @@ IF20_cve_search() { /etc/init.d/redis-server restart CNT=0 while [[ "${CVE_INST}" -eq 1 ]]; do - MONGODB_HOST="172.36.0.1" cvexplore database populate + cvexplore database initialize # ./sbin/db_mgmt_cpe_dictionary.py -p || true # ./sbin/db_mgmt_json.py -p || true # ./sbin/db_updater.py -f || true @@ -195,7 +195,6 @@ IF20_cve_search() { echo -e "\\n""${GREEN}""${BOLD}""CVE database is up and running. No installation process performed!""${NC}" fi cd "${HOME_PATH}" || ( echo "Could not install EMBA component cve-search" && exit 1 ) - # TODO change the cron too sed -e "s#EMBA_INSTALL_PATH#$(pwd)#" config/emba_updater.init > config/emba_updater sed -e "s#EMBA_INSTALL_PATH#$(pwd)#" config/emba_updater_data.init > config/emba_updater_data chmod +x config/emba_updater diff --git a/installer/R00_emba_remove.sh b/installer/R00_emba_remove.sh index e0a70b11a..af93316b8 100755 --- a/installer/R00_emba_remove.sh +++ b/installer/R00_emba_remove.sh @@ -52,7 +52,7 @@ R00_emba_remove() { fi if [[ -f /etc/mongod.conf ]]; then echo -e "\\n""${ORANGE}""Removing EMBAs mongod configuration""${NC}" - sed -i 's/bindIp\:\ 172.36.0.1/bindIp\:\ 127.0.0.1/g' /etc/mongod.conf # inverse of IF20 line 118 + sed -i "s/bindIp\:\ ${MONGODB_HOST}/bindIp\:\ 127.0.0.1/g" /etc/mongod.conf # inverse of IF20 line 118 fi if [[ -f /etc/apt/sources.list.d/mongodb-org-4.4.list ]]; then echo -e "\\n""${ORANGE}""Removing mongod sources.list configuration""${NC}" From bd4bc7666766ed46238980879aaa08f37e419da3 Mon Sep 17 00:00:00 2001 From: Benedikt Kuehne Date: Wed, 15 Nov 2023 11:50:54 +0100 Subject: [PATCH 5/6] add mongodb_host var to installer --- installer.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/installer.sh b/installer.sh index f37e2cffb..da1197f1c 100755 --- a/installer.sh +++ b/installer.sh @@ -55,6 +55,9 @@ export UBUNTU_OS=0 export WSL=0 export GH_ACTION=0 export SSL_REPOS=0 +# cve_search +export MONGODB_HOST="172.36.0.1" +# export MONGODB_PORT=27017 ## Color definition export RED="\033[0;31m" From 13e1eb34fa9901e093692a8f88bb973be17827ab Mon Sep 17 00:00:00 2001 From: Benedikt Kuehne Date: Wed, 15 Nov 2023 12:52:00 +0100 Subject: [PATCH 6/6] go back to db_updater --- config/emba_updater.init | 3 ++- installer/IF20_cve_search.sh | 7 +++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/config/emba_updater.init b/config/emba_updater.init index a660f9f06..09abe30ce 100644 --- a/config/emba_updater.init +++ b/config/emba_updater.init @@ -6,6 +6,7 @@ BASE_PATH="$(pwd)" LOG_DIR="/var/log" [ -d EMBA_INSTALL_PATH ] || exit 0 +[ -x EMBA_INSTALL_PATH/external/cve-search/sbin/db_updater.py ] || exit 0 [ -x /etc/init.d/redis-server ] || exit 0 [ -d "$LOG_DIR" ] || exit 0 @@ -27,7 +28,7 @@ echo "[*] EMBA update - cve-search update" | tee -a "$LOG_DIR"/emba_update.log service mongod start | tee -a "$LOG_DIR"/emba_update.log /etc/init.d/redis-server start | tee -a "$LOG_DIR"/emba_update.log -cvexplore database update | tee -a "$LOG_DIR"/emba_update.log +EMBA_INSTALL_PATH/external/cve-search/sbin/db_updater.py -v | tee -a "$LOG_DIR"/emba_update.log echo "[*] EMBA update - update local docker image" | tee -a "$LOG_DIR"/emba_update.log docker pull embeddedanalyzer/emba | tee -a "$LOG_DIR"/emba_update.log diff --git a/installer/IF20_cve_search.sh b/installer/IF20_cve_search.sh index 680eadabd..9c88b53b9 100755 --- a/installer/IF20_cve_search.sh +++ b/installer/IF20_cve_search.sh @@ -179,10 +179,9 @@ IF20_cve_search() { /etc/init.d/redis-server restart CNT=0 while [[ "${CVE_INST}" -eq 1 ]]; do - cvexplore database initialize - # ./sbin/db_mgmt_cpe_dictionary.py -p || true - # ./sbin/db_mgmt_json.py -p || true - # ./sbin/db_updater.py -f || true + ./sbin/db_mgmt_cpe_dictionary.py -p || true + ./sbin/db_mgmt_json.py -p || true + ./sbin/db_updater.py -f || true if [[ $(./bin/search.py -p busybox 2>/dev/null | grep -c ":\ CVE-") -gt 18 ]]; then break fi