From d2ada6bf637293a72c069d874014e8833243dbcc Mon Sep 17 00:00:00 2001 From: psyray Date: Sun, 9 Jun 2024 02:35:36 +0200 Subject: [PATCH 1/3] fix(celery): update git tools at starts if directory exists --- web/celery-entrypoint.sh | 90 ++++++++++++++++++++++++++-------------- 1 file changed, 58 insertions(+), 32 deletions(-) diff --git a/web/celery-entrypoint.sh b/web/celery-entrypoint.sh index fb6a5335..828efd29 100755 --- a/web/celery-entrypoint.sh +++ b/web/celery-entrypoint.sh @@ -19,8 +19,8 @@ Package: firefox Pin: version 1:1snap1-0ubuntu2 Pin-Priority: -1 ' | tee /etc/apt/preferences.d/mozilla-firefox -apt update -apt install firefox -y +apt-get update +apt-get install firefox -y # Temporary fix for whatportis bug - See https://github.com/yogeshojha/rengine/issues/984 sed -i 's/purge()/truncate()/g' /usr/local/lib/python3.10/dist-packages/whatportis/cli.py @@ -52,56 +52,77 @@ fi if [ ! -d "/usr/src/github/Sublist3r" ] then echo "Cloning Sublist3r" - git clone https://github.com/aboul3la/Sublist3r /usr/src/github/Sublist3r + git clone https://github.com/aboul3la/Sublist3r /usr/src/github/Sublist3r --quiet +else + echo "Updating Sublist3r" + git -C /usr/src/github/Sublist3r/ pull --quiet fi -python3 -m pip install -r /usr/src/github/Sublist3r/requirements.txt +echo "Installing dependencies..." +python3 -m pip -q install -r /usr/src/github/Sublist3r/requirements.txt # clone OneForAll if [ ! -d "/usr/src/github/OneForAll" ] then echo "Cloning OneForAll" - git clone https://github.com/shmilylty/OneForAll /usr/src/github/OneForAll + git clone https://github.com/shmilylty/OneForAll /usr/src/github/OneForAll --quiet +else + echo "Updating OneForAll" + git -C /usr/src/github/OneForAll/ pull --quiet fi -python3 -m pip install -r /usr/src/github/OneForAll/requirements.txt +echo "Installing dependencies..." +python3 -m pip -q install -r /usr/src/github/OneForAll/requirements.txt # clone eyewitness if [ ! -d "/usr/src/github/EyeWitness" ] then echo "Cloning EyeWitness" - git clone https://github.com/FortyNorthSecurity/EyeWitness /usr/src/github/EyeWitness - # pip install -r /usr/src/github/Eyewitness/requirements.txt + git clone https://github.com/FortyNorthSecurity/EyeWitness /usr/src/github/EyeWitness --quiet +else + echo "Updating EyeWitness" + git -C /usr/src/github/EyeWitness/ pull --quiet fi # clone theHarvester if [ ! -d "/usr/src/github/theHarvester" ] then echo "Cloning theHarvester" - git clone https://github.com/laramies/theHarvester /usr/src/github/theHarvester + git clone https://github.com/laramies/theHarvester /usr/src/github/theHarvester --quiet +else + echo "Updating theHarvester" + git -C /usr/src/github/theHarvester/ pull --quiet fi -python3 -m pip install -r /usr/src/github/theHarvester/requirements/base.txt +echo "Installing dependencies..." +python3 -m pip -q install -r /usr/src/github/theHarvester/requirements/base.txt # clone vulscan if [ ! -d "/usr/src/github/scipag_vulscan" ] then echo "Cloning Nmap Vulscan script" - git clone https://github.com/scipag/vulscan /usr/src/github/scipag_vulscan + git clone https://github.com/scipag/vulscan /usr/src/github/scipag_vulscan --quiet echo "Symlinking to nmap script dir" ln -s /usr/src/github/scipag_vulscan /usr/share/nmap/scripts/vulscan - echo "Usage in reNgine, set vulscan/vulscan.nse in nmap_script scanEngine port_scan config parameter" +else + echo "Updating Nmap Vulscan script" + git -C /usr/src/github/scipag_vulscan/ pull --quiet fi +echo "Usage in reNgine, set vulscan/vulscan.nse in nmap_script scanEngine port_scan config parameter" # install h8mail -python3 -m pip install h8mail +echo "Installing dependencies..." +python3 -m pip -q install h8mail # install gf patterns if [ ! -d "/root/Gf-Patterns" ]; then echo "Installing GF Patterns" - mkdir ~/.gf - cp -r $GOPATH/src/github.com/tomnomnom/gf/examples/*.json ~/.gf - git clone https://github.com/1ndianl33t/Gf-Patterns ~/Gf-Patterns - mv ~/Gf-Patterns/*.json ~/.gf + mkdir -p ~/.gf + cp -r $GOPATH/pkg/mod/github.com/tomnomnom/gf*/examples/*.json ~/.gf + git clone https://github.com/1ndianl33t/Gf-Patterns ~/Gf-Patterns --quiet +else + echo "Updating GF Patterns" + git -C /root/Gf-Patterns/ pull --quiet fi +mv ~/Gf-Patterns/*.json ~/.gf # store scan_results if [ ! -d "/usr/src/scan_results" ] @@ -110,46 +131,51 @@ then fi # test tools, required for configuration -naabu && subfinder && amass -nuclei +naabu;subfinder;amass;nuclei if [ ! -d "/root/nuclei-templates/geeknik_nuclei_templates" ]; then echo "Installing Geeknik Nuclei templates" - git clone https://github.com/geeknik/the-nuclei-templates.git ~/nuclei-templates/geeknik_nuclei_templates + git clone https://github.com/geeknik/the-nuclei-templates.git ~/nuclei-templates/geeknik_nuclei_templates --quiet else echo "Removing old Geeknik Nuclei templates and updating new one" rm -rf ~/nuclei-templates/geeknik_nuclei_templates - git clone https://github.com/geeknik/the-nuclei-templates.git ~/nuclei-templates/geeknik_nuclei_templates + git clone https://github.com/geeknik/the-nuclei-templates.git ~/nuclei-templates/geeknik_nuclei_templates --quiet fi -if [ ! -f ~/nuclei-templates/ssrf_nagli.yaml ]; -then - echo "Downloading ssrf_nagli for Nuclei" - wget https://raw.githubusercontent.com/NagliNagli/BountyTricks/main/ssrf.yaml -O ~/nuclei-templates/ssrf_nagli.yaml -fi +echo "Downloading ssrf_nagli for Nuclei" +wget https://raw.githubusercontent.com/NagliNagli/BountyTricks/main/ssrf.yaml -O ~/nuclei-templates/ssrf_nagli.yaml if [ ! -d "/usr/src/github/CMSeeK" ] then echo "Cloning CMSeeK" - git clone https://github.com/Tuhinshubhra/CMSeeK /usr/src/github/CMSeeK - pip install -r /usr/src/github/CMSeeK/requirements.txt + git clone https://github.com/Tuhinshubhra/CMSeeK /usr/src/github/CMSeeK --quiet +else + echo "Updating CMSeeK" + git -C /usr/src/github/CMSeeK/ pull --quiet fi +python3 -m pip -q install -r /usr/src/github/CMSeeK/requirements.txt # clone ctfr if [ ! -d "/usr/src/github/ctfr" ] then echo "Cloning CTFR" - git clone https://github.com/UnaPibaGeek/ctfr /usr/src/github/ctfr + git clone https://github.com/UnaPibaGeek/ctfr /usr/src/github/ctfr --quiet +else + echo "Updating CTFR" + git -C /usr/src/github/ctfr/ pull --quiet fi # clone gooFuzz if [ ! -d "/usr/src/github/goofuzz" ] then echo "Cloning GooFuzz" - git clone https://github.com/m3n0sd0n4ld/GooFuzz.git /usr/src/github/goofuzz - chmod +x /usr/src/github/goofuzz/GooFuzz + git clone https://github.com/m3n0sd0n4ld/GooFuzz.git /usr/src/github/goofuzz --quiet +else + echo "Updating GooFuzz" + git -C /usr/src/github/goofuzz/ pull --quiet fi +chmod +x /usr/src/github/goofuzz/GooFuzz exec "$@" @@ -157,7 +183,7 @@ exec "$@" echo 'alias httpx="/go/bin/httpx"' >> ~/.bashrc # TEMPORARY FIX, httpcore is causing issues with celery, removing it as temp fix -python3 -m pip uninstall -y httpcore +python3 -m pip -q uninstall -y httpcore if [ ! "$CELERY_LOGLEVEL" ]; then export CELERY_LOGLEVEL='info' From 1fc2f53b8d31bd40d4519d923ef1dde1442a40bc Mon Sep 17 00:00:00 2001 From: psyray Date: Sun, 9 Jun 2024 02:54:51 +0200 Subject: [PATCH 2/3] fix(tool): add missing status string --- web/celery-entrypoint.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/web/celery-entrypoint.sh b/web/celery-entrypoint.sh index 828efd29..30f9c099 100755 --- a/web/celery-entrypoint.sh +++ b/web/celery-entrypoint.sh @@ -154,6 +154,7 @@ else echo "Updating CMSeeK" git -C /usr/src/github/CMSeeK/ pull --quiet fi +echo "Installing dependencies..." python3 -m pip -q install -r /usr/src/github/CMSeeK/requirements.txt # clone ctfr From 68e2029d274a0e6858223596cbc1f0561c8e49b9 Mon Sep 17 00:00:00 2001 From: psyray Date: Mon, 10 Jun 2024 00:59:44 +0200 Subject: [PATCH 3/3] fix(tools): reset package install to apt in celery entrypoint --- web/celery-entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/celery-entrypoint.sh b/web/celery-entrypoint.sh index 30f9c099..223c5cf7 100755 --- a/web/celery-entrypoint.sh +++ b/web/celery-entrypoint.sh @@ -19,8 +19,8 @@ Package: firefox Pin: version 1:1snap1-0ubuntu2 Pin-Priority: -1 ' | tee /etc/apt/preferences.d/mozilla-firefox -apt-get update -apt-get install firefox -y +apt update +apt install firefox -y # Temporary fix for whatportis bug - See https://github.com/yogeshojha/rengine/issues/984 sed -i 's/purge()/truncate()/g' /usr/local/lib/python3.10/dist-packages/whatportis/cli.py