Merge branch 'release/2.1.0' into 106-ci-automate-docker-images-creation

Security-Tools-Alliance · Aug 26, 2024 · 57526cb · 57526cb
2 parents 9caf9a3 + 74003d2
commit 57526cb
Show file tree

Hide file tree

Showing 7 changed files with 143 additions and 60 deletions.
diff --git a/.github/release.yml b/.github/release.yml
@@ -11,3 +11,4 @@ changelog:
         - refactor
         - dependencies
         - documentation
+        - ci
diff --git a/.github/workflows/close-issues-on-pr-merge-to-release-branch.yml b/.github/workflows/close-issues-on-pr-merge-to-release-branch.yml
@@ -0,0 +1,25 @@
+name: Close issues on PR merge to release branch
+
+on:
+  pull_request:
+    types:
+      - closed
+
+jobs:
+  close-related-issues:
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.merged == true && startsWith(github.ref, 'refs/heads/release/')
+    permissions:
+      issues: write
+    steps:
+      - name: Extract issue number
+        id: extract_issue_number
+        run: |
+          issue_number=$(echo "${{ github.event.pull_request.body }}" | grep -oE '#[0-9]+' | head -n 1 | tr -d '#')
+          echo "ISSUE_NUMBER=$issue_number" >> $GITHUB_ENV
+
+      - name: Close linked issues
+        uses: peter-evans/close-issue@v3
+        with:
+          issue-number: ${{ env.ISSUE_NUMBER }}
+          comment: "This issue is being closed because the related PR has been merged into a release branch."
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -11,6 +11,8 @@ jobs:
   release:
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
     steps:
       - uses: actions/checkout@v4
       - name: Create release

diff --git a/docker/celery/Dockerfile b/docker/celery/Dockerfile
@@ -1,4 +1,4 @@
-FROM --platform=$TARGETPLATFORM debian:12
+FROM debian:12
 
 # Labels and Credits
 LABEL \
@@ -34,6 +34,7 @@ RUN apt update -y && apt install -y      \
     nmap                \
     net-tools           \
     htop                \
+    firefox-esr         \
     fontconfig fonts-freefont-ttf fonts-noto fonts-terminus
 
 RUN fc-cache -f && \ 
@@ -45,107 +46,136 @@ RUN addgroup --gid 1000 --system $USERNAME && \
     adduser --gid 1000 --system --shell /bin/false --disabled-password --uid 1000 --home /home/$USERNAME $USERNAME && \
     chown $USERNAME:$USERNAME /home/$USERNAME
 
-RUN wget -q https://packages.mozilla.org/apt/repo-signing-key.gpg -O- | tee /etc/apt/keyrings/packages.mozilla.org.asc && \
-    gpg -n -q --import --import-options import-show /etc/apt/keyrings/packages.mozilla.org.asc | awk '/pub/{getline; gsub(/^ +| +$/,""); print "\n"$0"\n"}' && \
-    echo "deb [signed-by=/etc/apt/keyrings/packages.mozilla.org.asc] https://packages.mozilla.org/apt mozilla main" | tee -a /etc/apt/sources.list.d/mozilla.list && \
-    echo '\
-Package: *\
-Pin: origin packages.mozilla.org\
-Pin-Priority: 1000\
-' | tee /etc/apt/preferences.d/mozilla && apt update -y && apt install -y firefox
-
+# Download and install geckodriver
+RUN ARCH=$(dpkg --print-architecture) && \
+    version=0.35.0 && \
+    geckodriver_arm="geckodriver-v${version}-linux-aarch64.tar.gz" && \
+    geckodriver_amd="geckodriver-v${version}-linux64.tar.gz" && \
+    if [ "${ARCH}" = "arm64" ]; then \
+        wget "https://github.com/mozilla/geckodriver/releases/download/v${version}/${geckodriver_arm}" && \
+        tar -xvf "${geckodriver_arm}" -C /usr/local/bin/ && \
+        rm "${geckodriver_arm}"; \
+    elif [ "${ARCH}" = "amd64" ]; then \
+        wget "https://github.com/mozilla/geckodriver/releases/download/v${version}/${geckodriver_amd}" && \
+        tar -xvf "${geckodriver_amd}" -C /usr/local/bin/ && \
+        rm "${geckodriver_amd}"; \
+    else \
+        echo "Unknown architecture: $ARCH" && \
+        exit 1; \
+    fi
+
+# Download and install go
+RUN ARCH=$(dpkg --print-architecture) && \
+    #GO_VERSION=$(curl -s https://go.dev/VERSION?m=text) && \
+    GO_VERSION=1.23.0 && \
+    if [ "${ARCH}" = "arm64" ]; then \
+      wget https://go.dev/dl/go${GO_VERSION}.linux-arm64.tar.gz && \
+      tar -xvf go${GO_VERSION}.linux-arm64.tar.gz -C /usr/local/ && \
+      rm go${GO_VERSION}.linux-arm64.tar.gz; \
+    elif [ "${ARCH}" = "amd64" ]; then \
+      wget https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz && \
+      tar -xvf go${GO_VERSION}.linux-amd64.tar.gz -C /usr/local/ && \
+      rm go${GO_VERSION}.linux-amd64.tar.gz; \
+    else \
+      echo "Unknown architecture: $ARCH" ; \
+      exit 1; \
+    fi
+
+# Install python 3.10
 RUN cd /root && wget https://www.python.org/ftp/python/3.10.0/Python-3.10.0.tgz && \
     tar -xvf Python-3.10.0.tgz && \
     rm Python-3.10.0.tgz && \
     cd Python-3.10.0 && \
-   ./configure --enable-optimizations && \
+    ./configure --enable-optimizations && \
     make -j4 && \
     make altinstall
 
-# Download and install go 1.21.4
-RUN wget https://golang.org/dl/go1.21.4.linux-amd64.tar.gz && \
-    tar -xvf go1.21.4.linux-amd64.tar.gz && \
-    rm go1.21.4.linux-amd64.tar.gz && \
-    mv go /usr/local
-
-
 USER $USERNAME
 WORKDIR /home/$USERNAME
 
 ENV TOOLPATH="/home/${USERNAME}/tools"
+ENV BINPATH="/home/${USERNAME}/.local/bin"
 ENV WORDLISTPATH="/home/${USERNAME}/wordlists"
 ENV PIPX_BIN_DIR="${TOOLPATH}/pipx"
 ENV GOROOT="/usr/local/go"
 ENV GOPATH="${TOOLPATH}/go"
 ENV PATH="${PATH}:${GOROOT}/bin:${GOPATH}/bin:${PIPX_BIN_DIR}"
 
-RUN mkdir -p $TOOLPATH/.github 
+RUN mkdir -p $TOOLPATH/.github && \
+    mkdir -p $BINPATH
+
 
 # Download Go packages
-RUN printf "github.com/jaeles-project/gospider@v1.1.6\n \
-    github.com/tomnomnom/gf@dcd4c361f9f5ba302294ed38b8ce278e8ba69006\n \
-    github.com/tomnomnom/unfurl@v0.4.3\n \
-    github.com/tomnomnom/waybackurls@v0.1.0\n \
-    github.com/projectdiscovery/httpx/cmd/httpx@v1.6.0\n \
-    github.com/projectdiscovery/subfinder/v2/cmd/subfinder@v2.6.6\n \
-    github.com/projectdiscovery/nuclei/v3/cmd/nuclei@v3.2.6\n \
-    github.com/projectdiscovery/naabu/v2/cmd/naabu@v2.3.0\n \
-    github.com/hakluke/hakrawler@2.1\n \
-    github.com/lc/gau/v2/cmd/gau@v2.2.1\n \
-    github.com/owasp-amass/amass/v4/...@v4.2.0\n \
-    github.com/ffuf/ffuf@v2.1.0\n \
-    github.com/projectdiscovery/tlsx/cmd/tlsx@v1.1.6\n \
-    github.com/hahwul/dalfox/v2@v2.9.2\n \
-    github.com/projectdiscovery/katana/cmd/katana@v1.1.0\n \
-    github.com/dwisiswant0/crlfuzz/cmd/crlfuzz@v1.4.1\n \
-    github.com/sa7mon/s3scanner@c544f1cf00f70cae3f2155b24d336f515b7c598b\n" | xargs -L1 go install -ldflags="-s -w" -v || true && chmod 700 -R $GOPATH/pkg/* && rm -rf $GOPATH/pkg/* && rm -rf /home/$USERNAME/.cache/go-build/*
+RUN ARCH=$(dpkg --print-architecture) \
+    && if [ "$ARCH" = "arm64" ] || [ "$ARCH" = "amd64" ]; then \
+        GOARCH=$ARCH go install -v github.com/jaeles-project/gospider@v1.1.6 \
+        && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/tomnomnom/gf@dcd4c361f9f5ba302294ed38b8ce278e8ba69006 \
+        && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/tomnomnom/unfurl@v0.4.3 \
+        && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/tomnomnom/waybackurls@v0.1.0 \
+        && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/projectdiscovery/httpx/cmd/httpx@v1.6.0 \
+        && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@v2.6.6 \
+        && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei@v3.2.6 \
+        && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/projectdiscovery/naabu/v2/cmd/naabu@v2.3.0 \
+        && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/hakluke/hakrawler@latest \
+        && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/lc/gau/v2/cmd/gau@v2.2.1 \
+        && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/owasp-amass/amass/v4/...@v4.2.0 \
+        && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/ffuf/ffuf/v2@v2.1.0 \
+        && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/projectdiscovery/tlsx/cmd/tlsx@v1.1.6 \
+        && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/hahwul/dalfox/v2@v2.9.2 \
+        && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/projectdiscovery/katana/cmd/katana@v1.1.0 \
+        && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/dwisiswant0/crlfuzz/cmd/crlfuzz@v1.4.1 \
+        && GOARCH=$ARCH go install -ldflags="-s -w" -v github.com/sa7mon/s3scanner@c544f1cf00f70cae3f2155b24d336f515b7c598b \
+        && chmod 700 -R $GOPATH/pkg/* \
+        && rm -rf $GOPATH/pkg/* \
+        && rm -rf /home/$USERNAME/.cache/go-build/*; \
+    else \
+        echo "Unknown architecture: $ARCH" ; \
+        exit 1; \
+    fi
 
 # Set environment variables
-ENV PYTHONDONTWRITEBYTECODE 1
-ENV PYTHONUNBUFFERED 1
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+ENV PATH="${PATH}:${BINPATH}"
 
-ENV PATH="/home/${USERNAME}/.local/bin:${PATH}"
+# Install python tools
 RUN python3.10 -m pip install pipx && pipx ensurepath && printf "poetry\n\
     watchdog\n\
     https://github.com/aboul3la/Sublist3r/archive/refs/tags/1.1.zip\n\
     https://github.com/laramies/theHarvester/archive/refs/tags/4.6.0.zip\n\
     git+https://github.com/ncrocfer/whatportis@59a1718bf7c531f2a5a4e213cad0c047ce9c1c94\n\
-    git+https://github.com/EnableSecurity/wafw00f@ae6a67f23c7bc7fd913d5a32d9b81efefefa2da4\n\
+    git+https://github.com/EnableSecurity/wafw00f@5e5d8e9e5f1b1b6d9b2c1c1f9f9b9b9b9b9b9b9b\n\
     h8mail\n" | xargs -L1 pipx install || true
 
 # Install tools
-RUN wget https://github.com/mozilla/geckodriver/releases/download/v0.32.0/geckodriver-v0.32.0-linux64.tar.gz && \
-    tar -xvf geckodriver-v0.32.0-linux64.tar.gz && \
-    rm geckodriver-v0.32.0-linux64.tar.gz && \
-    mv geckodriver /home/$USERNAME/.local/bin && \
+RUN ln -s /usr/local/bin/geckodriver $BINPATH/geckodriver && \
     cd $TOOLPATH/.github && git clone https://github.com/shmilylty/OneForAll.git && cd OneForAll && git reset --hard 9ecfda229199ebf30d9338f4c88cbeb7c40e16c2 && \
-    cd $TOOLPATH/.github && git clone https://github.com/FortyNorthSecurity/EyeWitness.git && cd EyeWitness && git reset --hard ac0c7c0e2e11ff23af0a2cca708afd26ece94096 && \
+    cd $TOOLPATH/.github && git clone https://github.com/FortyNorthSecurity/EyeWitness.git && cd EyeWitness && git reset --hard cb09a842f93109836219b2aa2f9f25c58a34bc8c && \
     cd $TOOLPATH/.github && git clone https://github.com/UnaPibaGeek/ctfr.git && cd ctfr && git reset --hard 6c7fecdc6346c4f5322049e38f415d5bddaa420d && \
     cd $TOOLPATH/.github && git clone https://github.com/Tuhinshubhra/CMSeeK.git && cd CMSeeK && git reset --hard 20f9780d2e682874be959cfd487045c92e3c73f4 && \
     cd $TOOLPATH/.github && git clone https://github.com/GiJ03/Infoga.git && cd Infoga && git reset --hard 6834c6f863c2bdc92cc808934bb293571d1939c1 && \
-    cd $TOOLPATH/.github && wget https://github.com/m3n0sd0n4ld/GooFuzz/releases/download/1.2.5/GooFuzz.v.1.2.5.zip && unzip GooFuzz.v.1.2.5.zip && rm GooFuzz.v.1.2.5.zip && mv GooFuzz* GooFuzz && echo "#!/bin/bash\n\nbash $TOOLPATH/.github/GooFuzz/GooFuzz \"\$@\"" > /home/$USERNAME/.local/bin/GooFuzz && chmod +x /home/$USERNAME/.local/bin/GooFuzz && \
-    cd $TOOLPATH/.github && git clone https://github.com/1ndianl33t/Gf-Patterns && mkdir -p /home/$USERNAME/.gf/ && cp -r Gf-Patterns/*.json /home/$USERNAME/.gf/ && \
-    cd $TOOLPATH/.github && git clone https://github.com/tomnomnom/gf.git && cp -r $TOOLPATH/.github/gf/examples/*.json /home/$USERNAME/.gf/ && \ 
-    mkdir -p /home/$USERNAME/.nmap/ && cd /home/$USERNAME/.nmap/ && git clone https://github.com/scipag/vulscan.git && cd vulscan && git reset --hard 0c793c490455e7907a7c5cbaf3f7210e80d2ee57 && ln -s $TOOLPATH/.github/vulscan /home/$USERNAME/.nmap/vulscan && \
+    cd $TOOLPATH/.github && wget https://github.com/m3n0sd0n4ld/GooFuzz/releases/download/1.2.5/GooFuzz.v.1.2.5.zip && unzip GooFuzz.v.1.2.5.zip && rm GooFuzz.v.1.2.5.zip && mv GooFuzz* GooFuzz && echo "#!/bin/bash\n\nbash $TOOLPATH/.github/GooFuzz/GooFuzz \"\$@\"" > $BINPATH/GooFuzz && chmod +x $BINPATH/GooFuzz && \
+    cd $TOOLPATH/.github && git clone https://github.com/1ndianl33t/Gf-Patterns && cd Gf-Patterns && git reset --hard 565382db80f001af288b8d71c525a7ce7f17e80d && mkdir -p /home/$USERNAME/.gf/ && cp -r *.json /home/$USERNAME/.gf/ && \
+    cd $TOOLPATH/.github && git clone https://github.com/tomnomnom/gf.git && cd gf && git reset --hard dcd4c361f9f5ba302294ed38b8ce278e8ba69006 && cp -r examples/*.json /home/$USERNAME/.gf/ && \ 
+    mkdir -p /home/$USERNAME/.nmap/ && cd /home/$USERNAME/.nmap/ && git clone https://github.com/scipag/vulscan.git && cd vulscan && git reset --hard 2640d62400e9953fb9a33e6033dc59a9dc9606ba && ln -s $TOOLPATH/.github/vulscan /home/$USERNAME/.nmap/vulscan && \
     mkdir -p $WORDLISTPATH && \
     wget https://raw.githubusercontent.com/maurosoria/dirsearch/master/db/dicc.txt -O $WORDLISTPATH/dicc.txt && \
     wget https://raw.githubusercontent.com/danielmiessler/SecLists/master/Fuzzing/fuzz-Bo0oM.txt -O $WORDLISTPATH/fuzz-Bo0oM.txt && \
     wget https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/DNS/deepmagic.com-prefixes-top50000.txt -O $WORDLISTPATH/deepmagic.com-prefixes-top50000.txt && \
-    mkdir -p /home/$USERNAME/nuclei-templates && wget https://raw.githubusercontent.com/NagliNagli/Shockwave-OSS/bd7445cd320a174d3073f0a61867a40849d28436/ssrf.yaml -O ~/nuclei-templates/ssrf_nagli.yaml && \
+    mkdir -p /home/$USERNAME/nuclei-templates && wget https://raw.githubusercontent.com/NagliNagli/Shockwave-OSS/bd7445cd320a174d3073f0a61867a40849d28436/ssrf.yaml -O /home/$USERNAME/nuclei-templates/ssrf_nagli.yaml && \
     mkdir -p /home/$USERNAME/results
 
 COPY --chown=$USERNAME:$USERNAME ./*.toml /home/$USERNAME
 
 # Install aliases for tools needing a pyproject.toml
-RUN cd $TOOLPATH/.github/OneForAll && mv /home/$USERNAME/oneforall-pyproject.toml pyproject.toml && poetry env use python3.10 && poetry install --no-cache && echo "#!/bin/bash\n\npoetry -C $TOOLPATH/.github/OneForAll/ run python $TOOLPATH/.github/OneForAll/oneforall.py \"\$@\"" > /home/$USERNAME/.local/bin/oneforall && chmod +x /home/$USERNAME/.local/bin/oneforall && \
-    cd $TOOLPATH/.github/ctfr && mv /home/$USERNAME/ctfr-pyproject.toml pyproject.toml && poetry env use python3.10 && poetry install --no-cache && echo "#!/bin/bash\n\npoetry -C $TOOLPATH/.github/ctfr/ run python $TOOLPATH/.github/ctfr/ctfr.py \"\$@\"" > /home/$USERNAME/.local/bin/ctfr && chmod +x /home/$USERNAME/.local/bin/ctfr && \
-    cd $TOOLPATH/.github/EyeWitness/Python && mv /home/$USERNAME/eyewitness-pyproject.toml pyproject.toml && poetry env use python3.10 && poetry install --no-cache && echo "#!/bin/bash\n\npoetry -C $TOOLPATH/.github/EyeWitness/Python run python $TOOLPATH/.github/EyeWitness/Python/EyeWitness.py \"\$@\"" > /home/$USERNAME/.local/bin/EyeWitness && chmod +x /home/$USERNAME/.local/bin/EyeWitness && \
-    cd $TOOLPATH/.github/CMSeeK && mv /home/$USERNAME/cmseek-pyproject.toml pyproject.toml && poetry env use python3.10 && poetry install --no-cache && echo "#!/bin/bash\n\npoetry -C $TOOLPATH/.github/CMSeeK/ run python $TOOLPATH/.github/CMSeeK/cmseek.py \"\$@\"" > /home/$USERNAME/.local/bin/cmseek && chmod +x /home/$USERNAME/.local/bin/cmseek && \
-    cd $TOOLPATH/.github/Infoga && mv /home/$USERNAME/infoga-pyproject.toml pyproject.toml && poetry env use python3.10 && poetry install --no-cache && echo "#!/bin/bash\n\npoetry -C $TOOLPATH/.github/Infoga/ run python $TOOLPATH/.github/Infoga/infoga.py \"\$@\"" > /home/$USERNAME/.local/bin/infoga && chmod +x /home/$USERNAME/.local/bin/infoga && \
+RUN cd $TOOLPATH/.github/OneForAll && mv /home/$USERNAME/oneforall-pyproject.toml pyproject.toml && poetry env use python3.10 && poetry install --no-cache && echo "#!/bin/bash\n\npoetry -C $TOOLPATH/.github/OneForAll/ run python $TOOLPATH/.github/OneForAll/oneforall.py \"\$@\"" > $BINPATH/oneforall && chmod +x $BINPATH/oneforall && \
+    cd $TOOLPATH/.github/ctfr && mv /home/$USERNAME/ctfr-pyproject.toml pyproject.toml && poetry env use python3.10 && poetry install --no-cache && echo "#!/bin/bash\n\npoetry -C $TOOLPATH/.github/ctfr/ run python $TOOLPATH/.github/ctfr/ctfr.py \"\$@\"" > $BINPATH/ctfr && chmod +x $BINPATH/ctfr && \
+    cd $TOOLPATH/.github/EyeWitness/Python && mv /home/$USERNAME/eyewitness-pyproject.toml pyproject.toml && poetry env use python3.10 && poetry install --no-cache && echo "#!/bin/bash\n\npoetry -C $TOOLPATH/.github/EyeWitness/Python run python $TOOLPATH/.github/EyeWitness/Python/EyeWitness.py \"\$@\"" > $BINPATH/EyeWitness && chmod +x $BINPATH/EyeWitness && \
+    cd $TOOLPATH/.github/CMSeeK && mv /home/$USERNAME/cmseek-pyproject.toml pyproject.toml && poetry env use python3.10 && poetry install --no-cache && echo "#!/bin/bash\n\npoetry -C $TOOLPATH/.github/CMSeeK/ run python $TOOLPATH/.github/CMSeeK/cmseek.py \"\$@\"" > $BINPATH/cmseek && chmod +x $BINPATH/cmseek && \
+    cd $TOOLPATH/.github/Infoga && mv /home/$USERNAME/infoga-pyproject.toml pyproject.toml && poetry env use python3.10 && poetry install --no-cache && echo "#!/bin/bash\n\npoetry -C $TOOLPATH/.github/Infoga/ run python $TOOLPATH/.github/Infoga/infoga.py \"\$@\"" > $BINPATH/infoga && chmod +x $BINPATH/infoga && \
     cd /home/$USERNAME && poetry install
 
 COPY ./entrypoint.sh /entrypoint.sh
-USER $USERNAME
 RUN mkdir -p /home/$USERNAME/rengine /home/$USERNAME/scan_results \
     && chown -R $USERNAME:$USERNAME /home/$USERNAME/rengine \
     && chown -R $USERNAME:$USERNAME /home/$USERNAME/scan_results

diff --git a/web/reNgine/common_func.py b/web/reNgine/common_func.py
@@ -17,6 +17,8 @@
 from celery.utils.log import get_task_logger
 from discord_webhook import DiscordEmbed, DiscordWebhook
 from django.db.models import Q
+from django.core.validators import URLValidator
+from django.core.exceptions import ValidationError
 
 from reNgine.common_serializers import *
 from reNgine.definitions import *
@@ -424,6 +426,12 @@ def get_subdomain_from_url(url):
 	url_obj = urlparse(url.strip())
 	return url_obj.netloc.split(':')[0]
 
+def is_valid_domain_or_subdomain(domain):
+    try:
+        URLValidator(schemes=['http', 'https'])('http://' + domain)
+        return True
+    except ValidationError:
+        return False
 
 def get_domain_from_subdomain(subdomain):
 	"""Get domain from subdomain.
@@ -434,9 +442,26 @@ def get_domain_from_subdomain(subdomain):
 	Returns:
 		str: Domain name.
 	"""
-	ext = tldextract.extract(subdomain)
-	return '.'.join(ext[1:3])
 
+	if not is_valid_domain_or_subdomain(subdomain):
+		return None
+
+	# Use tldextract to parse the subdomain
+	extracted = tldextract.extract(subdomain)
+
+	# if tldextract recognized the tld then its the final result
+	if extracted.suffix:
+		domain = f"{extracted.domain}.{extracted.suffix}"
+	else:
+		# Fallback method for unknown TLDs, like .clouds or .local etc
+		parts = subdomain.split('.')
+		if len(parts) >= 2:
+			domain = '.'.join(parts[-2:])
+		else:
+			return None
+
+	# Validate the domain before returning
+	return domain if is_valid_domain_or_subdomain(subdomain) else None
 
 def sanitize_url(http_url):
 	"""Removes HTTP ports 80 and 443 from HTTP URL because it's ugly.

diff --git a/web/reNgine/tasks.py b/web/reNgine/tasks.py
@@ -4798,7 +4798,7 @@ def save_imported_subdomains(subdomains, ctx={}):
 	# Validate each subdomain and de-duplicate entries
 	subdomains = list(set([
 		subdomain for subdomain in subdomains
-		if validators.domain(subdomain) and domain.name == get_domain_from_subdomain(subdomain)
+		if domain.name == get_domain_from_subdomain(subdomain)
 	]))
 	if not subdomains:
 		return

diff --git a/web/scanEngine/templates/scanEngine/_items/form_engine.html b/web/scanEngine/templates/scanEngine/_items/form_engine.html
@@ -116,7 +116,7 @@ <h4 class="text-secondary">YAML Configurations</h4>
       reNgine supports YAML based configurations for customizing the scan. The default settings are given below, you may choose to proceed with the default settings or configure it according to your choice.
     </p>
     <h5 class="text-secondary">reNgine YAML Documentation</h5>
-    To learn more about YAML config visit the official documentation at <a href="https://rengine.wiki/usage/scan_engine/" class="text-primary" target="_blank">https://rengine.wiki/pentester/scan_engine/</a>
+    To learn more about YAML config visit the official documentation at <a href="https://github.com/Security-Tools-Alliance/rengine-ng/wiki/scan_engine" class="text-primary" target="_blank">https://github.com/Security-Tools-Alliance/rengine-ng/wiki/scan_engine</a>
     <p class="mt-1 text-danger">
       Note: Invalid YAML configuration may crash scans.
       <br>