From 7cd181804cca8cd5d00760c22a03e0ffb875b17e Mon Sep 17 00:00:00 2001 From: Alexandre ZANNI <16578570+noraj@users.noreply.github.com> Date: Sun, 3 Oct 2021 23:54:08 +0200 Subject: [PATCH 001/262] not exposing postgres publicly --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index c0601829..c9e60722 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} - POSTGRES_PORT=${POSTGRES_PORT} ports: - - "5432:5432" + - "127.0.0.1:5432:5432" volumes: - postgres_data:/var/lib/postgresql/data/ networks: From ccb6065f710560a6aced30876e091695c9b6b6cb Mon Sep 17 00:00:00 2001 From: Alexandre ZANNI <16578570+noraj@users.noreply.github.com> Date: Mon, 4 Oct 2021 00:00:22 +0200 Subject: [PATCH 002/262] do not expose the internal container either --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index c9e60722..8063377b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -100,7 +100,7 @@ services: - tool_config:/root/.config - static_volume:/usr/src/app/staticfiles/ ports: - - "8000:8000" + - "127.0.0.1:8000:8000" depends_on: - db - celery From 90bbbfbed8ae0f611cef17bee93e821c4a60fc4c Mon Sep 17 00:00:00 2001 From: AnonymousWP <50231698+AnonymousWP@users.noreply.github.com> Date: Tue, 18 Apr 2023 10:39:57 +0200 Subject: [PATCH 003/262] ops(migrations): automate migration process --- Makefile | 3 +++ install.sh | 1 + make.bat | 2 ++ 3 files changed, 6 insertions(+) diff --git a/Makefile b/Makefile index 7e266075..930a7886 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,9 @@ build: ## Build all services. username: ## Generate Username (Use only after make up). ${COMPOSE_PREFIX_CMD} docker-compose ${COMPOSE_ALL_FILES} exec web python3 manage.py createsuperuser +migrate: ## Apply migrations + ${COMPOSE_PREFIX_CMD} docker-compose ${COMPOSE_ALL_FILES} exec web python3 manage.py migrate + pull: ## Pull Docker images. docker login docker.pkg.github.com ${COMPOSE_PREFIX_CMD} docker-compose ${COMPOSE_ALL_FILES} pull diff --git a/install.sh b/install.sh index 4fe4be57..55feeff1 100755 --- a/install.sh +++ b/install.sh @@ -123,6 +123,7 @@ if [ "${failed}" -eq 0 ]; then echo "Creating an account" echo "#########################################################################" make username + make migrate tput setaf 2 && printf "\n%s\n" "Thank you for installing reNgine, happy recon!!" else diff --git a/make.bat b/make.bat index 0a99cb6c..a1579788 100644 --- a/make.bat +++ b/make.bat @@ -15,6 +15,8 @@ if "%1" == "up" docker-compose %COMPOSE_ALL_FILES% up -d --build %SERVICES% if "%1" == "build" docker-compose %COMPOSE_ALL_FILES% build %SERVICES% :: Generate Username (Use only after make up). if "%1" == "username" docker-compose %COMPOSE_ALL_FILES% exec web python3 manage.py createsuperuser +:: Apply migrations +if "%1" == "migrate" docker-compose %COMPOSE_ALL_FILES% exec web python3 manage.py migrate :: Pull Docker images. if "%1" == "pull" docker login docker.pkg.github.com & docker-compose %COMPOSE_ALL_FILES% pull :: Down all services. From 9c9fdecadcf4e2696e46beeb5f834402c55212bb Mon Sep 17 00:00:00 2001 From: AnonymousWP <50231698+AnonymousWP@users.noreply.github.com> Date: Tue, 18 Apr 2023 11:13:12 +0200 Subject: [PATCH 004/262] fix(migrations): add `sudo` Without sudo, it results in an error --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 930a7886..f0a1a6d1 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ username: ## Generate Username (Use only after make up). ${COMPOSE_PREFIX_CMD} docker-compose ${COMPOSE_ALL_FILES} exec web python3 manage.py createsuperuser migrate: ## Apply migrations - ${COMPOSE_PREFIX_CMD} docker-compose ${COMPOSE_ALL_FILES} exec web python3 manage.py migrate + ${COMPOSE_PREFIX_CMD} sudo docker-compose ${COMPOSE_ALL_FILES} exec web python3 manage.py migrate pull: ## Pull Docker images. docker login docker.pkg.github.com From 1656539c564a38e3514ffadc1f37b3cbcfcc102e Mon Sep 17 00:00:00 2001 From: AnonymousWP <50231698+AnonymousWP@users.noreply.github.com> Date: Tue, 18 Apr 2023 12:20:46 +0200 Subject: [PATCH 005/262] fix(migrations): revert sudo and use `echo` Using `make migrate` right away did not work, tried multiple things. --- Makefile | 2 +- install.sh | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index f0a1a6d1..930a7886 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ username: ## Generate Username (Use only after make up). ${COMPOSE_PREFIX_CMD} docker-compose ${COMPOSE_ALL_FILES} exec web python3 manage.py createsuperuser migrate: ## Apply migrations - ${COMPOSE_PREFIX_CMD} sudo docker-compose ${COMPOSE_ALL_FILES} exec web python3 manage.py migrate + ${COMPOSE_PREFIX_CMD} docker-compose ${COMPOSE_ALL_FILES} exec web python3 manage.py migrate pull: ## Pull Docker images. docker login docker.pkg.github.com diff --git a/install.sh b/install.sh index 55feeff1..9e7a41c4 100755 --- a/install.sh +++ b/install.sh @@ -60,7 +60,6 @@ else tput setaf 2; echo "Docker installed!!!" fi - echo " " tput setaf 4; echo "#########################################################################" @@ -75,7 +74,6 @@ else tput setaf 2; echo "docker-compose installed!!!" fi - echo " " tput setaf 4; echo "#########################################################################" @@ -105,8 +103,6 @@ else exit 1 fi - - echo " " tput setaf 4; echo "#########################################################################" @@ -123,9 +119,9 @@ if [ "${failed}" -eq 0 ]; then echo "Creating an account" echo "#########################################################################" make username - make migrate tput setaf 2 && printf "\n%s\n" "Thank you for installing reNgine, happy recon!!" + echo "In case you have unapplied migrations (see above in red), run 'make migrate'" else tput setaf 1 && printf "\n%s\n" "reNgine installation failed!!" fi From f51c1737a56ab1378321b4b5df03332a87942703 Mon Sep 17 00:00:00 2001 From: Raynald Date: Wed, 18 Oct 2023 14:10:04 +0200 Subject: [PATCH 006/262] Add stack trace into male logs if DEBUG True --- web/reNgine/settings.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/web/reNgine/settings.py b/web/reNgine/settings.py index 4fb209ac..954fc11a 100644 --- a/web/reNgine/settings.py +++ b/web/reNgine/settings.py @@ -201,6 +201,11 @@ 'version': 1, 'disable_existing_loggers': True, 'handlers': { + 'file': { + 'level': 'ERROR', + 'class': 'logging.FileHandler', + 'filename': 'errors.log', + }, 'null': { 'class': 'logging.NullHandler' }, @@ -240,6 +245,11 @@ } }, 'loggers': { + 'django': { + 'handlers': ['file'], + 'level': 'ERROR' if DEBUG else 'CRITICAL', + 'propagate': True, + }, '': { 'handlers': ['brief'], 'level': 'DEBUG' if DEBUG else 'INFO', From 1c9b379cc858b6b6b1cd0bc1df867b69e0dca8d6 Mon Sep 17 00:00:00 2001 From: Raynald Date: Thu, 19 Oct 2023 10:42:55 +0200 Subject: [PATCH 007/262] Update README with debug procedure --- README.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/README.md b/README.md index 679c8e13..99ce920f 100644 --- a/README.md +++ b/README.md @@ -412,6 +412,52 @@ You can also [join our Discord channel #development](https://discord.gg/JuhHdHTt ![-----------------------------------------------------](https://raw.githubusercontent.com/andreasbm/readme/master/assets/lines/aqua.png) +### Submitting issues + +You can submit issue to the project but you should do it in a way that helps developers to resolve it as quickly as possible. + +For that, you need to add as much valuable informations as possible. + +You can have this valuable informations by doing this steps : + +- Go to the root of the git cloned project +- Edit `web/entrypoint.sh` and add at the top `export DEBUG=1` +This should give you this result + ```python + #!/bin/bash + + export DEBUG=1 + + python3 manage.py migrate + python3 manage.py runserver 0.0.0.0:8000 + + exec "$@" + ``` +- Restart the web container `docker-compose restart web` +- To deactivate set **DEBUG** to **0** and restart web container again + +Then, with **DEBUG** set to **1**, in the `make logs` output you could see the full stack trace to debug reNgine. + +Example with the tool arsenal version check API bug. +``` +web_1 | File "/usr/local/lib/python3.10/dist-packages/celery/app/task.py", line 411, in __call__ +web_1 | return self.run(*args, **kwargs) +web_1 | TypeError: run_command() got an unexpected keyword argument 'echo' +``` +Now you know the real error is `TypeError: run_command() got an unexpected keyword argument 'echo'` + +And you can post the full stack trace to your newly created issue to help developers to track the root cause of the bug and correct the bug easily + +__Activating debug like this also give you the full stack trace in the browser__ instead of a 500 Error without any details. +So don't forget to open the developer console and check for any XHR request with 500 error. +If there's any check the response of this request to get your detailed error. + + + +Happy issuing ;) + +![-----------------------------------------------------](https://raw.githubusercontent.com/andreasbm/readme/master/assets/lines/aqua.png) + ### First-time Open Source contributors Please note that reNgine is beginner friendly. If you have never done open-source before, we encourage you to do so. **We will be happy and proud of your first PR ever.** From d197511aa10d1cff35975ac8366a3a403876a095 Mon Sep 17 00:00:00 2001 From: Raynald Date: Thu, 19 Oct 2023 12:59:49 +0200 Subject: [PATCH 008/262] Filter by important subdomains in API call --- web/api/views.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/web/api/views.py b/web/api/views.py index 8fbaaa66..b4a101ba 100644 --- a/web/api/views.py +++ b/web/api/views.py @@ -1803,6 +1803,9 @@ def get_queryset(self): subdomains = Subdomain.objects.filter(target_domain__project__slug=project) + if 'is_important' in req.query_params: + subdomains = subdomains.filter(is_important=True) + if target_id: self.queryset = ( subdomains From 619377b2f9cfbc0a2b9012eebccc56a7a81f5b78 Mon Sep 17 00:00:00 2001 From: Raynald Date: Thu, 19 Oct 2023 16:27:17 +0200 Subject: [PATCH 009/262] Add important subdomains button --- web/templates/base/_items/subdomain_tab_content.html | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/web/templates/base/_items/subdomain_tab_content.html b/web/templates/base/_items/subdomain_tab_content.html index 6b7e5808..3c2bb7a2 100644 --- a/web/templates/base/_items/subdomain_tab_content.html +++ b/web/templates/base/_items/subdomain_tab_content.html @@ -93,7 +93,11 @@ - + + + + + From 795ddb5a7c751843e8b4aa119e0f9b02383da6d6 Mon Sep 17 00:00:00 2001 From: Raynald Date: Fri, 20 Oct 2023 00:45:32 +0200 Subject: [PATCH 010/262] Add listener on is_important button --- web/startScan/templates/startScan/detail_scan.html | 4 ++++ web/startScan/templates/startScan/subdomains.html | 4 ++++ web/targetApp/templates/target/summary.html | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/web/startScan/templates/startScan/detail_scan.html b/web/startScan/templates/startScan/detail_scan.html index 5d12d153..155e14aa 100644 --- a/web/startScan/templates/startScan/detail_scan.html +++ b/web/startScan/templates/startScan/detail_scan.html @@ -2418,6 +2418,10 @@