Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Log debug messages to file in tests #1892

Merged
merged 14 commits into from
Oct 28, 2024
7 changes: 4 additions & 3 deletions .github/workflows/distro_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ jobs:
strategy:
fail-fast: false
matrix:
os: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04", "debian", "archlinux", "fedora", "gentoo/python", "python:3.10-alpine", "kalilinux/kali-rolling", "parrotsec/security"]
os: ["ubuntu:22.04", "ubuntu:24.04", "debian", "archlinux", "fedora", "kalilinux/kali-rolling", "parrotsec/security"]
steps:
- uses: actions/checkout@v4
- name: Install Python and Poetry
Expand All @@ -32,8 +32,9 @@ jobs:
elif [ "$ID" = "fedora" ]; then
dnf install -y curl git bash gcc make openssl-devel bzip2-devel libffi-devel zlib-devel xz-devel tk-devel gdbm-devel readline-devel sqlite-devel
elif [ "$ID" = "gentoo" ]; then
echo "media-libs/libglvnd X" >> /etc/portage/package.use/libglvnd
emerge-webrsync
emerge --update --newuse dev-vcs/git curl bash
emerge --update --newuse dev-vcs/git media-libs/mesa curl bash
fi
fi

Expand All @@ -60,4 +61,4 @@ jobs:
export BBOT_DISTRO_TESTS=true
poetry env use python3.11
poetry install
poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=DEBUG .
poetry run pytest --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO .
5 changes: 5 additions & 0 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,11 @@ jobs:
- name: Run tests
run: |
poetry run pytest --exitfirst --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot .
- name: Upload Debug Logs
uses: actions/upload-artifact@v3
with:
name: pytest-debug-logs
path: pytest_debug.log
- name: Upload Code Coverage
uses: codecov/codecov-action@v3
with:
Expand Down
2 changes: 0 additions & 2 deletions bbot/core/config/logger.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import sys
import atexit
import logging
from copy import copy
import multiprocessing
Expand Down Expand Up @@ -71,7 +70,6 @@ def __init__(self, core):
# Start the QueueListener
self.listener = logging.handlers.QueueListener(self.queue, *self.log_handlers.values())
self.listener.start()
atexit.register(self.listener.stop)

self.log_level = logging.INFO

Expand Down
10 changes: 9 additions & 1 deletion bbot/core/helpers/depsinstaller/installer.py
Original file line number Diff line number Diff line change
Expand Up @@ -342,7 +342,15 @@ def install_core_deps(self):
# ensure tldextract data is cached
self.parent_helper.tldextract("evilcorp.co.uk")
# command: package_name
core_deps = {"unzip": "unzip", "curl": "curl"}
core_deps = {
"unzip": "unzip",
"zipinfo": "unzip",
"curl": "curl",
"git": "git",
"make": "make",
"gcc": "gcc",
"bash": "bash",
}
for command, package_name in core_deps.items():
if not self.parent_helper.which(command):
to_install.add(package_name)
Expand Down
2 changes: 1 addition & 1 deletion bbot/modules/gowitness.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ async def setup(self):

# make sure we have a working chrome install
chrome_test_pass = False
for binary in ("chrome", "chromium", custom_chrome_path):
for binary in ("chrome", "chromium", "chromium-browser", custom_chrome_path):
binary_path = self.helpers.which(binary)
if binary_path and Path(binary_path).is_file():
chrome_test_proc = await self.run_process([binary_path, "--version"])
Expand Down
10 changes: 8 additions & 2 deletions bbot/modules/wpscan.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ class wpscan(BaseModule):
deps_apt = ["curl", "make", "gcc"]
deps_ansible = [
{
"name": "Install Ruby Deps (Debian/Ubuntu)",
"name": "Install Ruby Deps (Debian)",
"package": {"name": ["ruby-rubygems", "ruby-dev"], "state": "present"},
"become": True,
"when": "ansible_facts['os_family'] == 'Debian'",
Expand All @@ -48,7 +48,13 @@ class wpscan(BaseModule):
"name": "Install Ruby Deps (Fedora)",
"package": {"name": ["rubygems", "ruby-devel"], "state": "present"},
"become": True,
"when": "ansible_facts['os_family'] == 'Fedora'",
"when": "ansible_facts['os_family'] == 'RedHat'",
},
{
"name": "Install Ruby Deps (Alpine)",
"package": {"name": ["ruby-dev", "ruby-bundler"], "state": "present"},
"become": True,
"when": "ansible_facts['os_family'] == 'Alpine'",
},
{
"name": "Install wpscan gem",
Expand Down
32 changes: 23 additions & 9 deletions bbot/test/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,31 @@
from bbot.core.helpers.misc import execute_sync_or_async
from bbot.core.helpers.interactsh import server_list as interactsh_servers

# silence stdout + trace
root_logger = logging.getLogger()
pytest_debug_file = Path(__file__).parent.parent.parent / "pytest_debug.log"
print(f"pytest_debug_file: {pytest_debug_file}")
debug_handler = logging.FileHandler(pytest_debug_file)
debug_handler.setLevel(logging.DEBUG)
debug_format = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)s %(message)s")
debug_handler.setFormatter(debug_format)
root_logger.addHandler(debug_handler)

test_config = OmegaConf.load(Path(__file__).parent / "test.conf")
if test_config.get("debug", False):
os.environ["BBOT_DEBUG"] = "True"
logging.getLogger("bbot").setLevel(logging.DEBUG)
CORE.logger.log_level = logging.DEBUG
else:
# silence stdout + trace
root_logger = logging.getLogger()
for h in root_logger.handlers:
h.addFilter(lambda x: x.levelname not in ("STDOUT", "TRACE"))

os.environ["BBOT_DEBUG"] = "True"
CORE.logger.log_level = logging.DEBUG

# silence all stderr output:
stderr_handler = CORE.logger.log_handlers["stderr"]
stderr_handler.setLevel(logging.CRITICAL)
handlers = list(CORE.logger.listener.handlers)
handlers.remove(stderr_handler)
CORE.logger.listener.handlers = tuple(handlers)

for h in root_logger.handlers:
h.addFilter(lambda x: x.levelname not in ("STDOUT", "TRACE"))


CORE.merge_default(test_config)

Expand Down
Loading