diff --git a/drakcore/debian/drakcore.install b/drakcore/debian/drakcore.install index 84ee802d4..ec3d92e77 100644 --- a/drakcore/debian/drakcore.install +++ b/drakcore/debian/drakcore.install @@ -1,6 +1,5 @@ drakcore/config.ini /etc/drakcore/ drakcore/uwsgi.ini /etc/drakcore/ -drakcore/systemd/minio.env /etc/drakcore/ drakcore/systemd/*.service /etc/systemd/system/ diff --git a/drakcore/debian/rules b/drakcore/debian/rules index 35f50599e..42f13d526 100644 --- a/drakcore/debian/rules +++ b/drakcore/debian/rules @@ -12,7 +12,6 @@ build: # Config files cp drakcore/config.dist.ini drakcore/config.ini cp drakcore/uwsgi.dist.ini drakcore/uwsgi.ini - cp drakcore/systemd/minio.dist.env drakcore/systemd/minio.env # Download minio if [ ! -f drakcore/systemd/minio ] ; then wget -O drakcore/systemd/minio https://debs.icedev.pl/manual/minio ; fi chmod +x drakcore/systemd/minio diff --git a/drakcore/drakcore/app.py b/drakcore/drakcore/app.py index 4542f7b7c..b914976c0 100644 --- a/drakcore/drakcore/app.py +++ b/drakcore/drakcore/app.py @@ -7,18 +7,18 @@ import requests import logging -from flask import Flask, jsonify, request, send_file, send_from_directory, abort -from minio.error import NoSuchKey +from flask import Flask, jsonify, request, send_file, redirect, send_from_directory, Response, abort from karton2 import Config, Producer, Task, Resource +from minio.error import NoSuchKey from drakcore.system import SystemService -from drakcore.util import find_config +from drakcore.util import get_config from drakcore.analysis import AnalysisProxy from drakcore.database import Database app = Flask(__name__, static_folder='frontend/build/static') -conf = Config(find_config()) +conf = get_config() rs = SystemService(conf).rs minio = SystemService(conf).minio diff --git a/drakcore/drakcore/config.dist.ini b/drakcore/drakcore/config.dist.ini index 2bee58325..6dce2b590 100644 --- a/drakcore/drakcore/config.dist.ini +++ b/drakcore/drakcore/config.dist.ini @@ -1,18 +1,26 @@ [redis] +; Redis server +; used for task scheduling and other non-persistent data host=localhost port=6379 [minio] -access_key={MINIO_ACCESS_KEY} -secret_key={MINIO_SECRET_KEY} +; MinIO server +; used to store job queue and analysis results address=localhost:9000 bucket=karton2 secure=0 -[drakmon] -listen_host=0.0.0.0 -listen_port=5000 +; MinIO access credentials +; +; NOTE: +; if this is empty, credentials will be read from /etc/drakcore/minio.env +; +; you only need to fill this out if you are using external MinIO +access_key= +secret_key= +[drakmon] ; (advanced) disable drak-system service ; use this if you connect to external karton ; instance instead of the self hosted one diff --git a/drakcore/drakcore/process.py b/drakcore/drakcore/process.py index a68d47ad6..4200d85ec 100644 --- a/drakcore/drakcore/process.py +++ b/drakcore/drakcore/process.py @@ -4,10 +4,10 @@ import functools from io import StringIO -from karton2 import Consumer, Config, Karton, LocalResource +from karton2 import Consumer, Karton, LocalResource from minio.error import NoSuchKey from drakcore.postprocess import REGISTERED_PLUGINS -from drakcore.util import find_config +from drakcore.util import get_config class LocalLogBuffer(logging.Handler): @@ -89,7 +89,7 @@ def process(self): def main(): - conf = Config(find_config()) + conf = get_config() processor = AnalysisProcessor(conf, REGISTERED_PLUGINS) processor.loop() diff --git a/drakcore/drakcore/system.py b/drakcore/drakcore/system.py index 3a439c263..928f3787e 100644 --- a/drakcore/drakcore/system.py +++ b/drakcore/drakcore/system.py @@ -4,7 +4,7 @@ from minio import Minio from karton2 import Config from karton2.services.system import SystemService -from drakcore.util import find_config +from drakcore.util import get_config def get_minio_helper(config: Config): @@ -20,7 +20,7 @@ def get_minio_helper(config: Config): def main(): - config = Config(find_config()) + config = get_config() service = SystemService(config) system_disable = config.config["drakmon"].get("system_disable", "1") diff --git a/drakcore/drakcore/systemd/minio.dist.env b/drakcore/drakcore/systemd/minio.dist.env deleted file mode 100644 index d82c3c095..000000000 --- a/drakcore/drakcore/systemd/minio.dist.env +++ /dev/null @@ -1,2 +0,0 @@ -MINIO_ACCESS_KEY={MINIO_ACCESS_KEY} -MINIO_SECRET_KEY={MINIO_SECRET_KEY} diff --git a/drakcore/drakcore/util.py b/drakcore/drakcore/util.py index 68bec347e..451cf9b22 100644 --- a/drakcore/drakcore/util.py +++ b/drakcore/drakcore/util.py @@ -1,5 +1,8 @@ import base64 import os +import sys + +from karton2 import Config def find_config(): @@ -14,21 +17,43 @@ def find_config(): raise RuntimeError("Configuration file was not found neither in {} nor {}".format(local_path, etc_path)) -def setup_config(): - print('Generating MinIO access key and secret key...') - access_key = base64.b64encode(os.urandom(30)).decode('ascii').replace('+', '-').replace('/', '_') - secret_key = base64.b64encode(os.urandom(30)).decode('ascii').replace('+', '-').replace('/', '_') +def get_config(): + cfg = Config(find_config()) + + try: + access_key = cfg.config['minio']['access_key'] + secret_key = cfg.config['minio']['secret_key'] + except KeyError: + sys.stderr.write('WARNING! Misconfiguration: section [minio] of config.ini doesn\'t contain access_key or secret_key.\n') + return cfg + + if not access_key and not secret_key: + if not os.path.exists('/etc/drakcore/minio.env'): + raise RuntimeError('ERROR! MinIO access credentials are not configured (and can not be auto-detected), unable to start.\n') - with open('/etc/drakcore/config.ini', 'r') as f: - data = f.read() - data = data.replace('{MINIO_ACCESS_KEY}', access_key).replace('{MINIO_SECRET_KEY}', secret_key) + with open('/etc/drakcore/minio.env', 'r') as f: + minio_cfg = [line.strip().split('=', 1) for line in f if line.strip() and '=' in line] + minio_cfg = {k: v for k, v in minio_cfg} - with open('/etc/drakcore/config.ini', 'w') as f: - f.write(data) + try: + cfg.config['minio']['access_key'] = minio_cfg['MINIO_ACCESS_KEY'] + cfg.config['minio']['secret_key'] = minio_cfg['MINIO_SECRET_KEY'] + cfg.minio_config = dict(cfg.config.items("minio")) + except KeyError: + sys.stderr.write('WARNING! Misconfiguration: minio.env doesn\'t contain MINIO_ACCESS_KEY or MINIO_SECRET_KEY.\n') - with open('/etc/drakcore/minio.env', 'r') as f: - data = f.read() - data = data.replace('{MINIO_ACCESS_KEY}', access_key).replace('{MINIO_SECRET_KEY}', secret_key) + return cfg + + +def setup_config(): + if os.path.exists('/etc/drakcore/minio.env'): + print('MinIO environment file already exists, skipping...') + return + + print('Generating MinIO environment file...') + access_key = base64.b64encode(os.urandom(30)).decode('ascii').replace('+', '-').replace('/', '_') + secret_key = base64.b64encode(os.urandom(30)).decode('ascii').replace('+', '-').replace('/', '_') with open('/etc/drakcore/minio.env', 'w') as f: - f.write(data) + f.write(f'MINIO_ACCESS_KEY={access_key}\n') + f.write(f'MINIO_SECRET_KEY={secret_key}\n') diff --git a/drakrun/drakrun/config.dist.ini b/drakrun/drakrun/config.dist.ini index d29a2ea12..c6383a11e 100644 --- a/drakrun/drakrun/config.dist.ini +++ b/drakrun/drakrun/config.dist.ini @@ -1,14 +1,27 @@ [redis] +; Redis server +; used for task scheduling and other non-persistent data host=localhost port=6379 [minio] -access_key= -secret_key= +; MinIO server +; used to store job queue and analysis results address=localhost:9000 bucket=karton2 secure=0 +; MinIO access credentials +; +; NOTE: +; if this is empty, credentials will be read from /etc/drakcore/minio.env +; +; you only need to fill this out if you are: +; * using external MinIO server +; * using multi-node setup (drakcore is not on the same machine as drakrun) +access_key= +secret_key= + [drakrun] ; whether guest VMs should have access to the Internet or no net_enable=0 diff --git a/drakrun/drakrun/drakpush.py b/drakrun/drakrun/drakpush.py index a5692b1a9..4c4fcedba 100644 --- a/drakrun/drakrun/drakpush.py +++ b/drakrun/drakrun/drakpush.py @@ -3,6 +3,7 @@ from karton2 import Producer, Config, Resource, Task from drakrun.config import ETC_DIR +from drakrun.util import patch_config def main(): @@ -12,7 +13,7 @@ def main(): parser.add_argument('--timeout', default=600, type=int, help='analysis timeout in seconds', required=False) args = parser.parse_args() - conf = Config(os.path.join(ETC_DIR, 'config.ini')) + conf = patch_config(Config(os.path.join(ETC_DIR, 'config.ini'))) producer = Producer(conf) task = Task({"type": "sample", "stage": "recognized", "platform": "win32"}) diff --git a/drakrun/drakrun/draksetup.py b/drakrun/drakrun/draksetup.py index 642357614..28cf07aa8 100644 --- a/drakrun/drakrun/draksetup.py +++ b/drakrun/drakrun/draksetup.py @@ -46,9 +46,7 @@ def detect_defaults(): conf = configparser.ConfigParser() conf.read(os.path.join(ETC_DIR, "config.ini")) - conf_patched = False - minio_access_key = conf.get('minio', 'access_key') out_interface = conf.get('drakrun', 'out_interface') if not out_interface: @@ -57,24 +55,9 @@ def detect_defaults(): if default_if: logging.info(f"Detected default network interface: {default_if}") conf['drakrun']['out_interface'] = default_if - conf_patched = True else: logging.warning("Unable to detect default network interface.") - if os.path.exists("/etc/drakcore/config.ini"): - if not minio_access_key: - logging.info("Detected single-node setup, copying minio and redis sections from /etc/drakcore/config.ini") - core_conf = configparser.ConfigParser() - core_conf.read("/etc/drakcore/config.ini") - - conf['redis'] = core_conf['redis'] - conf['minio'] = core_conf['minio'] - conf_patched = True - - if conf_patched: - with open(os.path.join(ETC_DIR, "config.ini"), "w") as f: - conf.write(f) - def ensure_zfs(ctx, param, value): if value is not None and ctx.params['storage_backend'] != "zfs": diff --git a/drakrun/drakrun/main.py b/drakrun/drakrun/main.py index ec2993a50..5d991cb00 100644 --- a/drakrun/drakrun/main.py +++ b/drakrun/drakrun/main.py @@ -27,6 +27,7 @@ from drakrun.drakparse import parse_logs from drakrun.config import ETC_DIR, LIB_DIR, InstallInfo from drakrun.storage import get_storage_backend +from drakrun.util import patch_config INSTANCE_ID = None PROFILE_DIR = os.path.join(LIB_DIR, "profiles") @@ -608,7 +609,7 @@ def cmdline_main(): def main(): conf_path = os.path.join(ETC_DIR, "config.ini") - conf = Config(conf_path) + conf = patch_config(Config(conf_path)) if not conf.config.get('minio', 'access_key').strip(): logging.warning(f"Detected blank value for minio access_key in {conf_path}. " diff --git a/drakrun/drakrun/util.py b/drakrun/drakrun/util.py new file mode 100644 index 000000000..cffc91aaf --- /dev/null +++ b/drakrun/drakrun/util.py @@ -0,0 +1,31 @@ +import base64 +import os +import sys + +from karton2 import Config + + +def patch_config(cfg): + try: + access_key = cfg.config['minio']['access_key'] + secret_key = cfg.config['minio']['secret_key'] + except KeyError: + sys.stderr.write('WARNING! Misconfiguration: section [minio] of config.ini doesn\'t contain access_key or secret_key.\n') + return cfg + + if not access_key and not secret_key: + if not os.path.exists('/etc/drakcore/minio.env'): + raise RuntimeError('ERROR! MinIO access credentials are not configured (and can not be auto-detected), unable to start.\n') + + with open('/etc/drakcore/minio.env', 'r') as f: + minio_cfg = [line.strip().split('=', 1) for line in f if line.strip() and '=' in line] + minio_cfg = {k: v for k, v in minio_cfg} + + try: + cfg.config['minio']['access_key'] = minio_cfg['MINIO_ACCESS_KEY'] + cfg.config['minio']['secret_key'] = minio_cfg['MINIO_SECRET_KEY'] + cfg.minio_config = dict(cfg.config.items("minio")) + except KeyError: + sys.stderr.write('WARNING! Misconfiguration: minio.env doesn\'t contain MINIO_ACCESS_KEY or MINIO_SECRET_KEY.\n') + + return cfg diff --git a/examples/README.md b/examples/README.md index c74ba61e3..5ab675562 100644 --- a/examples/README.md +++ b/examples/README.md @@ -19,7 +19,20 @@ In `consumer.py`, we provide an exemplary script which is able to collect the re ### Usage -1. Create a new `config.ini` file in this directory. Copy `[minio]` and `[redis]` sections from `/etc/drakrun/config.py`. -2. Run `python3 consumer.py` -3. Done! The script will capture each completed analysis and print some logs from these analyses as an example. +1. Create a new `config.ini` according to the following template: + ``` + [redis] + host = localhost + port = 6379 + + [minio] + address = localhost:9000 + bucket = karton2 + secure = 0 + access_key = + secret_key = + ``` +2. Fill out `access_key` and `secret_key` fields. Copy these values from `/etc/drakcore/minio.env`. +3. Run `python3 consumer.py` +4. Done! The script will capture each completed analysis and print some logs from these analyses as an example.