Fix MinIO credential generation on drakcore installation

drakcore/debian/drakcore.install

@@ -1,6 +1,5 @@
 drakcore/config.ini /etc/drakcore/
 drakcore/uwsgi.ini /etc/drakcore/
-drakcore/systemd/minio.env /etc/drakcore/
 
 drakcore/systemd/*.service /etc/systemd/system/
 

drakcore/debian/rules

@@ -12,7 +12,6 @@ build:
 	# Config files
 	cp drakcore/config.dist.ini drakcore/config.ini
 	cp drakcore/uwsgi.dist.ini drakcore/uwsgi.ini
-	cp drakcore/systemd/minio.dist.env drakcore/systemd/minio.env
 	# Download minio
 	if [ ! -f drakcore/systemd/minio ] ; then wget -O drakcore/systemd/minio https://debs.icedev.pl/manual/minio ; fi
 	chmod +x drakcore/systemd/minio

drakcore/drakcore/app.py

@@ -7,17 +7,17 @@
 import logging
 
 from flask import Flask, jsonify, request, send_file, redirect, send_from_directory, Response, abort
-from karton2 import Config, Producer, Resource, Task
+from karton2 import Producer, Resource, Task
 from minio.error import NoSuchKey
 from datetime import datetime
 from time import mktime
 
 from drakcore.system import SystemService
-from drakcore.util import find_config
+from drakcore.util import get_config
 
 
 app = Flask(__name__, static_folder='frontend/build/static')
-conf = Config(find_config())
+conf = get_config()
 
 drakmon_cfg = {k: v for k, v in conf.config.items("drakmon")}
 

drakcore/drakcore/config.dist.ini

@@ -1,18 +1,26 @@
 [redis]
+; Redis server
+; used for task scheduling and other non-persistent data
 host=localhost
 port=6379
 
 [minio]
-access_key={MINIO_ACCESS_KEY}
-secret_key={MINIO_SECRET_KEY}
+; MinIO server
+; used to store job queue and analysis results
 address=localhost:9000
 bucket=karton2
 secure=0
 
-[drakmon]
-listen_host=0.0.0.0
-listen_port=5000
+; MinIO access credentials
+;
+; NOTE:
+; if this is empty, credentials will be read from /etc/drakcore/minio.env
+;
+; you only need to fill this out if you are using external MinIO
+access_key=
+secret_key=
 
+[drakmon]
 ; (advanced) disable drak-system service
 ; use this if you connect to external karton
 ; instance instead of the self hosted one

drakcore/drakcore/process.py

@@ -4,10 +4,10 @@
 import functools
 from io import StringIO
 
-from karton2 import Consumer, Config, Karton, LocalResource
+from karton2 import Consumer, Karton, LocalResource
 from minio.error import NoSuchKey
 from drakcore.postprocess import REGISTERED_PLUGINS
-from drakcore.util import find_config
+from drakcore.util import get_config
 
 
 class LocalLogBuffer(logging.Handler):
@@ -89,7 +89,7 @@ def process(self):
 
 
 def main():
-    conf = Config(find_config())
+    conf = get_config()
     processor = AnalysisProcessor(conf, REGISTERED_PLUGINS)
     processor.loop()
 

drakcore/drakcore/system.py

@@ -4,7 +4,7 @@
 from minio import Minio
 from karton2 import Config
 from karton2.services.system import SystemService
-from drakcore.util import find_config
+from drakcore.util import get_config
 
 
 def get_minio_helper(config: Config):
@@ -20,7 +20,7 @@ def get_minio_helper(config: Config):
 
 
 def main():
-    config = Config(find_config())
+    config = get_config()
     service = SystemService(config)
 
     system_disable = config.config["drakmon"].get("system_disable", "1")

drakcore/drakcore/util.py

@@ -1,5 +1,8 @@
 import base64
 import os
+import sys
+
+from karton2 import Config
 
 
 def find_config():
@@ -14,21 +17,43 @@ def find_config():
         raise RuntimeError("Configuration file was not found neither in {} nor {}".format(local_path, etc_path))
 
 
-def setup_config():
-    print('Generating MinIO access key and secret key...')
-    access_key = base64.b64encode(os.urandom(30)).decode('ascii').replace('+', '-').replace('/', '_')
-    secret_key = base64.b64encode(os.urandom(30)).decode('ascii').replace('+', '-').replace('/', '_')
+def get_config():
+    cfg = Config(find_config())
+
+    try:
+        access_key = cfg.config['minio']['access_key']
+        secret_key = cfg.config['minio']['secret_key']
+    except KeyError:
+        sys.stderr.write('WARNING! Misconfiguration: section [minio] of config.ini doesn\'t contain access_key or secret_key.\n')
+        return cfg
+
+    if not access_key and not secret_key:
+        if not os.path.exists('/etc/drakcore/minio.env'):
+            raise RuntimeError('ERROR! MinIO access credentials are not configured (and can not be auto-detected), unable to start.\n')
 
-    with open('/etc/drakcore/config.ini', 'r') as f:
-        data = f.read()
-        data = data.replace('{MINIO_ACCESS_KEY}', access_key).replace('{MINIO_SECRET_KEY}', secret_key)
+        with open('/etc/drakcore/minio.env', 'r') as f:
+            minio_cfg = [line.strip().split('=', 1) for line in f if line.strip() and '=' in line]
+            minio_cfg = {k: v for k, v in minio_cfg}
 
-    with open('/etc/drakcore/config.ini', 'w') as f:
-        f.write(data)
+        try:
+            cfg.config['minio']['access_key'] = minio_cfg['MINIO_ACCESS_KEY']
+            cfg.config['minio']['secret_key'] = minio_cfg['MINIO_SECRET_KEY']
+            cfg.minio_config = dict(cfg.config.items("minio"))
+        except KeyError:
+            sys.stderr.write('WARNING! Misconfiguration: minio.env doesn\'t contain MINIO_ACCESS_KEY or MINIO_SECRET_KEY.\n')
 
-    with open('/etc/drakcore/minio.env', 'r') as f:
-        data = f.read()
-        data = data.replace('{MINIO_ACCESS_KEY}', access_key).replace('{MINIO_SECRET_KEY}', secret_key)
+    return cfg
+
+
+def setup_config():
+    if os.path.exists('/etc/drakcore/minio.env'):
+        print('MinIO environment file already exists, skipping...')
+        return
+
+    print('Generating MinIO environment file...')
+    access_key = base64.b64encode(os.urandom(30)).decode('ascii').replace('+', '-').replace('/', '_')
+    secret_key = base64.b64encode(os.urandom(30)).decode('ascii').replace('+', '-').replace('/', '_')
 
     with open('/etc/drakcore/minio.env', 'w') as f:
-        f.write(data)
+        f.write(f'MINIO_ACCESS_KEY={access_key}\n')
+        f.write(f'MINIO_SECRET_KEY={secret_key}\n')

drakrun/drakrun/config.dist.ini

@@ -1,14 +1,27 @@
 [redis]
+; Redis server
+; used for task scheduling and other non-persistent data
 host=localhost
 port=6379
 
 [minio]
-access_key=
-secret_key=
+; MinIO server
+; used to store job queue and analysis results
 address=localhost:9000
 bucket=karton2
 secure=0
 
+; MinIO access credentials
+;
+; NOTE:
+; if this is empty, credentials will be read from /etc/drakcore/minio.env
+;
+; you only need to fill this out if you are:
+; * using external MinIO server
+; * using multi-node setup (drakcore is not on the same machine as drakrun)
+access_key=
+secret_key=
+
 [drakrun]
 ; whether guest VMs should have access to the Internet or no
 net_enable=0

drakrun/drakrun/drakpush.py

@@ -3,6 +3,7 @@
 
 from karton2 import Producer, Config, Resource, Task
 from drakrun.config import ETC_DIR
+from drakrun.util import patch_config
 
 
 def main():
@@ -12,7 +13,7 @@ def main():
     parser.add_argument('--timeout', default=600, type=int, help='analysis timeout in seconds', required=False)
     args = parser.parse_args()
 
-    conf = Config(os.path.join(ETC_DIR, 'config.ini'))
+    conf = patch_config(Config(os.path.join(ETC_DIR, 'config.ini')))
     producer = Producer(conf)
 
     task = Task({"type": "sample", "stage": "recognized", "platform": "win32"})

drakrun/drakrun/draksetup.py

@@ -46,9 +46,7 @@ def detect_defaults():
 
     conf = configparser.ConfigParser()
     conf.read(os.path.join(ETC_DIR, "config.ini"))
-    conf_patched = False
 
-    minio_access_key = conf.get('minio', 'access_key')
     out_interface = conf.get('drakrun', 'out_interface')
 
     if not out_interface:
@@ -57,24 +55,9 @@ def detect_defaults():
         if default_if:
             logging.info(f"Detected default network interface: {default_if}")
             conf['drakrun']['out_interface'] = default_if
-            conf_patched = True
         else:
             logging.warning("Unable to detect default network interface.")
 
-    if os.path.exists("/etc/drakcore/config.ini"):
-        if not minio_access_key:
-            logging.info("Detected single-node setup, copying minio and redis sections from /etc/drakcore/config.ini")
-            core_conf = configparser.ConfigParser()
-            core_conf.read("/etc/drakcore/config.ini")
-
-            conf['redis'] = core_conf['redis']
-            conf['minio'] = core_conf['minio']
-            conf_patched = True
-
-    if conf_patched:
-        with open(os.path.join(ETC_DIR, "config.ini"), "w") as f:
-            conf.write(f)
-
 
 def ensure_zfs(ctx, param, value):
     if value is not None and ctx.params['storage_backend'] != "zfs":

drakrun/drakrun/main.py

@@ -27,6 +27,7 @@
 from drakrun.drakparse import parse_logs
 from drakrun.config import ETC_DIR, LIB_DIR, InstallInfo
 from drakrun.storage import get_storage_backend
+from drakrun.util import patch_config
 
 INSTANCE_ID = None
 PROFILE_DIR = os.path.join(LIB_DIR, "profiles")
@@ -606,7 +607,7 @@ def cmdline_main():
 
 def main():
     conf_path = os.path.join(ETC_DIR, "config.ini")
-    conf = Config(conf_path)
+    conf = patch_config(Config(conf_path))
 
     if not conf.config.get('minio', 'access_key').strip():
         logging.warning(f"Detected blank value for minio access_key in {conf_path}. "

drakrun/drakrun/util.py

@@ -0,0 +1,31 @@
+import base64
+import os
+import sys
+
+from karton2 import Config
+
+
+def patch_config(cfg):
+    try:
+        access_key = cfg.config['minio']['access_key']
+        secret_key = cfg.config['minio']['secret_key']
+    except KeyError:
+        sys.stderr.write('WARNING! Misconfiguration: section [minio] of config.ini doesn\'t contain access_key or secret_key.\n')
+        return cfg
+
+    if not access_key and not secret_key:
+        if not os.path.exists('/etc/drakcore/minio.env'):
+            raise RuntimeError('ERROR! MinIO access credentials are not configured (and can not be auto-detected), unable to start.\n')
+
+        with open('/etc/drakcore/minio.env', 'r') as f:
+            minio_cfg = [line.strip().split('=', 1) for line in f if line.strip() and '=' in line]
+            minio_cfg = {k: v for k, v in minio_cfg}
+
+        try:
+            cfg.config['minio']['access_key'] = minio_cfg['MINIO_ACCESS_KEY']
+            cfg.config['minio']['secret_key'] = minio_cfg['MINIO_SECRET_KEY']
+            cfg.minio_config = dict(cfg.config.items("minio"))
+        except KeyError:
+            sys.stderr.write('WARNING! Misconfiguration: minio.env doesn\'t contain MINIO_ACCESS_KEY or MINIO_SECRET_KEY.\n')
+
+    return cfg

examples/README.md

@@ -19,7 +19,20 @@ In `consumer.py`, we provide an exemplary script which is able to collect the re
 
 ### Usage
 
-1. Create a new `config.ini` file in this directory. Copy `[minio]` and `[redis]` sections from `/etc/drakrun/config.py`.
-2. Run `python3 consumer.py`
-3. Done! The script will capture each completed analysis and print some logs from these analyses as an example.
+1. Create a new `config.ini` according to the following template:
+   ```
+   [redis]
+   host = localhost
+   port = 6379
+
+   [minio]
+   address = localhost:9000
+   bucket = karton2
+   secure = 0
+   access_key =
+   secret_key =
+   ```
+2. Fill out `access_key` and `secret_key` fields. Copy these values from `/etc/drakcore/minio.env`.
+3. Run `python3 consumer.py`
+4. Done! The script will capture each completed analysis and print some logs from these analyses as an example.