Merge pull request #21 from DFIRKuiper/v2.0.10

V2.0.10

README.md

@@ -123,6 +123,7 @@ $ git clone https://github.com/DFIRKuiper/Kuiper.git
 Change your current directory location to the new Kuiper directory, and run the **kuiper_install.sh** bash script as root.
 
 ```
+$ sudo apt-get update && sudo apt-get upgrade
 $ cd Kuiper/
 $ sudo ./kuiper_install.sh -install 
 ```
@@ -141,6 +142,31 @@ If everything runs correctly now you should be able to use Kuiper through the li
 
 Happy hunting :).
 
+#### Files Paths
+
+1. before installation, make sure to change the certificate paths if needed
+
+in `kuiper-nginx.conf` change the path of certificates
+
+```
+    ssl_certificate     /home/kuiper/kuiper/cert/MyCertificate.crt;
+    ssl_certificate_key /home/kuiper/kuiper/cert/MyKey.key;
+```
+
+2. also change the socket file path
+
+```
+        proxy_pass              http://unix:/home/kuiper/kuiper/kuiper.sock;
+```
+
+3. change mode permission for the following files
+
+```
+chmod +x ./kuiper_install.sh
+chmod +x ./app/parsers/WinEvents/evtx_dump
+chmod +x ./app/parsers/MFT_Parser/mft_dump
+```
+
 # Issues Tracking and Contribution
 
 We are happy to receive any issues, contribution, and ideas.

VirualMachine/README.md

@@ -1,11 +1,10 @@
 # Virtual Machine
-Note: we recommend to use this VM machine for testing only, it is preferable to use the latests version from releases
 
 ## Download Link
 
 To download a virtual machine with Kuiper and all its dependencies installed, please visit the following link 
 ```
-https://mega.nz/folder/OslmmaxL#7AUJEdU9hiAHHuD19uNF2w
+https://mega.nz/folder/zw0hAIoB#oQhc7ijQMP6lF37-Czeprw
 ```
 
 ## Credentials

app/__init__.py

@@ -2,17 +2,21 @@
 import yaml
 
 import inspect
-from datetime import datetime
+from datetime import datetime, timedelta
 
-from flask import Flask
+from flask import Flask, g , session, render_template
 from flask import request, redirect, url_for
-
-
+ 
+import urllib, json
 from celery import Celery
 from celery.bin import worker
 
 #from flask.ext.celery import Celery
 
+# ldap authentication
+from flask_simpleldap import LDAP, LDAPException
+
+
 app = Flask(__name__)
 
 
@@ -23,8 +27,6 @@
 app.config['UPLOADED_FILES_DEST'] 		= os.path.abspath( ''.join(y['Directories']['artifacts_upload']) )		# uploaded artifacts files
 app.config['UPLOADED_FILES_DEST_RAW'] 	= os.path.abspath( ''.join(y['Directories']['artifacts_upload_raw']) )	# uploaded artifacts raw files
 app.config['PARSER_PATH'] 				= os.path.abspath( ''.join(y['Directories']['app_parsers']) )			# parser folder
-
-
 app.config['CELERY_BROKER_URL'] 	    = y['CELERY']['CELERY_BROKER_URL']
 app.config['CELERY_RESULT_BACKEND']     = y['CELERY']['CELERY_RESULT_BACKEND']
 app.config['CELERY_TASK_ACKS_LATE']     = y['CELERY']['CELERY_TASK_ACKS_LATE']
@@ -40,6 +42,9 @@
     y['CELERY']['CELERY_WORKER_NAME'] = "celery@" + y['CELERY']['CELERY_WORKER_NAME']
 
 
+app.secret_key = y['Kuiper']['secret_key']
+
+
 app.config['DB_NAME']                   = y['MongoDB']['DB_NAME']
 app.config['DB_IP']                   	= y['MongoDB']['DB_IP']
 app.config['DB_PORT']                   = y['MongoDB']['DB_PORT']
@@ -48,6 +53,19 @@
 
 
 
+app.config['LDAP_HOST']                 = y['LDAP_auth']['LDAP_HOST']
+app.config['LDAP_PORT']                 = y['LDAP_auth']['LDAP_PORT'] 
+app.config['LDAP_SCHEMA']               = y['LDAP_auth']['LDAP_SCHEMA']
+app.config['LDAP_USE_SSL']              = y['LDAP_auth']['LDAP_USE_SSL']  
+app.config['LDAP_BASE_DN']              = y['LDAP_auth']['LDAP_BASE_DN']
+app.config['LDAP_USERNAME']             = y['LDAP_auth']['LDAP_USERNAME']
+app.config['LDAP_PASSWORD']             = y['LDAP_auth']['LDAP_PASSWORD']
+
+app.config['LDAP_USER_OBJECT_FILTER']   = y['LDAP_auth']['LDAP_USER_OBJECT_FILTER']
+
+if y['LDAP_auth']['enabled']:
+    ldap = LDAP(app) 
+
 # ===================== Logger - START ===================== # 
 # class Logger to handle all Kuiper logs
 class Logger:
@@ -105,11 +123,95 @@ def logger(self, level , type , message , reason=""):
 
 
 
-from controllers import case_management,admin_management
-
+from controllers import case_management,admin_management,API_management
 
 
 # redirector to the actual home page 
 @app.route('/')
 def home():
-    return redirect(url_for('home_page'))
+    return redirect(url_for('home_page'))
+
+
+# ================= ldap authentication 
+
+def is_authenticated():
+    return 'last_visit' in session and 'user_id' in session
+
+@app.before_request
+def before_request():
+    g.user = None
+    if y['LDAP_auth']['enabled'] == False or request.full_path.startswith('/static') or request.full_path.startswith('/login') or request.full_path.startswith('/logout'):
+        return
+
+    # check if the api token correct
+    if request.full_path.startswith('/api'):
+        request_str =  urllib.unquote(request.data).decode('utf8')
+        request_json = json.loads(request_str)['data']
+
+        if request_json['api_token'] == y['Kuiper']['api_token']:
+            return
+
+    if is_authenticated():
+        session_expired = datetime.now() > session['last_visit'] + timedelta(minutes = 30)
+        if 'last_visit' not in session or session_expired:
+            return redirect(url_for('login', message="Session expired!", url=request.full_path))
+
+        session['last_visit'] = datetime.now()
+        return 
+
+
+    return redirect(url_for('login', message=None))
+
+
+
+
+@app.route('/login', methods=['GET', 'POST'])
+def login():
+    if y['LDAP_auth']['enabled'] == False:
+        return redirect(url_for('home'))
+
+    message = request.args.get('message' , None)
+    if is_authenticated() and message != "Session expired!":
+        return redirect(url_for('home'))
+    if request.method == 'POST':
+        user = request.form['user']
+        passwd = request.form['passwd']
+        try:
+            test = ldap.bind_user(user, passwd)
+        except LDAPException as e :
+            message = str(e)
+            return render_template('login.html' , msg=message )
+
+        if test is None or passwd == '':
+            return render_template('login.html' , msg='Invalid credentials')
+        else:
+            session['user_id'] = request.form['user']
+            session['last_visit'] = datetime.now()
+
+            url = request.args.get('url' , None)
+            if url is None:
+                return redirect(url_for('home'))
+            else:
+                return redirect(url)
+
+    message = request.args.get('message' , None)
+    return render_template('login.html' , msg=message )
+
+@app.route('/logout', methods=['GET'])
+def logout():
+
+    if y['LDAP_auth']['enabled'] == False:
+        return redirect(url_for('home'))
+
+
+    try:
+        del session['user_id']
+        del session['last_visit']
+    except: 
+        pass
+
+    message = request.args.get('message' , None)
+    url     = request.args.get('url' , None)
+    if message is not None and url is not None:
+        return redirect(url_for('login', message=message, url=request.full_path)) 
+    return redirect(url_for('login')) 

app/__version__

@@ -1 +1 @@
-Kuiper:1.0.1
+Kuiper:2.0.10

app/controllers/API_management.py

@@ -0,0 +1,87 @@
+#/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+import json
+import os
+import urllib
+import shutil
+import yaml 
+import zipfile
+import hashlib
+import base64 
+
+
+from flask import Flask
+from flask import request, redirect, render_template, url_for, flash
+from flask import jsonify
+
+from app import app
+
+import parser_management
+
+from app.database.dbstuff import *
+from app.database.elkdb import *
+
+from werkzeug.utils import secure_filename
+from bson import json_util
+from bson.json_util import dumps
+
+
+
+
+
+
+
+# retrive all artifacts records using ajax
+@app.route('/api/get_artifacts/<case_id>', methods=["POST"])
+def api_get_artifacts(case_id):
+    if request.method == "POST":
+
+        request_str =  urllib.unquote(request.data).decode('utf8')
+        request_json = json.loads(request_str)['data']
+        logger.logger(level=logger.DEBUG , type="api", message="Case["+case_id+"]: get artifacts request", reason=json.dumps(request_json))
+
+        # == from - to
+        body = {
+                "from": request_json['seq_num'] * 20000,
+                "size":20000,
+
+            }
+
+        # == query
+        if request_json['query'] != None:
+            request_json['query'] = request_json['query'].strip()
+            query = '*' if request_json['query'] == "" or request_json['query'] is None else request_json['query']
+            body["query"] = {
+                "query_string" : {
+                    "query" : '!(data_type:\"tag\") AND ' + query,
+                    "default_field" : "catch_all"
+                }
+            }
+
+        # == sort 
+        if request_json['sort_by'] != None:
+            order = "asc" if request_json['sort_by']['order'] == 0 else "desc"
+            body["sort"] = {request_json['sort_by']['name'] : {"order" : order}}
+        else:
+            body["sort"] = {'Data.@timestamp' : {"order" : 'asc'}}
+
+
+        # == fields
+        if request_json['fields'] != None :
+            body['_source'] = request_json['fields'].split(",")
+
+        logger.logger(level=logger.DEBUG , type="api", message="Case["+case_id+"]: Query artifacts", reason=json.dumps(body))
+
+
+
+        # request the data from elasticsearch
+        res = db_es.query( case_id, body )
+        if res[0] == False:
+            logger.logger(level=logger.ERROR , type="api", message="Case["+case_id+"]: Failed query artifacts from dataabase", reason=res[1])
+            return json.dumps({'res_total' : 0 , 'res_records' : [] , 'aggs' : []})
+
+
+
+        return json.dumps({'data': res[1] , 'total': res[1]['hits']['total']['value']})
+

app/controllers/admin_management.py

@@ -180,6 +180,9 @@ def get_folder_size(start_path = '.'):
 # =================================================
 
 
+
+
+
 # =================== Cases =======================
 #call admin page will all content from content_management
 @app.route('/admin/')

app/controllers/case_management.py

@@ -162,13 +162,13 @@ def upload_file(file , case_id , machine=None , base64_name=None):
 
     if isUploadMachine:
         # if the option is upload machine
-        machine_name    = source_filename.rstrip('.zip')
+        machine_name    = source_filename.rstrip('.zip').replace("/" , "_")
         machine_id      = case_id + "_" + machine_name
     else:
         # if upload artifacts
         machine_id      = machine
 
-
+    
     try: 
         # ======= validate machine exists or not  
         # check if machine already exists
@@ -289,7 +289,7 @@ def upload_file(file , case_id , machine=None , base64_name=None):
         if isUploadMachine:
             create_m = db_cases.add_machine({
                 'main_case'     : case_id,
-                'machinename'   : machine_name
+                'machinename'   : machine_name.replace("/" , "_")
             })
 
             if create_m[0] == False: # if creating the machine failed
@@ -604,7 +604,7 @@ def main_upload_artifacts(main_case_id,machine_case_id):
 def add_machine(case_id):
     if request.method == 'POST':
         machine_details = {
-            'machinename'   :request.form['machinename'],
+            'machinename'   :request.form['machinename'].replace("/" , "_"),
             'main_case'     :case_id,
             'ip'            :request.form['ip'],
 
@@ -647,8 +647,16 @@ def delete_machine(case_id , machines_list):
             if es_machine:
                 logger.logger(level=logger.INFO , type="case", message="Case["+case_id+"]: Machine ["+machine+"] deleted")
 
-        # delete all records for the machines
-
+            # delete all files
+            try:
+                files_folder    = app.config["UPLOADED_FILES_DEST"]     + "/" + case_id + "/" + machine + "/"
+                raw_folder      = app.config["UPLOADED_FILES_DEST_RAW"] + "/" + case_id + "/" + machine + "/"
+                shutil.rmtree(files_folder, ignore_errors=True)
+                shutil.rmtree(raw_folder, ignore_errors=True)
+                logger.logger(level=logger.DEBUG , type="case", message="Case["+case_id+"]: deleted machine folders ["+machine+"]" , reason=files_folder + "," + raw_folder)
+            except Exception as e:
+                logger.logger(level=logger.ERROR , type="case", message="Case["+case_id+"]: Failed deleting machine folders ["+machine+"]" , reason=str(e))
+
         return redirect(url_for('all_machines',case_id=case_id))
 
 
@@ -1061,7 +1069,6 @@ def case_add_tag_ajax(case_id):
         record_id = None
         ajax_data = json.loads(ajax_str)['data']
 
-
         Data = {
             "tag" : ajax_data['time'] ,
             "@timestamp" : ajax_data['time']
@@ -1096,8 +1103,7 @@ def case_add_tag_ajax(case_id):
 
                 logger.logger(level=logger.INFO , type="case", message="Case["+case_id+"]: Tag created")
                 return json.dumps({"result" : 'successful' , 'tag_id' : up[3][0]})
-
-
+            return json.dumps({"result" : 'successful' , 'tag_id' : up[3][0]})
 
 
 # =================== Alerts =======================