firemisp.py

#!/usr/bin/env python


# FireMisp - Python script for pushing FireEye json alerts
# into MISP over http
#
# Alexander Jaeger (deralexxx)
#
# The MIT License (MIT) see https://github.com/deralexxx/FireMISP/blob/master/LICENSE
#
# Based on the idea of:
# Please see: https://github.com/spcampbell/firestic
#

from http.server import BaseHTTPRequestHandler,HTTPServer
from socketserver import ThreadingMixIn

import simplejson as json

from firemisp_settings import *

# TODO: check that
import sys
import os



filename1 = ""


def init_misp(url, key):
    """

    :param url:
    :type url:
    :param key:
    :type key:
    :return:
    :rtype:
    """
    return PyMISP(url, key, misp_verifycert, 'json')


class MyRequestHandler(BaseHTTPRequestHandler):

    logger.debug("my request handler")
    # ---------- GET handler to check if httpserver up ----------
    def do_GET(self):
        logger.debug("someone get")
        pingresponse = {"name": "FireMisp is up"}
        if self.path == "/ping":
            self.send_response(code=200)
            self.send_header("Content-type:", "text/html")
            self.end_headers()
            self.wfile.write(bytes(str(pingresponse), 'UTF-8'))
            logger.debug(pingresponse)
        else:
            self.send_response(501)
            self.send_header("Content-type:", "text/html")
            self.end_headers()
            self.wfile.write(bytes("Error", 'UTF-8'))

    def _parse_POST(self):
        import cgi
        ctype, pdict = cgi.parse_header(self.headers.getheader('content-type'))
        if ctype == 'multipart/form-data':
            postvars = cgi.parse_multipart(self.rfile, pdict)
        elif ctype == 'application/x-www-form-urlencoded':
            length = int(self.headers.getheader('content-length'))
            postvars = cgi.parse_qs(
                self.rfile.read(length), keep_blank_values=1)
        else:
            postvars = {}

        return postvars

    # -------------- POST handler: where the magic happens --------------
    def do_POST(self):
        logger.debug("someone sended a post")
        # get the posted data and remove newlines
        # https://stackoverflow.com/questions/2121481/python3-http-server-post-example
        length = int(self.headers['Content-Length'])
        post_data2 = self.rfile.read(length).decode('utf-8')
        clean = post_data2.replace('\n', '')
        theJson = json.loads(clean)

        # TODO give something as response back
        self.wfile.write("Lorem Ipsum".encode("utf-8"))

        try:
            # Write the data to a file as well for debugging later on
            import datetime
            filename1 = './testing/real/'+datetime.datetime.now().strftime("%Y%m%d-%H%M%S")+".json"
            f = open(filename1, 'w')

            #f.write(data)
            json.dump(theJson, f)
            f.close

            self.send_response(200)
            self.end_headers()
            #processAlert(theJson)
            # deal with multiple alerts embedded as an array
            if isinstance(theJson['alert'], list):
                # alertJson = theJson
                # del alertJson['alert']
                for element in theJson['alert']:
                    alertJson = {}  # added for Issue #4
                    alertJson['alert'] = element
                    logger.info("Processing FireEye Alert: " + str(alertJson['alert']['id']))
                    processAlert(alertJson)
            else:
                logger.debug("Processing FireEye Alert: " + str(theJson['alert']['id']))
                processAlert(theJson)

        except ValueError as e:
            logger.error("Value Error2: %s %s",e.message,e.args)
            self.send_response(500)

# ---------------- end class MyRequestHandler ----------------


# ---------------- Class handles requests in a separate thread. ----------------

class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):
    pass

# ---------------- end class ThreadedHTTPServer ----------------



def processAlert(p_json):

    """
    create pyFireEyeAlert Instance of the json and makes all the mapping

    :param p_json:
    :type p_json:
    """

    logger.debug(p_json)
    fireinstance = pyFireEyeAlert(p_json)

    # This comment will be added to every attribute for reference
    auto_comment = "Auto generated by FireMisp "+ (fireinstance.alert_id)

    # create a MISP event
    logger.debug("alert %s ",fireinstance.alert_id)

    has_previous_event = True

    event = check_for_previous_events(fireinstance)

    map_alert_to_event(auto_comment, event, fireinstance)


def check_for_previous_events(fireeye_alert):

    """
    Default: no previous event detected


    check for:
        alert_id | ['alert']['id']

    :param fireeye_alert:
    :type fireeye_alert:
    :return:
        event id if an event is there
        false if no event is present
    :rtype:
    """
    event = False

    if event is False:
        # Based on alert id
        if fireeye_alert.alert_id:
            result = misp.search_all(fireeye_alert.alert_id)
            logger.debug("searching for %s result: %s", fireeye_alert.alert_id,result)
            event = check_misp_all_result(result)

            if event is not False:
                logger.debug("found event based on Alert ID")
                return_event = misp.get(str(event))  # not get_event!
                return return_event

        from urllib import quote
        # Based on Alert Url

        if fireeye_alert.alert_url and event is False:
            result = misp.search_all(quote(fireeye_alert.alert_url))
            logger.debug("searching for %s result: %s", fireeye_alert.alert_url,result)
            event = check_misp_all_result(result)
            if event is not False:
                logger.debug("found event based on alert url")

                return_event = misp.get(str(event))  # not get_event!
                return return_event

        # Based on ma_id
        if fireeye_alert.alert_ma_id and event == False:
            result = misp.search_all(quote(fireeye_alert.alert_ma_id))
            logger.debug("searching for %s result: %s", fireeye_alert.alert_ma_id,result)
            event = check_misp_all_result(result)
            if event is not False:
                logger.debug("found event based on MA ID")

                return_event = misp.get(str(event))  # not get_event!
                return return_event


        # check if source machine and destination are already known
        #TODO improve that!

        source_event_1 = None
        dest_event_2 = None
        event_3 = None

        if fireeye_alert.dst_ip and fireeye_alert.alert_src_ip:
            event = check_misp_two_criterias(misp,fireeye_alert.dst_ip,fireeye_alert.alert_src_ip)
            if event is not False:
                logger.debug("found event based on Source IP / Dst IP combination")

                return_event = misp.get(str(event))  # not get_event!
                return return_event

            # no previous event found and source domain set
        if fireeye_alert.attacker_email and fireeye_alert.victim_email:
            event = check_misp_two_criterias(misp, fireeye_alert.attacker_email, fireeye_alert.victim_email)
            if event is not False:
                logger.debug("found event based on src  dst Mail")

                return_event = misp.get(str(event))  # not get_event!
                return return_event

        # no previous event found and source domain set
        if fireeye_alert.alert_src_domain and fireeye_alert.dst_ip:
            event = check_misp_two_criterias(misp,fireeye_alert.alert_src_domain,fireeye_alert.dst_ip)
            if event is not False:
                logger.debug("found event based on src domain / dst IP")

                return_event = misp.get(str(event))  # not get_event!
                return return_event

        # no previous event found and source domain set
        if fireeye_alert.alert_src_domain and fireeye_alert.dst_ip:
            event = check_misp_two_criterias(misp, fireeye_alert.alert_src_domain, fireeye_alert.dst_ip)
            if event is not False:
                logger.debug("found event based on src domain / dst IP")

                return_event = misp.get(str(event))  # not get_event!
                return return_event

        #check if root infection is the same:
        if fireeye_alert.root_infection:
            result = misp.search_all(quote(fireeye_alert.root_infection))
            event =  check_misp_all_result(result)
            if event is not False:
                logger.debug("found event based on root infection")

                return_event = misp.get(str(event))  # not get_event!
                return return_event

    # if one of the above returns a value:
    previous_event = event
    # this looks hacky but it to avoid exceptions if there is no ['message within the result']

    if previous_event != '' and previous_event != False and previous_event != None:
        logger.debug("Will append my data to: %s", previous_event)
        event = misp.get(str(previous_event))  # not get_event!
    else:
        logger.debug("Will create a new event for it")
        # TODO: set occured day
        # misp is expecting: datetime.date.today().isoformat()
        if fireeye_alert.occured:
            logger.debug("Date will be %s", fireeye_alert.occured)
            # event = misp.new_event(0, 2, 0, "Auto generated by FireMisp " + fireinstance.alert_id,str(fireinstance.occured))
            event = misp.new_event(0, 2, 0, "Auto generated by FireMisp " + fireeye_alert.alert_id)

        else:
            event = misp.new_event(0, 2, 0, "Auto generated by FireMisp " + fireeye_alert.alert_id)
    return event


def check_misp_two_criterias(misp,criteria1,criteria2):

    logger.debug("Search for two criteria: %s %s",criteria1,criteria2)
    event1=False
    event2=False

    result1 = misp.search_all(criteria1)
    result2 = misp.search_all(criteria2)

    event1 = check_misp_all_result(result1)
    event2 = check_misp_all_result(result2)
    if event1 is not False and event2 is not False:
        if event1 == event2:
            logger.debug("Found a match with two given criteria")
            return event1
    #if no common event for both criterias exist, return false
    return False


def check_misp_all_result(result):
    logger.debug("Checking %s if it contains previous events",result)
    if 'message' in result:
        if result['message'] == 'No matches.':
            logger.error("No previous event found")
            # has really no event
            return False
    elif 'Event' in result:
        for e in result['response']:
            logger.debug("found a previous event!")
            previous_event = e['Event']['id']
            return previous_event
            break
    else:
        for e in result['response']:
            logger.debug("found a previous event2!")
            previous_event = e['Event']['id']
            return previous_event
            break


def map_alert_to_event(auto_comment, event, fireeye_alert):
    """

    START THE MAPPING here
    general info that should be there in every alert
    internal reference the alert ID


    :param auto_comment:
    :type auto_comment:
    :param event:
    :type event:
    :param fireeye_alert:
    :type fireeye_alert:
    """

    misp.add_internal_text(event, fireeye_alert.alert_id, to_ids=False, comment=auto_comment)
    ### Start Tagging here
    # TLP change it if you want to change default TLP
    logger.debug("asdasd %s",event['Event'])
    misp.add_tag(event['Event'], tag="tlp:amber")
    # General detected by a security system. So reflect in a tag
    misp.add_tag(event, "veris:discovery_method=\"Int - security alarm\"")
    # Severity Tag + Threat level of the Event
    if fireeye_alert.alert_severity:
        if fireeye_alert.alert_severity == 'majr':
            misp.add_tag(event, "csirt_case_classification:criticality-classification=\"1\"")
            # upgrade Threat level if set already
            misp.change_threat_level(event, 1)
        elif fireeye_alert.alert_severity == 'minr':
            misp.add_tag(event, "csirt_case_classification:criticality-classification=\"3\"")
            misp.add_tag(event, "veris:impact:overall_rating = \"Insignificant\"")
            misp.change_threat_level(event, 3)
        else:
            misp.add_tag(event, "csirt_case_classification:criticality-classification=\"3\"")
            misp.add_tag(event, "veris:impact:overall_rating = \"Unknown\"")
            misp.change_threat_level(event, 4)
    else:
        logger.info("No Event severity found")

    # Url of the original Alert
    if fireeye_alert.alert_url:
        misp.add_internal_link(event, fireeye_alert.alert_url, False, "Alert url: " +auto_comment)

    if fireeye_alert.alert_ma_id:
        misp.add_internal_text(event, fireeye_alert.alert_ma_id, False, "Alert ID "+auto_comment)


    # infos about the product detected it
    if fireeye_alert.product:
        if fireeye_alert.product == 'EMAIL_MPS' or fireeye_alert.product == 'Email MPS':
            misp.add_tag(event, "veris:action:social:vector=\"Email\"")
        elif fireeye_alert.product == 'Web MPS' or fireeye_alert.product == 'Web_MPS':
            misp.add_tag(event, "veris:action:malware:vector=\"Web drive-by\"")

    # if attack was by E-Mail
    if fireeye_alert.attacker_email:
        result_attribute = misp.add_email_src(event, fireeye_alert.attacker_email, False, auto_comment,distribution=5)

        for temp in result_attribute['Event']['Attribute']:
            attribute_id = temp
            break
        # TODO: that needs to be reviewed
        misp.add_tag(attribute_id, "PAP:WHITE", attribute=True)

    if fireeye_alert.alert_src_domain:
        misp.add_domain(event,fireeye_alert.alert_src_domain,"Payload delivery",False,auto_comment)

    if fireeye_alert.mail_subject:
        misp.add_email_subject(event, fireeye_alert.mail_subject, False, auto_comment)
    if fireeye_alert.victim_email:
        #TODO: Addd ldap details here as well!

        #veris:attribute:confidentiality:data_victim = "Employee"

        misp.add_email_dst(event, fireeye_alert.victim_email, 'Payload delivery', False, auto_comment)
    if fireeye_alert.malware_md5:
        logger.debug("Malware within the event %s", fireeye_alert.malware_md5)
        misp.add_hashes(event, "Payload delivery", fireeye_alert.malware_file_name, fireeye_alert.malware_md5, None, None,
                        None, auto_comment, False)
    if fireeye_alert.malware_http_header:
        misp.add_traffic_pattern(event, fireeye_alert.malware_http_header, 'Network activity', False, auto_comment)
    if fireeye_alert.alert_src_ip:
        misp.add_target_machine(event, fireeye_alert.alert_src_ip, False, auto_comment, None)


    if fireeye_alert.root_infection:
        misp.add_internal_comment(event,fireeye_alert.root_infection,False,"Root infection",None)

    if fireeye_alert.smtp_header:
        misp.add_internal_text(event,fireeye_alert.smtp_header,False,"smtp_header "+auto_comment)

        from email import parser
        msg = parser.Parser().parsestr(fireeye_alert.smtp_header, headersonly=True)

        if 'From' in msg:
            logger.debug("from found %s",msg['From'])
            misp.add_email_src(event,email=msg['From'], to_ids=False, comment="From: "+auto_comment)

        if 'To' in msg:
            #TODO: Add LDAP Info for the victim!

            logger.debug("to found %s",msg['To'])
            misp.add_email_dst(event,email=msg['To'],category='Payload delivery', to_ids=False,comment="to: "+auto_comment)
            misp.add_internal_other(event,reference=msg['To'],to_ids=False,comment="to: "+auto_comment)

            #TODO hacky to get a real mail adress
            msg_to = str.replace(msg['To'],"<","")
            msg_to = str.replace(msg_to, ">", "")
            fireeye_alert.victim_email=msg_to
            misp.add_email_dst(event,email=msg_to,category='Payload delivery', to_ids=False,comment="to: "+auto_comment)


            #TODO Add LDAP Again:
            '''

            ownerorg = search_for_mail(msg_to, 'department')
            ownerorg2 = search_for_mail(msg_to, 'company')

            if ownerorg is not False:
                misp.add_target_org(event, target=ownerorg, to_ids=False,
                                    comment="Owner dep of " + msg_to)
                misp.add_target_org(event, target=ownerorg2, to_ids=False,
                                    comment="Owner company of " + msg_to)
            '''


    if fireeye_alert.alert_src_url:
        misp.add_url(event, fireeye_alert.alert_src_url,'Payload delivery',False,auto_comment)

    ignore_destination = False
    if fireeye_alert.dst_ip:
        if fireeye_alert.dst_ip in whitelist:
            ignore_destination = True
        else:
            misp.add_ipdst(event, fireeye_alert.dst_ip, 'Network activity', True, "Destination IP " + auto_comment, None)

    if fireeye_alert.dst_mac and ignore_destination is not True:
        misp.add_traffic_pattern(event, fireeye_alert.dst_mac, 'Network activity', True, "Dst Mac " + auto_comment, None)

    if fireeye_alert.dst_port and ignore_destination is not True:
        misp.add_traffic_pattern(event, fireeye_alert.dst_port, 'Network activity', True, "Dst Port " + auto_comment,
                                 None)

    if fireeye_alert.alert_src_host:
        misp.add_target_machine(event, fireeye_alert.alert_src_host, False, auto_comment, None)

        # TODO Add LDAP again
        '''
        OS = search_host_and_fqdn(fireeye_alert.alert_src_host, 'operatingSystem')
        description = search_host_and_fqdn(fireeye_alert.alert_src_host, 'description')

        ownerorg = search_userprinciplename(description, 'department')
        ownerorg2 = search_userprinciplename(description, 'company')

        if ownerorg is not False:
            misp.add_target_org(event,target=ownerorg,to_ids=False,comment="Owner dep of "+fireeye_alert.alert_src_host)
            misp.add_target_org(event,target=ownerorg2,to_ids=False,comment="Owner company of "+fireeye_alert.alert_src_host)

        logger.debug("Searching for %s result %s", fireeye_alert.alert_src_host, OS)
        misp.add_internal_comment(event, OS, False, "OS of " + fireeye_alert.alert_src_host + auto_comment)
        misp.add_internal_text(event, description, False,
                               "description of " + fireeye_alert.alert_src_host + auto_comment)

        '''

    # TODO: this is not finished yet
    if fireeye_alert.c2services:
        misp.add_domain(event, fireeye_alert.c2_address, 'Network activity', True, auto_comment, None)
        misp.add_domain(event, fireeye_alert.c2_address, 'Network activity', True, auto_comment, None)
        misp.add_ipdst(event, fireeye_alert.c2_address, 'Network activity', True, "C2 IP " + auto_comment, None)
        misp.add_traffic_pattern(event, fireeye_alert.c2_port, 'Network activity', True, "C2 Port " + auto_comment, None)
        misp.add_traffic_pattern(event, fireeye_alert.c2_protocoll, 'Network activity', True, "C2 Protocol " + auto_comment, None)
        misp.add_traffic_pattern(event, fireeye_alert.c2_channel, 'Network activity', True, "C2 Channel " + auto_comment, None)

    # add raw alert as attachment
    if filename1 is not "":
            # TODO: that does not work at the moment
            misp.upload_sample(filename=filename1, category='External analysis', to_ids= False,comment='Full Alert' )
    misp.add_internal_comment(event, str(fireeye_alert.alert), distribution=False, comment="Full Alert")


def main():
    """
    Main method of the tool.

    Args:
        -

    Raises:
        -

    """

    if not HAVE_PYMISP:
        logger.log('error', "Missing dependency, install pymisp (`pip install pymisp`)")
        return


    server = ThreadedHTTPServer((firemisp_ip, firemisp_port), \
									MyRequestHandler)
    logger.info("Starting HTTP server %s %s",firemisp_ip,firemisp_port)
    try:
        server.serve_forever()
    except KeyboardInterrupt:
        logger.error("HTTP Server stopped")


if __name__ == "__main__":

    misp = init_misp(misp_url, misp_key)

    #clean the database for test purposes
    '''
    for i in range (2000,2400,1):
        misp.delete_event(i)
    exit()
    '''

    logging.basicConfig(level=logging.WARNING,
                        filename=firemisp_logfile,
                        format='%(asctime)s - %(levelname)s - %(message)s')
    main()