Added support to create timeline from FireEye HX .mans triage

README.md

@@ -27,7 +27,8 @@ Timesketch is an open source tool for collaborative forensic timeline analysis.
 #### Adding timelines
 * [Create timeline from JSON/JSONL/CSV file](docs/CreateTimelineFromJSONorCSV.md)
 * [Create timeline from Plaso file](docs/CreateTimelineFromPlaso.md)
-* [Enable Plaso upload via HTTP](docs/EnablePlasoUpload.md)
+* [Create timeline from MANS file](docs/CreateTimelineFromMANS.md)
+* [Enable Plaso and MANS upload via HTTP](docs/EnablePlasoUpload.md)
 
 #### Using Timesketch
 

docs/CreateTimelineFromMANS.md

@@ -0,0 +1,27 @@
+# Create timeline from MANS file
+
+You need to run version mans-to-es >=  1.0 on your Timesketch server. See the [mans_to_es documentation](https://github.com/LDO-CERT/mans_to_es) for installing mans_to_es and it's dependencies. If you haven't installed Timesketch yet, take a look at the [installation instructions](Installation.md).
+
+When you have working installations of Timesketch and mans_to_es you can, from the command line, do:
+
+```
+$ mans_to_es.py --help
+usage: MANS to ES [-h] [--filename FILENAME] [--name NAME] [--index INDEX]
+                  [--es_host ES_HOST] [--es_port ES_PORT]
+                  [--cpu_count CPU_COUNT] [--bulk_size BULK_SIZE] [--version]
+
+Push .mans information in Elasticsearch index
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --filename FILENAME   Path of the .mans file
+  --name NAME           Timeline name
+  --index INDEX         ES index name
+  --es_host ES_HOST     ES host
+  --es_port ES_PORT     ES port
+  --cpu_count CPU_COUNT
+                        cpu count
+  --bulk_size BULK_SIZE
+                        Bulk size for multiprocessing parsing and upload
+  --version             show program's version number and exit
+```

docs/EnablePlasoUpload.md

@@ -1,4 +1,4 @@
-# Enable Plaso upload via HTTP
+# Enable Plaso and MANS upload via HTTP
 
 To enable uploading and processing of Plaso storage files, there are a couple of things to do.
 
@@ -18,6 +18,10 @@ https://github.com/log2timeline/plaso/wiki/Ubuntu-Packaged-Release
 
     $ sudo apt-get install redis-server
 
+**Install mans_to_es**
+
+    $ pip install mans-to-es
+
 **Configure Timesketch** (/etc/timesketch.conf)
 
     UPLOAD_ENABLED = True

timesketch/lib/forms.py

@@ -243,9 +243,12 @@ class UploadFileForm(BaseForm):
         'file',
         validators=[
             FileRequired(),
-            FileAllowed(['plaso', 'csv', 'jsonl'],
-                        'Allowed file extensions: .plaso, .csv, or .jsonl')
-        ])
+            FileAllowed(
+                ['plaso', 'mans', 'csv', 'jsonl'],
+                'Allowed file extensions: .plaso, .mans, .csv, or .jsonl'
+            )
+        ]
+    )
     name = StringField('Timeline name', validators=[Optional()])
     sketch_id = IntegerField('Sketch ID', validators=[Optional()])
 

timesketch/lib/tasks.py

@@ -104,6 +104,8 @@ def _get_index_task_class(file_extension):
     """
     if file_extension == 'plaso':
         index_class = run_plaso
+    elif file_extension == 'mans':
+        index_class = run_mans
     elif file_extension in ['csv', 'jsonl']:
         index_class = run_csv_jsonl
     else:
@@ -484,3 +486,52 @@ def run_csv_jsonl(source_file_path, timeline_name, index_name, source_type):
     _set_timeline_status(index_name, status='ready')
 
     return index_name
+
+
+@celery.task(track_started=True, base=SqlAlchemyTask)
+def run_mans(source_file_path, timeline_name, index_name, source_type):
+    """Create a Celery task for processing mans file.
+
+    Args:
+        source_file_path: Path to mans file.
+        timeline_name: Name of the Timesketch timeline.
+        index_name: Name of the datastore index.
+        source_type: Type of file, csv or jsonl.
+
+    Returns:
+        Name (str) of the index.
+    """
+    # Log information to Celery
+    message = 'Index timeline [{0:s}] to index [{1:s}] (source: {2:s})'
+    logging.info(message.format(timeline_name, index_name, source_type))
+
+    try:
+        mans_to_es_path = current_app.config['MANS_TO_ES_PATH']
+    except KeyError:
+        mans_to_es_path = 'mans_to_es.py'
+
+    elastic_path = current_app.config['ELASTIC_HOST']
+    elastic_port = current_app.config['ELASTIC_PORT']
+
+    cmd = [
+        mans_to_es_path, '--filename', source_file_path,
+        '--name', timeline_name, '--index', index_name,
+        '--es_host', str(elastic_path), '--es_port', str(elastic_port)
+    ]
+
+    # Run mans_to_es.py
+    try:
+        if six.PY3:
+            subprocess.check_output(
+                cmd, stderr=subprocess.STDOUT, encoding='utf-8')
+        else:
+            subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError as e:
+        # Mark the searchindex and timelines as failed and exit the task
+        _set_timeline_status(index_name, status='fail', error_msg=e.output)
+        return e.output
+
+    # Mark the searchindex and timelines as ready
+    _set_timeline_status(index_name, status='ready')
+
+    return index_name

timesketch/templates/sketch/timelines.html

@@ -12,7 +12,7 @@
     {% if sketch.has_permission(current_user, 'write') and upload_enabled %}
         <div class="card">
             <h4>Import timeline</h4>
-            <p>You can upload either a Plaso storage file, JSONL, or a CSV file.<br>
+            <p>You can upload either a Plaso storage file, MANS, JSONL, or a CSV file.<br>
                 <i>Supported Plaso version: {{ plaso_version }}</i>
             </p>
             <ts-core-upload sketch-id="{{ sketch.id }}" visible="true" btn-text="'Select file'"></ts-core-upload>