Added support for importing FireEye HX mans files.

add support for FireEye HX .mans files

api_client/python/timesketch_api_client/sketch.py

@@ -463,7 +463,7 @@ def list_timelines(self):
         return timelines
 
     def upload(self, timeline_name, file_path, index=None):
-        """Upload a CSV, JSONL, or Plaso file to the server for indexing.
+        """Upload a CSV, JSONL, mans, or Plaso file to the server for indexing.
 
         Args:
             timeline_name: Name of the resulting timeline.

config/dpkg/timesketch-server.timesketch.default

@@ -113,13 +113,13 @@ GOOGLE_OIDC_HOSTED_DOMAIN = None
 GOOGLE_OIDC_USER_WHITELIST = []
 
 #-------------------------------------------------------------------------------
-# Upload and processing of Plaso storage files.
+# Upload and processing of Plaso storage or mans files.
 
 # To enable this feature you need to configure an upload directory and
 # how to reach the Redis database used by the distributed task queue.
 UPLOAD_ENABLED = False
 
-# Folder for temporarily storage of Plaso dump files before being processed and
+# Folder for temporarily storage of Plaso dump or mans files before being processed and
 # inserted into the datastore.
 UPLOAD_FOLDER = '/tmp'
 

contrib/timesketch-importer.sh

@@ -14,7 +14,7 @@
 # limitations under the License.
 
 # This script watches a directory for new files and executes the importer
-# on any .plaso, .csv and .jsonl files. To be place in /usr/local/bin/
+# on any .plaso, .mans, .csv and .jsonl files. To be place in /usr/local/bin/
 
 config_file="/etc/timesketch-importer.conf"
 if [ -s "$config_file" ]; then

data/timesketch.conf

@@ -113,13 +113,13 @@ GOOGLE_OIDC_HOSTED_DOMAIN = None
 GOOGLE_OIDC_USER_WHITELIST = []
 
 #-------------------------------------------------------------------------------
-# Upload and processing of Plaso storage files.
+# Upload and processing of Plaso storage or mans files.
 
 # To enable this feature you need to configure an upload directory and
 # how to reach the Redis database used by the distributed task queue.
 UPLOAD_ENABLED = False
 
-# Folder for temporarily storage of Plaso dump files before being processed and
+# Folder for temporarily storage of Plaso dump or mans files before being processed and
 # inserted into the datastore.
 UPLOAD_FOLDER = '/tmp'
 

importer_client/python/timesketch_import_client/importer.py

@@ -504,7 +504,7 @@ def add_excel_file(self, filepath, **kwargs):
         self.add_data_frame(data_frame)
 
     def add_file(self, filepath, delimiter=','):
-        """Add a CSV, JSONL or a PLASO file to the buffer.
+        """Add a CSV, JSONL, mans or a PLASO file to the buffer.
 
         Args:
             filepath: the path to the file to add.
@@ -535,7 +535,7 @@ def add_file(self, filepath, delimiter=','):
                         fh, delimiter=delimiter,
                         chunksize=self._threshold_entry):
                     self.add_data_frame(chunk_frame, part_of_iter=True)
-        elif file_ending == 'plaso':
+        elif file_ending in ('plaso', 'mans'):
             self._upload_binary_file(filepath)
 
         elif file_ending == 'jsonl':
@@ -550,7 +550,7 @@ def add_file(self, filepath, delimiter=','):
 
         else:
             raise TypeError(
-                'File needs to have a file extension of: .csv, .jsonl or '
+                'File needs to have a file extension of: .csv, .jsonl, mans or '
                 '.plaso')
 
     def add_json(self, json_entry, column_names=None):

importer_client/python/tools/timesketch_importer.py

@@ -51,10 +51,11 @@ def upload_file(
         return 'Sketch needs to be set'
 
     _, _, file_extension = file_path.rpartition('.')
-    if file_extension.lower() not in ('plaso', 'csv', 'jsonl'):
+    if file_extension.lower() not in ('plaso', 'mans', 'csv', 'jsonl'):
         return (
             'File needs to have one of the following extensions: '
-            '.plaso, .csv, .jsonl (not {0:s})').format(file_extension.lower())
+            '.plaso, .mans, .csv, '
+            '.jsonl (not {0:s})').format(file_extension.lower())
 
     with importer.ImportStreamer() as streamer:
         streamer.set_sketch(my_sketch)

requirements.txt

@@ -31,3 +31,4 @@ Werkzeug==0.16.0
 WTForms==2.2.1
 xlrd==1.2.0
 tabulate==0.8.6
+mans_to_es==1.6

timesketch/frontend/src/views/SketchManageTimelines.vue

@@ -48,7 +48,7 @@ limitations under the License.
 
             <b-message type="is-success">
               <p>
-                Upload a new timeline or choose an existing one from the list below. You can upload either a Plaso storage file, JSONL, or a CSV file.
+                Upload a new timeline or choose an existing one from the list below. You can upload either a Plaso storage file, mans, JSONL, or a CSV file.
                 <br>
                 If you are uploading a CSV or JSONL file make sure to read the <a href="https://github.com/google/timesketch/blob/master/docs/Users-Guide.md#adding-timelines" rel="noreferrer" target="_blank">documentation</a> to learn what columns are needed.
               </p>

timesketch/frontend/src/views/SketchOverview.vue

@@ -72,7 +72,7 @@ limitations under the License.
         <div class="card-content">
           <div class="content">
             <p>
-              Supported formats are Plaso storage file, JSONL, or a CSV file.
+              Supported formats are Plaso storage file, mans, JSONL, or a CSV file.
               If you are uploading a CSV or JSONL file make sure to read the <a href="https://github.com/google/timesketch/blob/master/docs/Users-Guide.md#adding-timelines" rel="noreferrer" target="_blank">documentation</a> to learn what columns are needed.
             </p>
             <ts-upload-timeline-form @toggleModal="showUploadTimelineModal = !showUploadTimelineModal"></ts-upload-timeline-form>

timesketch/lib/tasks.py

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Celery task for processing Plaso storage files."""
+"""Celery task for processing Plaso storage or mans files."""
 
 from __future__ import unicode_literals
 
@@ -29,6 +29,7 @@
 from flask import current_app
 from sqlalchemy import create_engine
 from elasticsearch.exceptions import RequestError
+from mans_to_es import MansToEs
 
 from timesketch import create_celery_app
 from timesketch.lib import errors
@@ -108,6 +109,8 @@ def _get_index_task_class(file_extension):
     """
     if file_extension == 'plaso':
         index_class = run_plaso
+    elif file_extension == 'mans':
+        index_class = run_mans
     elif file_extension in ['csv', 'jsonl']:
         index_class = run_csv_jsonl
     else:
@@ -519,3 +522,40 @@ def run_csv_jsonl(file_path, events, timeline_name, index_name, source_type):
     _set_timeline_status(index_name, status='ready')
 
     return index_name
+
+
+@celery.task(track_started=True, base=SqlAlchemyTask)
+def run_mans(file_path, events, timeline_name, index_name, source_type):
+    """Create a Celery task for processing mans file.
+
+    Args:
+        file_path: Path to the mans file.
+        events: A string with the events. Not used in mans.
+        timeline_name: Name of the Timesketch timeline.
+        index_name: Name of the datastore index.
+        source_type: Type of file, csv or jsonl.
+
+    Returns:
+        Name (str) of the index.
+    """
+    # Log information to Celery
+    message = 'Index timeline [{0:s}] to index [{1:s}] (source: {2:s})'
+    logging.info(message.format(timeline_name, index_name, source_type))
+
+    elastic_host = current_app.config['ELASTIC_HOST']
+    elastic_port = int(current_app.config['ELASTIC_PORT'])
+    try:
+        mte = MansToEs(filename=file_path, name=timeline_name, index=index_name,
+                       es_host=elastic_host, es_port=elastic_port)
+        mte.run()
+    except Exception as e:  # pylint: disable=broad-except
+        # Mark the searchindex and timelines as failed and exit the task
+        error_msg = traceback.format_exc()
+        _set_timeline_status(index_name, status='fail', error_msg=error_msg)
+        logging.error('Error: {0!s}\n{1:s}'.format(e, error_msg))
+        return None
+
+    # Mark the searchindex and timelines as ready
+    _set_timeline_status(index_name, status='ready')
+
+    return index_name

timesketch/templates/sketch/timelines.html

@@ -12,7 +12,7 @@
     {% if sketch.has_permission(current_user, 'write') and upload_enabled %}
         <div class="card">
             <h4>Import timeline</h4>
-            <p>You can upload either a Plaso storage file, JSONL, or a CSV file.<br>
+            <p>You can upload either a Plaso storage file, mans, JSONL, or a CSV file.<br>
                 <i>Supported Plaso version: {{ plaso_version }}</i>
             </p>
             <ts-core-upload sketch-id="{{ sketch.id }}" visible="true" btn-text="'Select file'"></ts-core-upload>

timesketch/tsctl.py

@@ -385,7 +385,7 @@ def run(self, file_path, sketch_id, username, timeline_name):
         extension = extension.lstrip('.')
         filename = os.path.basename(file_path_no_extension)
 
-        supported_extensions = ('plaso', 'csv', 'jsonl')
+        supported_extensions = ('plaso', 'mans', 'csv', 'jsonl')
 
         if not os.path.isfile(file_path):
             sys.exit('No such file: {0:s}'.format(file_path))