From 8e0ace6ebf22ffc9ad2a65fba16a3678c6978dbe Mon Sep 17 00:00:00 2001 From: garanews Date: Wed, 20 May 2020 10:02:08 +0200 Subject: [PATCH 1/5] add support for mans files add support for mans files using mans_to_es library Co-Authored-By: Arcuri Davide --- .../python/timesketch_api_client/sketch.py | 2 +- .../dpkg/timesketch-server.timesketch.default | 4 +- contrib/timesketch-importer.sh | 2 +- data/timesketch.conf | 4 +- .../timesketch_import_client/importer.py | 6 +-- .../python/tools/timesketch_importer.py | 4 +- requirements.txt | 1 + .../src/views/SketchManageTimelines.vue | 2 +- .../frontend/src/views/SketchOverview.vue | 2 +- timesketch/lib/tasks.py | 41 ++++++++++++++++++- timesketch/templates/sketch/timelines.html | 2 +- timesketch/tsctl.py | 2 +- 12 files changed, 56 insertions(+), 16 deletions(-) diff --git a/api_client/python/timesketch_api_client/sketch.py b/api_client/python/timesketch_api_client/sketch.py index 8c5a31f304..3754347542 100644 --- a/api_client/python/timesketch_api_client/sketch.py +++ b/api_client/python/timesketch_api_client/sketch.py @@ -463,7 +463,7 @@ def list_timelines(self): return timelines def upload(self, timeline_name, file_path, index=None): - """Upload a CSV, JSONL, or Plaso file to the server for indexing. + """Upload a CSV, JSONL, mans, or Plaso file to the server for indexing. Args: timeline_name: Name of the resulting timeline. diff --git a/config/dpkg/timesketch-server.timesketch.default b/config/dpkg/timesketch-server.timesketch.default index 85905a231a..1bebe03f5c 100644 --- a/config/dpkg/timesketch-server.timesketch.default +++ b/config/dpkg/timesketch-server.timesketch.default @@ -113,13 +113,13 @@ GOOGLE_OIDC_HOSTED_DOMAIN = None GOOGLE_OIDC_USER_WHITELIST = [] #------------------------------------------------------------------------------- -# Upload and processing of Plaso storage files. +# Upload and processing of Plaso storage or mans files. # To enable this feature you need to configure an upload directory and # how to reach the Redis database used by the distributed task queue. UPLOAD_ENABLED = False -# Folder for temporarily storage of Plaso dump files before being processed and +# Folder for temporarily storage of Plaso dump or mans files before being processed and # inserted into the datastore. UPLOAD_FOLDER = '/tmp' diff --git a/contrib/timesketch-importer.sh b/contrib/timesketch-importer.sh index 8073039ac0..96565a9955 100755 --- a/contrib/timesketch-importer.sh +++ b/contrib/timesketch-importer.sh @@ -14,7 +14,7 @@ # limitations under the License. # This script watches a directory for new files and executes the importer -# on any .plaso, .csv and .jsonl files. To be place in /usr/local/bin/ +# on any .plaso, .mans, .csv and .jsonl files. To be place in /usr/local/bin/ config_file="/etc/timesketch-importer.conf" if [ -s "$config_file" ]; then diff --git a/data/timesketch.conf b/data/timesketch.conf index 8bbe0a2405..9cd094f801 100644 --- a/data/timesketch.conf +++ b/data/timesketch.conf @@ -113,13 +113,13 @@ GOOGLE_OIDC_HOSTED_DOMAIN = None GOOGLE_OIDC_USER_WHITELIST = [] #------------------------------------------------------------------------------- -# Upload and processing of Plaso storage files. +# Upload and processing of Plaso storage or mans files. # To enable this feature you need to configure an upload directory and # how to reach the Redis database used by the distributed task queue. UPLOAD_ENABLED = False -# Folder for temporarily storage of Plaso dump files before being processed and +# Folder for temporarily storage of Plaso dump or mans files before being processed and # inserted into the datastore. UPLOAD_FOLDER = '/tmp' diff --git a/importer_client/python/timesketch_import_client/importer.py b/importer_client/python/timesketch_import_client/importer.py index d661885442..b033558a3b 100644 --- a/importer_client/python/timesketch_import_client/importer.py +++ b/importer_client/python/timesketch_import_client/importer.py @@ -504,7 +504,7 @@ def add_excel_file(self, filepath, **kwargs): self.add_data_frame(data_frame) def add_file(self, filepath, delimiter=','): - """Add a CSV, JSONL or a PLASO file to the buffer. + """Add a CSV, JSONL, mans or a PLASO file to the buffer. Args: filepath: the path to the file to add. @@ -535,7 +535,7 @@ def add_file(self, filepath, delimiter=','): fh, delimiter=delimiter, chunksize=self._threshold_entry): self.add_data_frame(chunk_frame, part_of_iter=True) - elif file_ending == 'plaso': + elif file_ending in ('plaso', 'mans'): self._upload_binary_file(filepath) elif file_ending == 'jsonl': @@ -550,7 +550,7 @@ def add_file(self, filepath, delimiter=','): else: raise TypeError( - 'File needs to have a file extension of: .csv, .jsonl or ' + 'File needs to have a file extension of: .csv, .jsonl, mans or ' '.plaso') def add_json(self, json_entry, column_names=None): diff --git a/importer_client/python/tools/timesketch_importer.py b/importer_client/python/tools/timesketch_importer.py index fc68ea6fe7..66eef8a1f3 100644 --- a/importer_client/python/tools/timesketch_importer.py +++ b/importer_client/python/tools/timesketch_importer.py @@ -51,10 +51,10 @@ def upload_file( return 'Sketch needs to be set' _, _, file_extension = file_path.rpartition('.') - if file_extension.lower() not in ('plaso', 'csv', 'jsonl'): + if file_extension.lower() not in ('plaso', 'mans', 'csv', 'jsonl'): return ( 'File needs to have one of the following extensions: ' - '.plaso, .csv, .jsonl (not {0:s})').format(file_extension.lower()) + '.plaso, .mans, .csv, .jsonl (not {0:s})').format(file_extension.lower()) with importer.ImportStreamer() as streamer: streamer.set_sketch(my_sketch) diff --git a/requirements.txt b/requirements.txt index 939bfdab3e..021dee0894 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,3 +31,4 @@ Werkzeug==0.16.0 WTForms==2.2.1 xlrd==1.2.0 tabulate==0.8.6 +mans_to_es==1.6 \ No newline at end of file diff --git a/timesketch/frontend/src/views/SketchManageTimelines.vue b/timesketch/frontend/src/views/SketchManageTimelines.vue index 53d606641e..0fc2ac2440 100644 --- a/timesketch/frontend/src/views/SketchManageTimelines.vue +++ b/timesketch/frontend/src/views/SketchManageTimelines.vue @@ -48,7 +48,7 @@ limitations under the License.

- Upload a new timeline or choose an existing one from the list below. You can upload either a Plaso storage file, JSONL, or a CSV file. + Upload a new timeline or choose an existing one from the list below. You can upload either a Plaso storage file, mans, JSONL, or a CSV file.
If you are uploading a CSV or JSONL file make sure to read the documentation to learn what columns are needed.

diff --git a/timesketch/frontend/src/views/SketchOverview.vue b/timesketch/frontend/src/views/SketchOverview.vue index 130bb6e34e..63c88e932f 100644 --- a/timesketch/frontend/src/views/SketchOverview.vue +++ b/timesketch/frontend/src/views/SketchOverview.vue @@ -72,7 +72,7 @@ limitations under the License.

- Supported formats are Plaso storage file, JSONL, or a CSV file. + Supported formats are Plaso storage file, mans, JSONL, or a CSV file. If you are uploading a CSV or JSONL file make sure to read the documentation to learn what columns are needed.

diff --git a/timesketch/lib/tasks.py b/timesketch/lib/tasks.py index 07658d89f8..dd2240e81a 100644 --- a/timesketch/lib/tasks.py +++ b/timesketch/lib/tasks.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Celery task for processing Plaso storage files.""" +"""Celery task for processing Plaso storage or mans files.""" from __future__ import unicode_literals @@ -108,6 +108,8 @@ def _get_index_task_class(file_extension): """ if file_extension == 'plaso': index_class = run_plaso + elif file_extension == 'mans': + index_class = run_mans elif file_extension in ['csv', 'jsonl']: index_class = run_csv_jsonl else: @@ -519,3 +521,40 @@ def run_csv_jsonl(file_path, events, timeline_name, index_name, source_type): _set_timeline_status(index_name, status='ready') return index_name + + +@celery.task(track_started=True, base=SqlAlchemyTask) +def run_mans(file_path, events, timeline_name, index_name, source_type): + """Create a Celery task for processing mans file. + + Args: + file_path: Path to the mans file. + events: A string with the events. Not used in mans. + timeline_name: Name of the Timesketch timeline. + index_name: Name of the datastore index. + source_type: Type of file, csv or jsonl. + + Returns: + Name (str) of the index. + """ + # Log information to Celery + message = 'Index timeline [{0:s}] to index [{1:s}] (source: {2:s})' + logging.info(message.format(timeline_name, index_name, source_type)) + + elastic_host = current_app.config['ELASTIC_HOST'] + elastic_port = int(current_app.config['ELASTIC_PORT']) + try: + mte = MansToEs(filename=file_path, name=timeline_name, index=index_name, + es_host=elastic_host, es_port=elastic_port) + mte.run() + except Exception as e: # pylint: disable=broad-except + # Mark the searchindex and timelines as failed and exit the task + error_msg = traceback.format_exc() + _set_timeline_status(index_name, status='fail', error_msg=error_msg) + logging.error('Error: {0!s}\n{1:s}'.format(e, error_msg)) + return None + + # Mark the searchindex and timelines as ready + _set_timeline_status(index_name, status='ready') + + return index_name diff --git a/timesketch/templates/sketch/timelines.html b/timesketch/templates/sketch/timelines.html index 3df4fed06c..c3f4d33fa7 100644 --- a/timesketch/templates/sketch/timelines.html +++ b/timesketch/templates/sketch/timelines.html @@ -12,7 +12,7 @@ {% if sketch.has_permission(current_user, 'write') and upload_enabled %}

Import timeline

-

You can upload either a Plaso storage file, JSONL, or a CSV file.
+

You can upload either a Plaso storage file, mans, JSONL, or a CSV file.
Supported Plaso version: {{ plaso_version }}

diff --git a/timesketch/tsctl.py b/timesketch/tsctl.py index f17901c8b8..d2ba6f70e6 100644 --- a/timesketch/tsctl.py +++ b/timesketch/tsctl.py @@ -385,7 +385,7 @@ def run(self, file_path, sketch_id, username, timeline_name): extension = extension.lstrip('.') filename = os.path.basename(file_path_no_extension) - supported_extensions = ('plaso', 'csv', 'jsonl') + supported_extensions = ('plaso', 'mans', 'csv', 'jsonl') if not os.path.isfile(file_path): sys.exit('No such file: {0:s}'.format(file_path)) From c88be9ca1f8f364c622714f5ea1057cf3fa128c0 Mon Sep 17 00:00:00 2001 From: garanews Date: Wed, 20 May 2020 10:10:22 +0200 Subject: [PATCH 2/5] Update timesketch_importer.py Co-Authored-By: Arcuri Davide --- importer_client/python/tools/timesketch_importer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/importer_client/python/tools/timesketch_importer.py b/importer_client/python/tools/timesketch_importer.py index 66eef8a1f3..5516eb2f2e 100644 --- a/importer_client/python/tools/timesketch_importer.py +++ b/importer_client/python/tools/timesketch_importer.py @@ -54,7 +54,8 @@ def upload_file( if file_extension.lower() not in ('plaso', 'mans', 'csv', 'jsonl'): return ( 'File needs to have one of the following extensions: ' - '.plaso, .mans, .csv, .jsonl (not {0:s})').format(file_extension.lower()) + '.plaso, .mans, .csv, ' + '.jsonl (not {0:s})').format(file_extension.lower()) with importer.ImportStreamer() as streamer: streamer.set_sketch(my_sketch) From a463558d01b1c1edae85ebd7f8badf493eb84059 Mon Sep 17 00:00:00 2001 From: garanews Date: Wed, 20 May 2020 10:16:09 +0200 Subject: [PATCH 3/5] Update tasks.py Co-Authored-By: Arcuri Davide --- timesketch/lib/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/timesketch/lib/tasks.py b/timesketch/lib/tasks.py index dd2240e81a..2e9336e252 100644 --- a/timesketch/lib/tasks.py +++ b/timesketch/lib/tasks.py @@ -29,6 +29,7 @@ from flask import current_app from sqlalchemy import create_engine from elasticsearch.exceptions import RequestError +from mans_to_es import MansToEs from timesketch import create_celery_app from timesketch.lib import errors From 00a6ff52f83a66a14b52070c00ff145c0ea703fb Mon Sep 17 00:00:00 2001 From: garanews Date: Wed, 20 May 2020 10:36:19 +0200 Subject: [PATCH 4/5] Update tasks.py Co-Authored-By: Arcuri Davide --- timesketch/lib/tasks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/timesketch/lib/tasks.py b/timesketch/lib/tasks.py index 2e9336e252..e989a0b554 100644 --- a/timesketch/lib/tasks.py +++ b/timesketch/lib/tasks.py @@ -110,7 +110,7 @@ def _get_index_task_class(file_extension): if file_extension == 'plaso': index_class = run_plaso elif file_extension == 'mans': - index_class = run_mans + index_class = run_mans elif file_extension in ['csv', 'jsonl']: index_class = run_csv_jsonl else: @@ -544,7 +544,7 @@ def run_mans(file_path, events, timeline_name, index_name, source_type): elastic_host = current_app.config['ELASTIC_HOST'] elastic_port = int(current_app.config['ELASTIC_PORT']) - try: + try: mte = MansToEs(filename=file_path, name=timeline_name, index=index_name, es_host=elastic_host, es_port=elastic_port) mte.run() @@ -553,7 +553,7 @@ def run_mans(file_path, events, timeline_name, index_name, source_type): error_msg = traceback.format_exc() _set_timeline_status(index_name, status='fail', error_msg=error_msg) logging.error('Error: {0!s}\n{1:s}'.format(e, error_msg)) - return None + return None # Mark the searchindex and timelines as ready _set_timeline_status(index_name, status='ready') From b347b630c127770bd1490a89a2d5e5f7d59489f0 Mon Sep 17 00:00:00 2001 From: garanews Date: Wed, 20 May 2020 10:36:50 +0200 Subject: [PATCH 5/5] Update tasks.py Co-Authored-By: Arcuri Davide --- timesketch/lib/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/timesketch/lib/tasks.py b/timesketch/lib/tasks.py index e989a0b554..4fcd6761a3 100644 --- a/timesketch/lib/tasks.py +++ b/timesketch/lib/tasks.py @@ -110,7 +110,7 @@ def _get_index_task_class(file_extension): if file_extension == 'plaso': index_class = run_plaso elif file_extension == 'mans': - index_class = run_mans + index_class = run_mans elif file_extension in ['csv', 'jsonl']: index_class = run_csv_jsonl else: