diff --git a/.gitignore b/.gitignore index 6d2e5694..7ab6f90c 100644 --- a/.gitignore +++ b/.gitignore @@ -130,3 +130,5 @@ Icon Network Trash Folder Temporary Items .apdisk +/configs/ +/data/ diff --git a/.isort.cfg b/.isort.cfg index 879faef0..6a5f484e 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,7 +1,7 @@ [settings] check=1 diff=1 -known_third_party=flask,flask_httpauth,pymongo,pytest,pytz,six +known_third_party=flask,flask_httpauth,jwt,pymongo,pytest,pytz,six,werkzeug known_first_party=medallion not_skip=__init__.py force_sort_within_sections=1 diff --git a/.travis.yml b/.travis.yml index eb16278a..38f9b35a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,4 +23,4 @@ script: after_success: - codecov before_script: - - mongo admin --eval 'db.createUser({user:"travis",pwd:"test",roles:[{role:"root",db:"admin"}]});' + - mongo admin --eval 'db.createUser({user:"root",pwd:"example",roles:[{role:"root",db:"admin"}]});' diff --git a/README.rst b/README.rst index d52c580b..3128ba4a 100644 --- a/README.rst +++ b/README.rst @@ -80,46 +80,83 @@ Make sure medallion is using the same port that your TAXII client will be connec The contains: -- configuration information for the backend plugin +- configuration information for the backend STIX 2.0 data plugin +- configuration information for the backend authorization plugin - a simple user name/password dictionary -To use the Memory back-end plug, include the following in the : +To use the Memory backend plugin, include the following in the : -.. code:: python +.. code:: json { "backend": { "module": "medallion.backends.memory_backend", "module_class": "MemoryBackend", - "filename": + "filename": "" } } -To use the Mongo DB back-end plug, include the following in the : +To use the Directory backend plugin, include the following in the : -.. code:: python +.. code:: json + + { + "backend": { + "module": "medallion.backends.directory_backend", + "module_class": "DirectoryBackend", + "path": "" + } + } + +The directory backend config also contains information for certain requests to the TAXII 2.0 server. +A complete config can be seen in this `example `_ + +The directory backend uses the path pointed to by the path config as its root. Each directory within becomes a TAXII 2.0 +api root. STIX 2.0 bundles as JSON files can be placed within the root, and the contents of each file will be aggregated +into a single collection. + +The directory backend caches the contents of the files in memory and is best suited for frequent reads and +infrequent writes. + +To use the Mongo DB backend plugin, include the following in the : + +.. code:: json { "backend": { "module": "medallion.backends.mongodb_backend", "module_class": "MongoBackend", - "uri": # e.g., "mongodb://localhost:27017/" + "uri": " # e.g., 'mongodb://localhost:27017/'" } } *Note: A Mongo DB should be available at some URL when using the Mongo DB back-end* -A description of the Mongo DB structure expected by the mongo db backend code is -described in `the documentation -`_. +A description of the Mongo DB structure expected by the mongo db STIX 2.0 data backend code is described in +`the documentation `_. -As required by the TAXII specification, *medallion* supports HTTP Basic -authorization. However, the user names and passwords are currently stored in -the in plain text. +As required by the TAXII specification, *medallion* supports HTTP Basic authorization. In addition, *medallion* supports +API Token authorization and JWT authorization. When stored in the , passwords are encrypted. Here is an example: -.. code:: python +.. code:: json + + { + "users": { + "admin": "pbkdf2:sha256:150000$vhWiAWXq$a16882c2eaf4dbb5c55566c93ec256c189ebce855b0081f4903f09a23e8b2344", + "user1": "pbkdf2:sha256:150000$TVpGAgEI$dd391524abb0d9107ff5949ef512c150523c388cfa6490d8556d604f90de329e", + "user2": "pbkdf2:sha256:150000$CUo7l9Vz$3ff2da22dcb84c9ba64e2df4d1ee9f7061c1da4f8506618f53457f615178e3f3" + }, + "api_keys": { + "123456": "admin", + "abcdef": "user1" + } + } + +*Note: the plaintext passwords for the above example are:* + +.. code:: json { "users": { @@ -129,6 +166,19 @@ Here is an example: } } +If JWT authorization is used, a secret key is required in the config: + +.. code:: json + + { + "flask": { + "SECRET_KEY": "CHANGE_ME" + } + } + +A script for generating user passwords is included +`generate_user_password.py `_ + The authorization is enabled using the python package `flask_httpauth `_. Authorization could be enhanced by changing the method "decorated" using @@ -147,6 +197,71 @@ Configs may also contain a "taxii" section as well, as shown below: All TAXII servers require a config, though if any of the sections specified above are missing, they will be filled with default values. +The backend for authorization can also be configured in the : + +To use the Memory Authorization backend plugin, include the following in the : + +.. code:: json + + { + "auth": { + "module": "medallion.backends.auth_memory_backend", + "module_class": "AuthMemoryBackend", + "users": {}, + "api_keys": {} + } + } + +To use the Mongo DB Authorization backend plugin, include the following in the : + +.. code:: json + + { + "auth": { + "module": "medallion.backends.auth_mongodb_backend", + "module_class": "AuthMongodbBackend", + "uri": "mongodb://root:example@localhost:27017/", + "db_name": "auth" + } + } + +The structure expected by the mongo db authorization backend code is: + +.. code:: json + + { + "user": { + "_id": "user@example.com", + "password": "pbkdf2:sha256:150000$vhWiAWXq$a16882c2eaf4dbb5c55566c93ec256c189ebce855b0081f4903f09a23e8b2344", + "company_name": "Example Organization", + "contact_name": "User", + "created": "", + "updated": "" + }, + "api_key": { + "_id": "", + "user_id": "user@example.com", + "created": "", + "last_used_at": "", + "last_used_from": "" + } + } + +A script for adding users and api-keys is included `auth_db_utils.py `_ + +Multiple authorization are supported by *medallion* at the same time and can be added to the : + +.. code:: json + + { + "multi-auth": [ + "basic", + "api_key" + ] + } + +Additional configurations can be seen in `example_configs `_ + We welcome contributions for other back-end plugins. Docker diff --git a/docker_utils/docker_config.json b/docker_utils/docker_config.json index 6e79e5e2..82068cf4 100644 --- a/docker_utils/docker_config.json +++ b/docker_utils/docker_config.json @@ -1,6 +1,6 @@ { "backend": { - "module": "medallion.backends.mongodb_backend", + "module": "medallion.backends.taxii.mongodb_backend", "module_class": "MongoBackend", "uri": "mongodb://root:example@mongo:27017/" }, diff --git a/example_configs/data_for_memory_config.json b/example_configs/data_for_memory_config.json new file mode 100644 index 00000000..cd54860c --- /dev/null +++ b/example_configs/data_for_memory_config.json @@ -0,0 +1,246 @@ +{ + "/discovery": { + "title": "Some TAXII Server", + "description": "This TAXII Server contains a listing of...", + "contact": "string containing contact information", + "default": "http://localhost:5000/api2/", + "api_roots": [ + "http://localhost:5000/api1/", + "http://localhost:5000/api2/", + "http://localhost:5000/trustgroup1/" + ] + }, + "api1": { + "information": { + "title": "General STIX 2.0 Collections", + "description": "A repo for general STIX data.", + "versions": [ + "taxii-2.0" + ], + "max_content_length": 9765625 + }, + "status": { + }, + "collections": [ + ] + }, + "api2": { + "information": { + "title": "STIX 2.0 Indicator Collections", + "description": "A repo for general STIX data.", + "versions": [ + "taxii-2.0" + ], + "max_content_length": 9765625 + }, + "status": [ + ], + "collections": [ + ] + }, + "trustgroup1": { + "information": { + "title": "Malware Research Group", + "description": "A trust group setup for malware researchers", + "versions": [ + "taxii-2.0" + ], + "max_content_length": 9765625 + }, + "status": [ + { + "id": "2d086da7-4bdc-4f91-900e-d77486753710", + "status": "pending", + "request_timestamp": "2016-11-02T12:34:34.12345Z", + "total_count": 4, + "success_count": 1, + "successes": [ + "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade" + ], + "failure_count": 1, + "failures": [ + { + "id": "malware--664fa29d-bf65-4f28-a667-bdb76f29ec98", + "message": "Unable to process object" + } + ], + "pending_count": 2, + "pendings": [ + "indicator--252c7c11-daf2-42bd-843b-be65edca9f61", + "relationship--045585ad-a22f-4333-af33-bfd503a683b5" + ] + }, + { + "id": "2d086da7-4bdc-4f91-900e-f4566be4b780", + "status": "pending", + "request_timestamp": "2016-11-02T12:34:34.12345Z", + "total_objects": 2, + "success_count": 0, + "successes": [ + ], + "failure_count": 0, + "failures": [ + ], + "pending_count": 0, + "pendings": [ + ] + } + ], + "collections": [ + { + "id": "91a7b528-80eb-42ed-a74d-c6fbd5a26116", + "title": "High Value Indicator Collection", + "description": "This data collection is for collecting high value IOCs", + "can_read": true, + "can_write": true, + "media_types": [ + "application/vnd.oasis.stix+json; version=2.0" + ], + "objects": [ + { + "created": "2017-01-27T13:49:53.997Z", + "description": "Poison Ivy", + "id": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111", + "labels": [ + "remote-access-trojan" + ], + "modified": "2017-01-27T13:49:53.997Z", + "name": "Poison Ivy", + "type": "malware" + }, + { + "created": "2014-05-08T09:00:00.000Z", + "id": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade", + "labels": [ + "file-hash-watchlist" + ], + "modified": "2014-05-08T09:00:00.000Z", + "name": "File hash for Poison Ivy variant", + "pattern": "[file:hashes.'SHA-256' = 'ef537f25c895bfa782526529a9b63d97aa631564d5d789c2b765448c8635fb6c']", + "type": "indicator", + "valid_from": "2014-05-08T09:00:00.000000Z" + }, + { + "created": "2014-05-08T09:00:00.000Z", + "id": "relationship--2f9a9aa9-108a-4333-83e2-4fb25add0463", + "modified": "2014-05-08T09:00:00.000Z", + "relationship_type": "indicates", + "source_ref": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade", + "target_ref": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111", + "type": "relationship" + } + ], + "manifest": [ + { + "id": "indicator--a932fcc6-e032-176c-126f-cb970a5a1ade", + "date_added": "2016-11-01T03:04:05Z", + "versions": [ + "2014-05-08T09:00:00.000Z" + ], + "media_types": [ + "application/vnd.oasis.stix+json; version=2.0" + ] + }, + { + "id": "malware--fdd60b30-b67c-11e3-b0b9-f01faf20d111", + "date_added": "2017-01-27T13:49:53.997Z", + "versions": [ + "2017-01-27T13:49:53.997Z" + ], + "media_types": [ + "application/vnd.oasis.stix+json; version=2.0" + ] + } + ] + }, + { + "id": "52892447-4d7e-4f70-b94d-d7f22742ff63", + "title": "Indicators from the past 24-hours", + "description": "This data collection is for collecting current IOCs", + "can_read": true, + "can_write": false, + "media_types": [ + "application/vnd.oasis.stix+json; version=2.0" + ], + "objects": [ + { + "created": "2016-11-03T12:30:59.000Z", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2017-01-27T13:49:53.935Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2016-11-03T12:30:59.000Z" + }, + { + "created": "2016-11-03T12:30:59.000Z", + "description": "Accessing this url will infect your machine with malware.", + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "labels": [ + "url-watchlist" + ], + "modified": "2016-11-03T12:30:59.000Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://x4z9arb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + } + ], + "manifest": [ + { + "id": "indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f", + "date_added": "2016-12-27T13:49:53Z", + "versions": [ + "2016-11-03T12:30:59.000Z", + "2017-01-27T13:49:53.935Z" + ], + "media_types": [ + "application/vnd.oasis.stix+json; version=2.0" + ] + } + ] + }, + { + "id": "64993447-4d7e-4f70-b94d-d7f33742ee63", + "title": "Secret Indicators", + "description": "Non accessilble", + "can_read":false, + "can_write": false, + "media_types": [ + "application/vnd.oasis.stix+json; version=2.0" + ], + "objects": [ + { + "created": "2016-11-03T12:30:59.000Z", + "description": "Accessing this url will infect your machine with malware.", + "id": "indicator--b81f86b9-975b-bb0b-775e-810c5bd45b4f", + "labels": [ + "url-watchlist" + ], + "modified": "2016-11-03T12:30:59.000Z", + "name": "Malicious site hosting downloader", + "pattern": "[url:value = 'http://z4z10farb.cn/4712']", + "type": "indicator", + "valid_from": "2017-01-27T13:49:53.935382Z" + } + ], + "manifest": [ + { + "id": "indicator--b81f86b9-975b-bb0b-775e-810c5bd45b4f", + "date_added": "2016-12-27T13:49:53Z", + "versions": [ + "2016-11-03T12:30:59.000Z", + "2017-01-27T13:49:53.935Z" + ], + "media_types": [ + "application/vnd.oasis.stix+json; version=2.0" + ] + } + ] + } + ] + } +} \ No newline at end of file diff --git a/example_configs/directory_backend_config_auth_from_file.json b/example_configs/directory_backend_config_auth_from_file.json new file mode 100644 index 00000000..c44d7a7d --- /dev/null +++ b/example_configs/directory_backend_config_auth_from_file.json @@ -0,0 +1,51 @@ +{ + "backend": { + "module": "medallion.backends.taxii.directory_backend", + "module_class": "DirectoryBackend", + "path": "./example_configs/directory/", + "discovery": { + "title": "Some TAXII Server", + "description": "This TAXII Server contains a listing of...", + "contact": "string containing contact information", + "host": "http://localhost:5000/" + }, + "api-root": { + "title": "", + "description": "", + "versions": [ + "taxii-2.0" + ], + "max-content-length": 9765625 + }, + "collection": { + "id": "", + "title": "", + "description": "", + "can_read": true, + "can_write": true, + "media_types": [ + "application/vnd.oasis.stix+json; version=2.0" + ] + } + }, + "taxii": { + "max_page_size": 100 + }, + "auth": { + "module": "medallion.backends.auth.memory_auth", + "module_class": "AuthMemoryBackend", + "users": { + "admin": "pbkdf2:sha256:150000$vhWiAWXq$a16882c2eaf4dbb5c55566c93ec256c189ebce855b0081f4903f09a23e8b2344", + "user1": "pbkdf2:sha256:150000$TVpGAgEI$dd391524abb0d9107ff5949ef512c150523c388cfa6490d8556d604f90de329e", + "user2": "pbkdf2:sha256:150000$CUo7l9Vz$3ff2da22dcb84c9ba64e2df4d1ee9f7061c1da4f8506618f53457f615178e3f3" + }, + "api_keys": { + "123456": "admin", + "abcdef": "user1" + } + }, + "multi-auth": [ + "basic", + "api_key" + ] +} diff --git a/example_configs/directory_backend_config_auth_from_mongodb.json b/example_configs/directory_backend_config_auth_from_mongodb.json new file mode 100644 index 00000000..b6417ef6 --- /dev/null +++ b/example_configs/directory_backend_config_auth_from_mongodb.json @@ -0,0 +1,44 @@ +{ + "backend": { + "module": "medallion.backends.taxii.directory_backend", + "module_class": "DirectoryBackend", + "path": "./example_configs/directory/", + "discovery": { + "title": "Some TAXII Server", + "description": "This TAXII Server contains a listing of...", + "contact": "string containing contact information", + "host": "http://localhost:5000/" + }, + "api-root": { + "title": "", + "description": "", + "versions": [ + "taxii-2.0" + ], + "max-content-length": 9765625 + }, + "collection": { + "id": "", + "title": "", + "description": "", + "can_read": true, + "can_write": false, + "media_types": [ + "application/vnd.oasis.stix+json; version=2.0" + ] + } + }, + "taxii": { + "max_page_size": 100 + }, + "auth": { + "module": "medallion.backends.auth.mongo_auth", + "module_class": "AuthMongodbBackend", + "uri": "mongodb://root:example@localhost:27017/", + "db_name": "auth" + }, + "multi-auth": [ + "basic", + "api_key" + ] +} diff --git a/example_configs/memory_backend_config_auth_from_file.json b/example_configs/memory_backend_config_auth_from_file.json new file mode 100644 index 00000000..8ec240f1 --- /dev/null +++ b/example_configs/memory_backend_config_auth_from_file.json @@ -0,0 +1,27 @@ +{ + "backend": { + "module": "medallion.backends.taxii.memory_backend", + "module_class": "MemoryBackend", + "filename": "./example_configs/data_for_memory_config.json" + }, + "taxii": { + "max_page_size": 100 + }, + "auth": { + "module": "medallion.backends.auth.memory_auth", + "module_class": "AuthMemoryBackend", + "users": { + "admin": "pbkdf2:sha256:150000$vhWiAWXq$a16882c2eaf4dbb5c55566c93ec256c189ebce855b0081f4903f09a23e8b2344", + "user1": "pbkdf2:sha256:150000$TVpGAgEI$dd391524abb0d9107ff5949ef512c150523c388cfa6490d8556d604f90de329e", + "user2": "pbkdf2:sha256:150000$CUo7l9Vz$3ff2da22dcb84c9ba64e2df4d1ee9f7061c1da4f8506618f53457f615178e3f3" + }, + "api_keys": { + "123456": "admin", + "abcdef": "user1" + } + }, + "multi-auth": [ + "basic", + "api_key" + ] +} diff --git a/medallion/__init__.py b/medallion/__init__.py index 2c42b77c..bd62587f 100644 --- a/medallion/__init__.py +++ b/medallion/__init__.py @@ -1,64 +1,75 @@ +from collections import OrderedDict +from datetime import datetime, timedelta import importlib +import json import logging +import random -from flask import Flask, Response, current_app, json -from flask_httpauth import HTTPBasicAuth +from flask import Flask, Response, current_app, g +from flask_httpauth import HTTPBasicAuth, HTTPTokenAuth, MultiAuth +import jwt +from werkzeug.security import check_password_hash from .exceptions import BackendError, ProcessingError +from .log import default_request_formatter from .version import __version__ # noqa from .views import MEDIA_TYPE_TAXII_V20 # Console Handler for medallion messages ch = logging.StreamHandler() -ch.setFormatter(logging.Formatter("[%(name)s] [%(levelname)-8s] [%(asctime)s] %(message)s")) +ch.setFormatter(default_request_formatter()) # Module-level logger log = logging.getLogger(__name__) log.addHandler(ch) -application_instance = Flask(__name__) -auth = HTTPBasicAuth() +jwt_auth = HTTPTokenAuth(scheme='JWT') +basic_auth = HTTPBasicAuth() +token_auth = HTTPTokenAuth(scheme='Token') +auth = MultiAuth(None) +DEFAULT_TAXII = {'max_page_size': 100} -def load_app(config_file): - with open(config_file, "r") as f: - configuration = json.load(f) +DEFAULT_AUTH = { + "module": "medallion.backends.auth.memory_auth", + "module_class": "AuthMemoryBackend", + "users": { + "admin": "pbkdf2:sha256:150000$vhWiAWXq$a16882c2eaf4dbb5c55566c93ec256c189ebce855b0081f4903f09a23e8b2344" + }, + "api_keys": { + "123456": "admin", + } +} + +DEFAULT_BACKEND = { + "module": "medallion.backends.taxii.memory_backend", + "module_class": "MemoryBackend", + "filename": "./medallion/test/data/memory_backend_no_data.json" +} - set_config(application_instance, "users", configuration) - set_config(application_instance, "taxii", configuration) - set_config(application_instance, "backend", configuration) - register_blueprints(application_instance) - return application_instance +def set_multi_auth_config(main_auth, *additional_auth): + type_to_app = { + 'jwt': jwt_auth, + 'api_key': token_auth, + 'basic': basic_auth + } + + auth.main_auth = type_to_app[main_auth] + additional_auth = [a for a in additional_auth if a != main_auth] + auth.additional_auth = tuple(type_to_app[a] for a in tuple(OrderedDict.fromkeys(additional_auth))) + + +def set_auth_config(flask_application_instance, config_info): + with flask_application_instance.app_context(): + log.debug("Registering medallion users configuration into {}".format(current_app)) + flask_application_instance.auth_backend = connect_to_backend(config_info) -def set_config(flask_application_instance, prop_name, config): +def set_taxii_config(flask_application_instance, config_info): with flask_application_instance.app_context(): - log.debug("Registering medallion {} configuration into {}".format(prop_name, current_app)) - if prop_name == "taxii": - try: - flask_application_instance.taxii_config = config[prop_name] - except KeyError: - flask_application_instance.taxii_config = {'max_page_size': 100} - elif prop_name == "users": - try: - flask_application_instance.users_backend = config[prop_name] - except KeyError: - log.warning("You did not give user information in your config.") - log.warning("We are giving you the default user information of:") - log.warning("User = user") - log.warning("Pass = pass") - flask_application_instance.users_backend = {"user": "pass"} - elif prop_name == "backend": - try: - flask_application_instance.medallion_backend = connect_to_backend(config[prop_name]) - except KeyError: - log.warning("You did not give backend information in your config.") - log.warning("We are giving medallion the default settings,") - log.warning("which includes a data file of 'default_data.json'.") - log.warning("Please ensure this file is in your CWD.") - back = {'module': 'medallion.backends.memory_backend', 'module_class': 'MemoryBackend', 'filename': None} - flask_application_instance.medallion_backend = connect_to_backend(back) + log.debug("Registering medallion taxii configuration into {}".format(current_app)) + flask_application_instance.taxii_config = config_info def connect_to_backend(config_info): @@ -79,33 +90,34 @@ def connect_to_backend(config_info): raise e -def register_blueprints(flask_application_instance): +def set_backend_config(flask_application_instance, config_info): + with flask_application_instance.app_context(): + log.debug("Registering medallion_backend into {}".format(current_app)) + current_app.medallion_backend = connect_to_backend(config_info) + + +def register_blueprints(app): from medallion.views import collections from medallion.views import discovery from medallion.views import manifest from medallion.views import objects + from medallion.views.others.auth import auth_bp + from medallion.views.others.healthcheck import healthecheck_bp - with flask_application_instance.app_context(): - log.debug("Registering medallion blueprints into {}".format(current_app)) - current_app.register_blueprint(collections.mod) - current_app.register_blueprint(discovery.mod) - current_app.register_blueprint(manifest.mod) - current_app.register_blueprint(objects.mod) - - -@auth.get_password -def get_pwd(username): - if username in current_app.users_backend: - return current_app.users_backend.get(username) - return None + log.debug("Registering medallion blueprints into {}".format(app)) + app.register_blueprint(collections.mod) + app.register_blueprint(discovery.mod) + app.register_blueprint(manifest.mod) + app.register_blueprint(objects.mod) + app.register_blueprint(auth_bp) + app.register_blueprint(healthecheck_bp) -@application_instance.errorhandler(500) def handle_error(error): e = { "title": "InternalError", "http_status": "500", - "description": str(error.args[0]), + "description": str(error), } return Response( response=json.dumps(e), @@ -114,7 +126,6 @@ def handle_error(error): ) -@application_instance.errorhandler(ProcessingError) def handle_processing_error(error): e = { "title": str(error.__class__.__name__), @@ -129,7 +140,6 @@ def handle_processing_error(error): ) -@application_instance.errorhandler(BackendError) def handle_backend_error(error): e = { "title": str(error.__class__.__name__), @@ -141,3 +151,118 @@ def handle_backend_error(error): status=error.status, mimetype=MEDIA_TYPE_TAXII_V20, ) + + +def register_error_handlers(app): + app.register_error_handler(500, handle_error) + app.register_error_handler(ProcessingError, handle_processing_error) + app.register_error_handler(BackendError, handle_backend_error) + + +def jwt_encode(username): + exp = datetime.utcnow() + timedelta(minutes=int(current_app.config.get("JWT_EXP", 60))) + payload = { + 'exp': exp, + 'user': username + } + return jwt.encode(payload, current_app.config['SECRET_KEY'], algorithm='HS256') + + +def jwt_decode(token): + return jwt.decode(token, current_app.config['SECRET_KEY'], algorithms=['HS256']) + + +@jwt_auth.verify_token +def verify_token(token): + current_dt = datetime.utcnow() + try: + decoded_token = jwt_decode(token) + is_authorized = datetime.utcfromtimestamp(float(decoded_token['exp'])) > current_dt + if is_authorized: + g.user = decoded_token['user'] + except jwt.exceptions.InvalidTokenError: + is_authorized = False + + return is_authorized + + +@basic_auth.verify_password +def verify_basic_auth(username, password): + password_hash = current_app.auth_backend.get_password_hash(username) + g.user = username + return False if password_hash is None else check_password_hash(password_hash, password) + + +@token_auth.verify_token +def api_key_auth(api_key): + user = current_app.auth_backend.get_username_for_api_key(api_key) + if not user: + return False + g.user = user + return True + + +def set_trace_id(): + g.trace_id = "{:08x}".format(random.randrange(0, 0x100000000)) + + +def log_after_request(response): + current_app.logger.info(response.status) + return response + + +class TaxiiFlask(Flask): + def __init__(self, *args, **kwargs): + super(TaxiiFlask, self).__init__(*args, **kwargs) + self.auth_backend = None + self.taxii_config = None + + +def create_app(cfg): + app = TaxiiFlask(__name__) + + if isinstance(cfg, dict): + configuration = cfg + else: + with open(cfg, "r") as f: + configuration = json.load(f) + + app.config.from_mapping(**configuration.get('flask', {})) + + app.logger = logging.getLogger('medallion-app') + app.logger.setLevel(logging.INFO) + handler = logging.StreamHandler() + handler.setFormatter(default_request_formatter()) + app.logger.addHandler(handler) + + if not app.debug: + # Shut up the werkzeug logger unless debugging. + logging.getLogger('werkzeug').setLevel(logging.CRITICAL) + + _ = app.before_request(set_trace_id) + _ = app.after_request(log_after_request) + + set_multi_auth_config(*configuration.get('multi-auth', ('basic',))) + + set_auth_config(app, configuration.get("auth", DEFAULT_AUTH)) + + if "auth" not in configuration: + log.warning("You did not give user information in your config.") + log.warning("We are giving you the default user information of:") + log.warning(" User = admin") + log.warning(" Pass = Password0") + + set_taxii_config(app, configuration.get("taxii", DEFAULT_TAXII)) + + set_backend_config(app, configuration.get("backend", DEFAULT_BACKEND)) + + if "backend" not in configuration: + log.warning("You did not give backend information in your config.") + log.warning("We are giving medallion the default settings,") + log.warning("which includes a data file of 'data_for_memory_config.json'.") + log.warning("This file is included in the 'example_configs' directory.") + + register_blueprints(app) + register_error_handlers(app) + + return app diff --git a/medallion/backends/auth/__init__.py b/medallion/backends/auth/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/medallion/backends/auth/base.py b/medallion/backends/auth/base.py new file mode 100644 index 00000000..c16df9bb --- /dev/null +++ b/medallion/backends/auth/base.py @@ -0,0 +1,10 @@ + +class AuthBackend(object): + + def get_password_hash(self, username): + """Given a username provide the password hash for verification.""" + raise NotImplementedError() + + def get_username_for_api_key(self, api_key): + """Given an API key provide the username for verification.""" + raise NotImplementedError() diff --git a/medallion/backends/auth/memory_auth.py b/medallion/backends/auth/memory_auth.py new file mode 100644 index 00000000..89ef5567 --- /dev/null +++ b/medallion/backends/auth/memory_auth.py @@ -0,0 +1,14 @@ +from medallion.backends.auth.base import AuthBackend + + +class AuthMemoryBackend(AuthBackend): + + def __init__(self, users, **kwargs): + self.users = users + self.api_keys = kwargs.get("api_keys", {}) + + def get_password_hash(self, username): + return self.users.get(username) + + def get_username_for_api_key(self, api_key): + return self.api_keys.get(api_key) diff --git a/medallion/backends/auth/mongodb_auth.py b/medallion/backends/auth/mongodb_auth.py new file mode 100644 index 00000000..858134ea --- /dev/null +++ b/medallion/backends/auth/mongodb_auth.py @@ -0,0 +1,44 @@ +import logging + +from medallion.backends.auth.base import AuthBackend + +try: + from pymongo import MongoClient + from pymongo.errors import ConnectionFailure +except ImportError: + raise ImportError("'pymongo' package is required to use this module.") + + +# Module-level logger +log = logging.getLogger(__name__) + + +class AuthMongodbBackend(AuthBackend): + def __init__(self, uri, **kwargs): + try: + self.client = MongoClient(uri) + self.db_name = kwargs["db_name"] + # The ismaster command is cheap and does not require auth. + # self.client.admin.command("ismaster") + except ConnectionFailure: + log.error("Unable to establish a connection to MongoDB server {}".format(uri)) + + def get_password_hash(self, username): + db = self.client[self.db_name] + users = db['users'] + user_obj = users.find_one({"_id": username}) + if user_obj: + return user_obj['password'] + else: + return None + + def get_username_for_api_key(self, api_key): + db = self.client[self.db_name] + api_keys = db['api_keys'] + api_key_obj = api_keys.find_one({"_id": api_key}) + + if api_key_obj: + username = api_key_obj['user_id'] + return username + else: + return None diff --git a/medallion/backends/taxii/__init__.py b/medallion/backends/taxii/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/medallion/backends/base.py b/medallion/backends/taxii/base.py similarity index 100% rename from medallion/backends/base.py rename to medallion/backends/taxii/base.py diff --git a/medallion/backends/taxii/directory_backend.py b/medallion/backends/taxii/directory_backend.py new file mode 100644 index 00000000..f6c6146d --- /dev/null +++ b/medallion/backends/taxii/directory_backend.py @@ -0,0 +1,342 @@ +import datetime +import json +import os +import uuid + +from medallion.backends.taxii.base import Backend +from medallion.exceptions import ProcessingError +from medallion.filters.basic_filter import BasicFilter +from medallion.utils.common import create_bundle, generate_status + + +class DirectoryBackend(Backend): + # access control is handled at the views level + + def __init__(self, path=None, **kwargs): + self.path = path + self.discovery_config = self.init_discovery_config(kwargs.get('discovery', None)) + self.api_root_config = self.init_api_root_config(kwargs.get('api-root', None)) + self.collection_config = self.init_collection_config(kwargs.get('collection', None)) + self.cache = {} + self.statuses = [] + + # noinspection PyMethodMayBeStatic + def init_discovery_config(self, discovery_config): + if not self.path: + raise ProcessingError('path was not specified in the config file', 400) + + if not os.path.isdir(self.path): + raise ProcessingError("directory '{}' was not found".format(self.path), 500) + + return discovery_config + + def update_discovery_config(self): + dc = self.discovery_config + collection_dirs = sorted([f for f in os.listdir(self.path) if os.path.isdir(os.path.join(self.path, f))]) + + if not collection_dirs: + raise ProcessingError('at least one api-root directory is required', 500) + + updated_roots = ['{}{}/'.format(dc['host'], f) for f in collection_dirs] + + self.discovery_config['default'] = updated_roots[0] + self.discovery_config['api_roots'] = updated_roots + + # noinspection PyMethodMayBeStatic + def init_api_root_config(self, api_root_config): + if api_root_config: + return api_root_config + else: + raise ProcessingError('api-root was not specified in the config file', 400) + + # noinspection PyMethodMayBeStatic + def init_collection_config(self, collection_config): + if collection_config: + return collection_config + else: + raise ProcessingError('collection was not specified in the config file', 400) + + def validate_requested_api_root(self, requested_api_root): + api_roots = self.discovery_config['api_roots'] + + host_port = self.discovery_config['default'].rsplit('/', 2)[0] + + full_api_root = '{}/{}/'.format(host_port, requested_api_root) + + return full_api_root in api_roots + + def server_discovery(self): + self.update_discovery_config() + return self.discovery_config + + def get_api_root_information(self, api_root): + self.update_discovery_config() + + api_roots = self.discovery_config['api_roots'] + + for r in api_roots: + c_dir = r.rsplit('/', 2)[1] + + if api_root == c_dir: + i_title = "Indicators from directory '{}'".format(c_dir) + + i = { + "title": i_title, + "description": "", + "versions": self.api_root_config['versions'], + "max-content-length": self.api_root_config['max-content-length'] + } + + return i + + def get_collections(self, api_root, start_index, end_index): + self.update_discovery_config() + + api_roots = self.discovery_config['api_roots'] + + collections = [] + + # Generate a collection object for each api_root + for r in api_roots: + c_dir = r.rsplit('/', 2)[1] + + if api_root == c_dir: + c_id = uuid.uuid3(uuid.NAMESPACE_URL, r) + c_title = "Indicators from directory '{}'".format(c_dir) + + c = { + "id": str(c_id), + "title": c_title, + "description": self.collection_config['description'], + "can_read": self.collection_config['can_read'], + "can_write": self.collection_config['can_write'], + "media_types": self.collection_config['media_types'] + } + + collections.append(c) + + count = len(collections) + + collections = collections if end_index == -1 else collections[start_index:end_index] + + return count, collections + + def get_collection(self, api_root, collection_id): + count, collections = self.get_collections(api_root, 0, -1) + + for c in collections: + if 'id' in c and collection_id == c['id']: + return c + + def set_modified_time_stamp(self, objects, modified): + for o in objects: + o['modified'] = modified + + return objects + + def get_modified_time_stamp(self, fp): + fp_modified = os.path.getmtime(fp) + dt = datetime.datetime.utcfromtimestamp(fp_modified) + modified = '{:%Y-%m-%dT%H:%M:%S.%fZ}'.format(dt) + + return modified + + def delete_from_cache(self, api_root): + p = os.path.join(self.path, api_root) + files = [f for f in os.listdir(p) if os.path.isfile(os.path.join(p, f)) and f.endswith('.json')] + + for f in self.cache[api_root]['files'].keys(): + if f not in files: + del self.cache[api_root]['files'][f] + + def add_to_cache(self, api_root, api_root_modified, file_name, file_modified): + fp = os.path.join(self.path, api_root, file_name) + + u_objects = [] + + with open(fp, 'r') as raw_json: + try: + stix2 = json.load(raw_json) + + if stix2.get('type', '') == 'bundle' and stix2.get('spec_version', '') == '2.0': + objects = stix2.get('objects', []) + u_objects = self.set_modified_time_stamp(objects, file_modified) + + if api_root not in self.cache: + self.cache[api_root] = {'modified': '', 'files': {}} + + self.cache[api_root]['modified'] = api_root_modified + self.cache[api_root]['files'][file_name] = {'modified': file_modified, 'objects': u_objects} + except Exception as e: + raise ProcessingError('error adding objects to cache', 500, e) + finally: + return u_objects + + def with_cache(self, api_root): + api_root_path = os.path.join(self.path, api_root) + api_root_modified = self.get_modified_time_stamp(api_root_path) + + if api_root in self.cache: + if self.cache[api_root]['modified'] == api_root_modified: + # Return objects from cache + objects = [] + for k, v in self.cache[api_root]['files'].items(): + objects.extend(v['objects']) + return objects + else: + # Cleanup the cache + self.delete_from_cache(api_root) + + # Add to the cache and return objects for collection + dir_list = os.listdir(api_root_path) + files = [f for f in dir_list if os.path.isfile(os.path.join(api_root_path, f)) and f.endswith('.json')] + + objects = [] + for f in files: + fp = os.path.join(api_root_path, f) + file_modified = self.get_modified_time_stamp(fp) + + cached_files = self.cache[api_root]['files'] + if f in cached_files and cached_files[f]['modified'] == file_modified: + objects.extend(cached_files[f]['objects']) + else: + u_objects = self.add_to_cache(api_root, api_root_modified, f, file_modified) + objects.extend(u_objects) + return objects + else: + # Update the cache and return the objects for the collection + dir_list = os.listdir(api_root_path) + files = [f for f in dir_list if os.path.isfile(os.path.join(api_root_path, f)) and f.endswith('.json')] + + objects = [] + for f in files: + fp = os.path.join(api_root_path, f) + file_modified = self.get_modified_time_stamp(fp) + + u_objects = self.add_to_cache(api_root, api_root_modified, f, file_modified) + objects.extend(u_objects) + return objects + + def get_objects_without_bundle(self, api_root, collection_id, filter_args, allowed_filters): + self.update_discovery_config() + + if self.validate_requested_api_root(api_root): + # Get the collection + collection = None + num_collections, collections = self.get_collections(api_root, 0, -1) + + for c in collections: + if 'id' in c and collection_id == c['id']: + collection = c + break + + if not collection: + raise ProcessingError("collection for api-root '{}' was not found".format(api_root), 500) + + # Add the objects to the collection + collection['objects'] = self.with_cache(api_root) + + # Filter the collection + filtered_objects = [] + + if filter_args: + full_filter = BasicFilter(filter_args) + filtered_objects.extend( + full_filter.process_filter( + collection.get('objects', []), + allowed_filters, + collection.get('manifest', []) + ) + ) + else: + filtered_objects.extend(collection.get('objects', [])) + + return filtered_objects + + def get_objects(self, api_root, collection_id, filter_args, allowed_filters, start_index, end_index): + # print('start_index: {}, end_index: {}'.format(start_index, end_index)) + + objects = self.get_objects_without_bundle(api_root, collection_id, filter_args, allowed_filters) + + objects.sort(key=lambda x: datetime.datetime.strptime(x['modified'], '%Y-%m-%dT%H:%M:%S.%fZ')) + + count = len(objects) + + objects = objects if end_index == -1 else objects[start_index:end_index] + + return count, create_bundle(objects) + + def get_object(self, api_root, collection_id, object_id, filter_args, allowed_filters): + objects = self.get_objects_without_bundle(api_root, collection_id, filter_args, allowed_filters) + + req_object = [i for i in objects if i['id'] == object_id] + + if len(req_object) == 1: + return create_bundle(req_object) + + def get_object_manifest(self, api_root, collection_id, filter_args, allowed_filters, start_index, end_index): + self.update_discovery_config() + + if self.validate_requested_api_root(api_root): + count, collections = self.get_collections(api_root, 0, -1) + + for collection in collections: + if 'id' in collection and collection_id == collection['id']: + manifest = collection.get('manifest', []) + if filter_args: + full_filter = BasicFilter(filter_args) + manifest = full_filter.process_filter( + manifest, + allowed_filters, + None + ) + + count = len(manifest) + + manifest = manifest if end_index == -1 else manifest[start_index:end_index] + + return count, manifest + + def add_objects(self, api_root, collection_id, objs, request_time): + failed = 0 + succeeded = 0 + pending = 0 + successes = [] + failures = [] + + file_name = '{}--{}.{}'.format(request_time, objs['id'], 'json') + p = os.path.join(self.path, api_root) + fp = os.path.join(p, file_name) + + try: + add_objs = objs['objects'] + num_objs = len(add_objs) + + try: + # Each add_object request writes the provided bundle to a new file + with open(fp, 'w') as out_file: + out_file.write(json.dumps(objs, indent=4, sort_keys=True)) + + succeeded += num_objs + successes = list(map(lambda x: x['id'], add_objs)) + + # Update the cache after the file is written + self.with_cache(api_root) + except IOError: + failed += num_objs + failures = list(map(lambda x: x['id'], add_objs)) + + except Exception as e: + raise ProcessingError('error adding objects', 500, e) + + status = generate_status(request_time, 'complete', succeeded, failed, + pending, successes_ids=successes, failures=failures) + + self.statuses.append(status) + + return status + + def get_status(self, api_root, status_id): + for s in self.statuses: + if status_id == s['id']: + return s diff --git a/medallion/backends/memory_backend.py b/medallion/backends/taxii/memory_backend.py similarity index 96% rename from medallion/backends/memory_backend.py rename to medallion/backends/taxii/memory_backend.py index 3cceaf9f..198ecdab 100644 --- a/medallion/backends/memory_backend.py +++ b/medallion/backends/taxii/memory_backend.py @@ -1,17 +1,18 @@ import copy import json -from ..exceptions import ProcessingError -from ..filters.basic_filter import BasicFilter -from ..utils.common import create_bundle, generate_status, iterpath -from .base import Backend +from medallion.backends.taxii.base import Backend +from medallion.exceptions import ProcessingError +from medallion.filters.basic_filter import BasicFilter +from medallion.utils.common import create_bundle, generate_status, iterpath class MemoryBackend(Backend): # access control is handled at the views level - def __init__(self, filename=None, **kwargs): + def __init__(self, **kwargs): + filename = kwargs.get("filename") if filename: self.load_data_from_file(filename) else: diff --git a/medallion/backends/mongodb_backend.py b/medallion/backends/taxii/mongodb_backend.py similarity index 93% rename from medallion/backends/mongodb_backend.py rename to medallion/backends/taxii/mongodb_backend.py index c88c31fd..27094f15 100644 --- a/medallion/backends/mongodb_backend.py +++ b/medallion/backends/taxii/mongodb_backend.py @@ -1,13 +1,17 @@ import logging -from pymongo import ASCENDING, MongoClient -from pymongo.errors import ConnectionFailure, ServerSelectionTimeoutError +from medallion.backends.taxii.base import Backend +from medallion.exceptions import MongoBackendError, ProcessingError +from medallion.filters.mongodb_filter import MongoDBFilter +from medallion.utils.common import (create_bundle, format_datetime, + generate_status, get_timestamp) + +try: + from pymongo import ASCENDING, MongoClient + from pymongo.errors import ConnectionFailure, ServerSelectionTimeoutError +except ImportError: + raise ImportError("'pymongo' package is required to use this module.") -from ..exceptions import MongoBackendError, ProcessingError -from ..filters.mongodb_filter import MongoDBFilter -from ..utils.common import (create_bundle, format_datetime, generate_status, - get_timestamp) -from .base import Backend # Module-level logger log = logging.getLogger(__name__) @@ -29,7 +33,8 @@ class MongoBackend(Backend): # access control is handled at the views level - def __init__(self, uri=None, **kwargs): + def __init__(self, **kwargs): + uri = kwargs.get("uri") try: self.client = MongoClient(uri) except ConnectionFailure: diff --git a/medallion/filters/basic_filter.py b/medallion/filters/basic_filter.py index 76c86cc9..1e367958 100644 --- a/medallion/filters/basic_filter.py +++ b/medallion/filters/basic_filter.py @@ -1,7 +1,7 @@ import copy import datetime as dt -from ..utils.common import convert_to_stix_datetime +from medallion.utils.common import convert_to_stix_datetime class BasicFilter(object): diff --git a/medallion/filters/mongodb_filter.py b/medallion/filters/mongodb_filter.py index 0fc75b25..923a92d0 100644 --- a/medallion/filters/mongodb_filter.py +++ b/medallion/filters/mongodb_filter.py @@ -1,5 +1,5 @@ -from ..utils.common import convert_to_stix_datetime -from .basic_filter import BasicFilter +from medallion.filters.basic_filter import BasicFilter +from medallion.utils.common import convert_to_stix_datetime class MongoDBFilter(BasicFilter): diff --git a/medallion/log.py b/medallion/log.py new file mode 100755 index 00000000..fa715d0f --- /dev/null +++ b/medallion/log.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python + +import json +import logging + +from flask import g, has_request_context, request + + +class RequestFormatter(logging.Formatter): + def format(self, record): + if has_request_context(): + record.method = request.method + record.path = request.full_path.rstrip('?') + record.server_protocol = request.environ.get('SERVER_PROTOCOL') + + source = request.headers.getlist('X-Forwarded-For') + if request.remote_addr not in source: + source.append(request.remote_addr) + + record.source = ",".join(source) + record.user = getattr(g, 'user', '-') + record.trace_id = getattr(g, 'trace_id', '-') + else: + record.method = '-' + record.path = '-' + record.server_protocol = '-' + record.source = '-' + record.user = '-' + record.trace_id = '-' + + return super(RequestFormatter, self).format(record) + + +def default_request_formatter(): + return RequestFormatter( + '%(asctime)s %(levelname)s [%(name)s] %(trace_id)s' + ' %(source)s %(user)s' + ' "%(method)s %(path)s %(server_protocol)s"' + ' %(message)s' + ) + + +def json_request_formatter(): + return RequestFormatter( + json.dumps({ + "name": "%(name)s", + "levelname": "%(levelname)s", + "asctime": "%(asctime)s", + "source": "%(source)s", + "method": "%(method)s", + "user": "%(user)s", + "path": "%(path)s", + "server_protocol": "%(server_protocol)s", + "message": "%(message)s", + "trace_id": "%(trace_id)s" + }) + ) diff --git a/medallion/scripts/auth_db_utils.py b/medallion/scripts/auth_db_utils.py new file mode 100644 index 00000000..d7d304f5 --- /dev/null +++ b/medallion/scripts/auth_db_utils.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python + +import argparse +import json +import sys +import uuid + +import six +from werkzeug.security import generate_password_hash + +from medallion.utils.common import format_datetime, get_timestamp + +try: + from pymongo import MongoClient + from pymongo.errors import ConnectionFailure +except ImportError: + raise ImportError("'pymongo' package is required to use this module.") + + +def make_connection(uri): + try: + client = MongoClient(uri) + # The ismaster command is cheap and does not require auth. + client.admin.command("ismaster") + + return client + except ConnectionFailure: + print("Unable to establish a connection to MongoDB server {}".format(uri)) + + +def add_auth_data_from_file(client, data): + # Insert the new user. + db = client['auth'] + users = db['users'] + api_keys = db['api_keys'] + + timestamp = get_timestamp() + + data['user']['created'] = format_datetime(timestamp) + data['api_key']['created'] = format_datetime(timestamp) + + users.insert_one(data['user']) + api_keys.insert_one(data['api_key']) + + +def add_api_key_for_user(client, email): + api_key = str(uuid.uuid4()).replace('-', '') + timestamp = get_timestamp() + api_key_obj = { + "_id": api_key, + "user_id": email, + "created": format_datetime(timestamp), + "last_used_at": "", + "last_used_from": "" + } + + # Check that the user exists. If the user exists, insert the new api_key, and update the corresponding user. + db = client['auth'] + users = db['users'] + user = users.find_one({"_id": email}) + + if user: + # Add an api key and print it. + api_keys = db['api_keys'] + api_keys.insert_one(api_key_obj) + + print("new api key: {} added for email: {}".format(api_key, email)) + else: + print("no user with email: {} was found in the database.".format(email)) + + +def add_user(client, user): + # Insert the new user. + db = client['auth'] + users = db['users'] + users.insert_one(user) + + +def main(): + uri = "mongodb://root:example@localhost:27017/" + + parser = argparse.ArgumentParser('medallion mongo-authdb script [OPTIONS]', description='Auth DB Utils') + + group = parser.add_mutually_exclusive_group() + + parser.add_argument('--uri', dest='uri', default=uri, help='Set the Mongo DB connection information') + + group.add_argument('-f', '--file', dest='file', help='Add a user with API key to the Auth DB') + group.add_argument('-u', '--user', dest='user', action='store_true', help='Add a user to the Auth DB') + group.add_argument('-k', '--apikey', dest='apikey', action='store_true', help='Add an API key to an existing user') + + args = parser.parse_args() + + client = make_connection(args.uri) + + if args.file is not None: + with open(args.file, 'r') as i: + data = json.load(i) + add_auth_data_from_file(client, data) + elif args.user: + email = six.moves.input('email address : ').strip() + + password1 = six.moves.input('password : ').strip() + password2 = six.moves.input('verify password : ').strip() + + if password1 != password2: + sys.exit('passwords were not the same') + + company_name = six.moves.input('company name : ').strip() + contact_name = six.moves.input('contact name : ').strip() + add_api_key = six.moves.input('add api key (y/n)? : ').strip() + + password_hash = generate_password_hash(password1) + + timestamp = get_timestamp() + + user = { + "_id": email, + "password": password_hash, + "company_name": company_name, + "contact_name": contact_name, + "created": format_datetime(timestamp), + "updated": format_datetime(timestamp), + } + + add_user(client, user) + + if add_api_key.lower() == 'y': + add_api_key_for_user(client, email) + elif args.apikey: + email = six.moves.input('email address : ') + + add_api_key_for_user(client, email) + + +if __name__ == "__main__": + main() diff --git a/medallion/scripts/generate_user_password.py b/medallion/scripts/generate_user_password.py new file mode 100644 index 00000000..f94a73f2 --- /dev/null +++ b/medallion/scripts/generate_user_password.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python + +import argparse +import textwrap + +import six +from werkzeug.security import check_password_hash, generate_password_hash + +from medallion import __version__ + + +class NewlinesHelpFormatter(argparse.RawDescriptionHelpFormatter): + """Custom help formatter to insert newlines between argument help texts. + """ + def _split_lines(self, text, width): + text = self._whitespace_matcher.sub(" ", text).strip() + txt = textwrap.wrap(text, width) + txt[-1] += "\n" + return txt + + +def _get_argparser(): + """Create and return an ArgumentParser for this application.""" + desc = "medallion generate-user-password script v{0}".format(__version__) + parser = argparse.ArgumentParser( + description=desc, + formatter_class=NewlinesHelpFormatter, + ) + + parser.add_argument( + "--hash-method", + default="sha256", + type=str, + help="The hash method to use (one that hashlib supports).", + ) + + parser.add_argument( + "--salt-length", + default=8, + type=int, + help="The length of the salt in letters.", + ) + + return parser + + +def main(): + parser = _get_argparser() + args = parser.parse_args() + method = "pbkdf2:{}".format(args.hash_method) + salt_length = args.salt_length + + password = six.moves.input("password:\t") + password_hash = generate_password_hash(password, method=method, salt_length=salt_length) + + password = six.moves.input("verify:\t\t") + if check_password_hash(password_hash, password): + print(password_hash) + else: + print("Failure!") + + +if __name__ == '__main__': + main() diff --git a/medallion/scripts/run.py b/medallion/scripts/run.py index c848aab6..f33ad147 100644 --- a/medallion/scripts/run.py +++ b/medallion/scripts/run.py @@ -1,10 +1,10 @@ +#!/usr/bin/env python + import argparse -import json import logging import textwrap -from medallion import (__version__, application_instance, register_blueprints, - set_config) +from medallion import __version__, create_app log = logging.getLogger("medallion") @@ -43,7 +43,7 @@ def _get_argparser(): parser.add_argument( "--debug-mode", - default=False, + default=None, action="store_true", help="If set, start application in debug mode.", ) @@ -71,19 +71,10 @@ def main(): medallion_args = medallion_parser.parse_args() log.setLevel(medallion_args.log_level) - with open(medallion_args.CONFIG_PATH, "r") as f: - configuration = json.load(f) - - set_config(application_instance, "users", configuration) - set_config(application_instance, "taxii", configuration) - set_config(application_instance, "backend", configuration) - register_blueprints(application_instance) - - application_instance.run( - host=medallion_args.host, - port=medallion_args.port, - debug=medallion_args.debug_mode, - ) + app = create_app(medallion_args.CONFIG_PATH) + app.run(host=medallion_args.host, + port=medallion_args.port, + debug=medallion_args.debug_mode) if __name__ == "__main__": diff --git a/medallion/test/__init__.py b/medallion/test/__init__.py index 2f26a474..615879f2 100644 --- a/medallion/test/__init__.py +++ b/medallion/test/__init__.py @@ -12,3 +12,12 @@ MANIFESTS_EP = GET_ADD_COLLECTION_EP + "manifest/" OBJECTS_EP = GET_ADD_COLLECTION_EP + "objects/" GET_OBJECT_EP = GET_OBJECTS_EP = ADD_OBJECTS_EP = OBJECTS_EP + +GET_COLLECTION_EP_FOR_DIRECTORY_BACKEND = COLLECTIONS_EP + "46bb17fa-0af3-3446-a570-b55cdfdc7881/" +GET_OBJECTS_FROM_DIRECTORY_BACKEND_EP_NOT_EXISTANT = COLLECTIONS_EP + "indicator--d772d620-9720-4457-897d-7030d1a972fd/" +GET_OBJECTS_FROM_DIRECTORY_BACKEND_EP = GET_COLLECTION_EP_FOR_DIRECTORY_BACKEND + "objects/" +GET_OBJECTS_FROM_DIRECTORY_BACKEND_EP_NOT_EXISTANT_OBJECTS = \ + COLLECTIONS_EP + "64bb17fa-0af3-3446-a570-b55cdfdc7818/" + "objects/" +ADD_OBJECTS_FOR_DIRECTORY_BACKEND_EP = GET_OBJECTS_FROM_DIRECTORY_BACKEND_EP + +LOGIN = "/login" diff --git a/medallion/test/base_test.py b/medallion/test/base_test.py index 6d658948..f1def127 100644 --- a/medallion/test/base_test.py +++ b/medallion/test/base_test.py @@ -1,16 +1,15 @@ -import base64 import os import unittest -from medallion import application_instance, register_blueprints, set_config +from medallion import create_app +from medallion.test import config as test_configs from medallion.test.data.initialize_mongodb import reset_db class TaxiiTest(unittest.TestCase): type = None DATA_FILE = os.path.join( - os.path.dirname(__file__), "data", "default_data.json", - ) + os.path.dirname(__file__), "data", "default_data.json") API_OBJECTS_2 = { "id": "bundle--8fab937e-b694-11e3-b71c-0800271e87d2", "objects": [ @@ -31,78 +30,25 @@ class TaxiiTest(unittest.TestCase): "type": "bundle", } - no_config = {} - - config_no_taxii = { - "backend": { - "module": "medallion.backends.memory_backend", - "module_class": "MemoryBackend", - "filename": DATA_FILE, - }, - "users": { - "admin": "Password0", - }, - } - - config_no_auth = { - "backend": { - "module": "medallion.backends.memory_backend", - "module_class": "MemoryBackend", - "filename": DATA_FILE, - }, - "taxii": { - "max_page_size": 20, - }, - } - - config_no_backend = { - "users": { - "admin": "Password0", - }, - "taxii": { - "max_page_size": 20, - }, - } + memory_config = test_configs.memory_config(DATA_FILE) + directory_config = test_configs.directory_config() + mongodb_config = test_configs.mongodb_config() - memory_config = { - "backend": { - "module": "medallion.backends.memory_backend", - "module_class": "MemoryBackend", - "filename": DATA_FILE, - }, - "users": { - "admin": "Password0", - }, - "taxii": { - "max_page_size": 20, - }, - } + no_config = {} - mongodb_config = { - "backend": { - "module": "medallion.backends.mongodb_backend", - "module_class": "MongoBackend", - "uri": "mongodb://travis:test@127.0.0.1:27017/", - }, - "users": { - "admin": "Password0", - }, - "taxii": { - "max_page_size": 20, - }, - } + config_no_taxii = {k: v for k, v in memory_config.items() if k != "taxii"} + config_no_auth = {k: v for k, v in memory_config.items() if k != "auth"} + config_no_backend = {k: v for k, v in memory_config.items() if k != "backend"} def setUp(self): - self.app = application_instance - self.app_context = application_instance.app_context() - self.app_context.push() - self.app.testing = True - register_blueprints(self.app) if self.type == "mongo": - reset_db(self.mongodb_config["backend"]["uri"]) + reset_db() self.configuration = self.mongodb_config elif self.type == "memory": + self.memory_config['backend']['filename'] = self.DATA_FILE self.configuration = self.memory_config + elif self.type == "directory": + self.configuration = self.directory_config elif self.type == "memory_no_config": self.configuration = self.no_config elif self.type == "no_taxii": @@ -113,17 +59,14 @@ def setUp(self): self.configuration = self.config_no_backend else: raise RuntimeError("Unknown backend!") - set_config(self.app, "backend", self.configuration) - set_config(self.app, "users", self.configuration) - set_config(self.app, "taxii", self.configuration) - self.client = application_instance.test_client() - if self.type == "memory_no_config" or self.type == "no_auth": - encoded_auth = "Basic " + \ - base64.b64encode(b"user:pass").decode("ascii") - else: - encoded_auth = "Basic " + \ - base64.b64encode(b"admin:Password0").decode("ascii") - self.auth = {"Authorization": encoded_auth} + + self.app = create_app(self.configuration) + self.app_context = self.app.app_context() + self.app_context.push() + + # TODO: It might be better to not reuse the test client. + self.client = self.app.test_client() + self.auth = {'Authorization': 'Token abc123'} def tearDown(self): self.app_context.pop() diff --git a/medallion/test/config.py b/medallion/test/config.py new file mode 100644 index 00000000..3e2c7db0 --- /dev/null +++ b/medallion/test/config.py @@ -0,0 +1,100 @@ +def _base_config(backend, auth): + return { + 'flask': { + 'SECRET_KEY': 'testsecret', + 'TESTING': True, + }, + 'multi-auth': ['api_key', 'jwt', 'basic'], + "taxii": { + "max_page_size": 20 + }, + 'backend': backend, + 'auth': auth, + } + + +def memory_config(data_file): + return _base_config({ + "module": "medallion.backends.taxii.memory_backend", + "module_class": "MemoryBackend", + "filename": data_file + }, { + "module": "medallion.backends.auth.memory_auth", + "module_class": "AuthMemoryBackend", + "users": { + "admin": "pbkdf2:sha256:150000$xaVt57AC$6edb6149e820fed48495f21bcf98bcc8663cd413bbd97b91d72c671f8f445bea", + "user1": "pbkdf2:sha256:150000$TVpGAgEI$dd391524abb0d9107ff5949ef512c150523c388cfa6490d8556d604f90de329e", + "user2": "pbkdf2:sha256:150000$CUo7l9Vz$3ff2da22dcb84c9ba64e2df4d1ee9f7061c1da4f8506618f53457f615178e3f3" + }, + "api_keys": { + "123456": "admin", + "abc123": "admin", + "abcdef": "user1" + } + }) + + +def directory_config(): + return _base_config({ + "module": "medallion.backends.taxii.directory_backend", + "module_class": "DirectoryBackend", + "path": "./medallion/test/directory/", + "discovery": { + "title": "Indicators from the directory backend", + "description": "Indicators from the directory backend", + "contact": "", + "host": "http://localhost:5000/" + }, + "api-root": { + "title": "", + "description": "", + "versions": [ + "taxii-2.0" + ], + "max-content-length": 9765625 + }, + "collection": { + "id": "", + "title": "", + "description": "", + "can_read": True, + "can_write": True, + "media_types": [ + "application/vnd.oasis.stix+json; version=2.0" + ] + } + }, { + "module": "medallion.backends.auth.memory_auth", + "module_class": "AuthMemoryBackend", + "users": { + "admin": "pbkdf2:sha256:150000$xaVt57AC$6edb6149e820fed48495f21bcf98bcc8663cd413bbd97b91d72c671f8f445bea", + "user1": "pbkdf2:sha256:150000$TVpGAgEI$dd391524abb0d9107ff5949ef512c150523c388cfa6490d8556d604f90de329e", + "user2": "pbkdf2:sha256:150000$CUo7l9Vz$3ff2da22dcb84c9ba64e2df4d1ee9f7061c1da4f8506618f53457f615178e3f3" + }, + "api_keys": { + "123456": "admin", + "abc123": "admin", + "abcdef": "user1" + } + }) + + +def mongodb_config(): + return _base_config({ + "module": "medallion.backends.taxii.mongodb_backend", + "module_class": "MongoBackend", + "uri": "mongodb://root:example@localhost:27017/" + }, { + "module": "medallion.backends.auth.memory_auth", + "module_class": "AuthMemoryBackend", + "users": { + "admin": "pbkdf2:sha256:150000$xaVt57AC$6edb6149e820fed48495f21bcf98bcc8663cd413bbd97b91d72c671f8f445bea", + "user1": "pbkdf2:sha256:150000$TVpGAgEI$dd391524abb0d9107ff5949ef512c150523c388cfa6490d8556d604f90de329e", + "user2": "pbkdf2:sha256:150000$CUo7l9Vz$3ff2da22dcb84c9ba64e2df4d1ee9f7061c1da4f8506618f53457f615178e3f3" + }, + "api_keys": { + "123456": "admin", + "abc123": "admin", + "abcdef": "user1" + } + }) diff --git a/medallion/test/data/config.json b/medallion/test/data/config.json deleted file mode 100644 index 2d861e0f..00000000 --- a/medallion/test/data/config.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "backend": { - "module": "medallion.backends.memory_backend", - "module_class": "MemoryBackend", - "filename": "../test/data/default_data.json" - }, - "users": { - "admin": "Password0", - "user1": "Password1", - "user2": "Password2" - }, - "taxii": { - "max_page_size": 100 - } -} diff --git a/medallion/test/data/memory_backend_no_data.json b/medallion/test/data/memory_backend_no_data.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/medallion/test/data/memory_backend_no_data.json @@ -0,0 +1 @@ +{} diff --git a/medallion/test/directory/trustgroup1/very-simple-playbook.json b/medallion/test/directory/trustgroup1/very-simple-playbook.json new file mode 100644 index 00000000..f160a81f --- /dev/null +++ b/medallion/test/directory/trustgroup1/very-simple-playbook.json @@ -0,0 +1,229 @@ +{ + "type": "bundle", + "id": "bundle--a5fa69e1-bcb4-43a6-b7d6-eda102017f77", + "spec_version": "2.0", + "objects": [ + { + "type": "report", + "id": "report--a5fa69e1-bcb4-43a6-b7d6-eda102017f77", + "created": "2019-06-26T13:42:34.268Z", + "modified": "2019-06-26T13:46:06.886Z", + "name": "Very Simple Playbook", + "description": "A very simple Playbook with one Campaign, two Attack Patterns, and one Indicator.", + "published": "2019-06-26T13:46:06.886Z", + "object_refs": [ + "intrusion-set--f7fa709b-a740-4bdc-b48c-3ea69ed39cdf", + "report--b3e138d1-6ef7-4bdc-924a-285f588330d7" + ], + "labels": [ + "intrusion-set" + ] + }, + { + "type": "intrusion-set", + "id": "intrusion-set--f7fa709b-a740-4bdc-b48c-3ea69ed39cdf", + "created": "2019-06-26T13:42:34.372Z", + "modified": "2019-06-26T13:46:06.887Z", + "name": "Very Simple Playbook" + }, + { + "type": "attack-pattern", + "id": "attack-pattern--20138b9d-1aac-4a26-8654-a36b6bbf2bba", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "created": "2019-01-16T17:39:59.787Z", + "modified": "2019-01-16T17:39:59.787Z", + "name": "T1192: Spearphishing Link", + "description": "Spearphishing with a link is a specific variant of spearphishing. It is different from other forms of spearphishing in that it employs the use of links to download malware contained in email, instead of attaching malicious files to the email itself, to avoid defenses that may inspect email attachments. \n\nAll forms of spearphishing are electronically delivered social engineering targeted at a specific individual, company, or industry. In this case, the malicious emails contain links. Generally, the links will be accompanied by social engineering text and require the user to actively click or copy and paste a URL into a browser, leveraging [User Execution](https://attack.mitre.org/techniques/T1204). The visited website may compromise the web browser using an exploit, or the user will be prompted to download applications, documents, zip files, or even executables depending on the pretext for the email in the first place. Adversaries may also include links that are intended to interact directly with an email reader, including embedded images intended to exploit the end system directly or verify the receipt of an email (i.e. web bugs/web beacons).", + "kill_chain_phases": [ + { + "kill_chain_name": "mitre-attack", + "phase_name": "initial-access" + }, + { + "kill_chain_name": "lockheed", + "phase_name": "delivery" + } + ], + "external_references": [ + { + "source_name": "mitre-attack", + "url": "https://attack.mitre.org/techniques/T1192", + "external_id": "T1192" + }, + { + "source_name": "capec", + "url": "https://capec.mitre.org/data/definitions/163.html", + "external_id": "CAPEC-163" + } + ], + "object_marking_refs": [ + "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" + ] + }, + { + "type": "attack-pattern", + "id": "attack-pattern--389735f1-f21c-4208-b8f0-f8031e7169b8", + "created_by_ref": "identity--c78cb6e5-0c4b-4611-8297-d1b8b55e40b5", + "created": "2019-01-16T17:39:39.364Z", + "modified": "2019-01-16T17:39:39.364Z", + "name": "T1176: Browser Extensions", + "description": "Browser extensions or plugins are small programs that can add functionality and customize aspects of internet browsers. They can be installed directly or through a browser's app store. Extensions generally have access and permissions to everything that the browser can access. (Citation: Wikipedia Browser Extension) (Citation: Chrome Extensions Definition)\n\nMalicious extensions can be installed into a browser through malicious app store downloads masquerading as legitimate extensions, through social engineering, or by an adversary that has already compromised a system. Security can be limited on browser app stores so may not be difficult for malicious extensions to defeat automated scanners and be uploaded. (Citation: Malicious Chrome Extension Numbers) Once the extension is installed, it can browse to websites in the background, (Citation: Chrome Extension Crypto Miner) (Citation: ICEBRG Chrome Extensions) steal all information that a user enters into a browser, to include credentials, (Citation: Banker Google Chrome Extension Steals Creds) (Citation: Catch All Chrome Extension) and be used as an installer for a RAT for persistence. There have been instances of botnets using a persistent backdoor through malicious Chrome extensions. (Citation: Stantinko Botnet) There have also been similar examples of extensions being used for command & control (Citation: Chrome Extension C2 Malware).", + "kill_chain_phases": [ + { + "kill_chain_name": "mitre-attack", + "phase_name": "persistence" + }, + { + "kill_chain_name": "lockheed", + "phase_name": "installation" + } + ], + "external_references": [ + { + "source_name": "mitre-attack", + "url": "https://attack.mitre.org/techniques/T1176", + "external_id": "T1176" + }, + { + "source_name": "Chrome Extensions Definition", + "url": "https://developer.chrome.com/extensions", + "description": "Chrome. (n.d.). What are Extensions?. Retrieved November 16, 2017." + }, + { + "source_name": "Malicious Chrome Extension Numbers", + "url": "https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/43824.pdf", + "description": "Jagpal, N., et al. (2015, August). Trends and Lessons from Three Years Fighting Malicious Extensions. Retrieved November 17, 2017." + }, + { + "source_name": "Chrome Extension Crypto Miner", + "url": "https://www.ghacks.net/2017/09/19/first-chrome-extension-with-javascript-crypto-miner-detected/", + "description": "Brinkmann, M. (2017, September 19). First Chrome extension with JavaScript Crypto Miner detected. Retrieved November 16, 2017." + }, + { + "source_name": "Banker Google Chrome Extension Steals Creds", + "url": "https://isc.sans.edu/forums/diary/BankerGoogleChromeExtensiontargetingBrazil/22722/", + "description": "Marinho, R. (n.d.). (Banker(GoogleChromeExtension)).targeting. Retrieved November 18, 2017." + }, + { + "source_name": "Catch All Chrome Extension", + "url": "https://isc.sans.edu/forums/diary/CatchAll+Google+Chrome+Malicious+Extension+Steals+All+Posted+Data/22976/https:/threatpost.com/malicious-chrome-extension-steals-data-posted-to-any-website/128680/)", + "description": "Marinho, R. (n.d.). \"Catch-All\" Google Chrome Malicious Extension Steals All Posted Data. Retrieved November 16, 2017." + }, + { + "source_name": "Chrome Extension C2 Malware", + "url": "https://kjaer.io/extension-malware/", + "description": "Kjaer, M. (2016, July 18). Malware in the browser: how you might get hacked by a Chrome extension. Retrieved November 22, 2017." + }, + { + "source_name": "Stantinko Botnet", + "url": "https://www.welivesecurity.com/2017/07/20/stantinko-massive-adware-campaign-operating-covertly-since-2012/", + "description": "Vachon, F., Faou, M. (2017, July 20). Stantinko: A massive adware campaign operating covertly since 2012. Retrieved November 16, 2017." + }, + { + "source_name": "Wikipedia Browser Extension", + "url": "https://en.wikipedia.org/wiki/Browser_extension", + "description": "Wikipedia. (2017, October 8). Browser Extension. Retrieved January 11, 2018." + }, + { + "source_name": "ICEBRG Chrome Extensions", + "url": "https://www.icebrg.io/blog/malicious-chrome-extensions-enable-criminals-to-impact-over-half-a-million-users-and-global-businesses", + "description": "De Tore, M., Warner, J. (2018, January 15). MALICIOUS CHROME EXTENSIONS ENABLE CRIMINALS TO IMPACT OVER HALF A MILLION USERS AND GLOBAL BUSINESSES. Retrieved January 17, 2018." + } + ], + "object_marking_refs": [ + "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168" + ] + }, + { + "type": "indicator", + "id": "indicator--7d72d620-9720-4457-897d-7030d1a972df", + "created": "2019-06-25T18:15:42.452Z", + "modified": "2019-06-25T18:24:15.157Z", + "name": "https://dailymemes.net", + "pattern": "[url:value = 'https://dailymemes.net']", + "valid_from": "2019-06-25T18:15:42.452Z", + "labels": [ + "malicious-activity" + ] + }, + { + "type": "relationship", + "id": "relationship--c141c773-338c-4575-a26a-c0dc3f156ad3", + "created": "2019-06-26T13:44:14.536Z", + "modified": "2019-06-26T13:46:06.887Z", + "relationship_type": "indicates", + "source_ref": "indicator--7d72d620-9720-4457-897d-7030d1a972df", + "target_ref": "campaign--b3e138d1-6ef7-4bdc-924a-285f588330d7" + }, + { + "type": "relationship", + "id": "relationship--5d399937-d736-413f-a6ec-f77cdf824032", + "created": "2019-06-25T18:15:42.578Z", + "modified": "2019-06-26T13:44:48.412Z", + "relationship_type": "uses", + "description": "Cat pictures with bonus malware", + "source_ref": "indicator--7d72d620-9720-4457-897d-7030d1a972df", + "target_ref": "attack-pattern--20138b9d-1aac-4a26-8654-a36b6bbf2bba" + }, + { + "type": "relationship", + "id": "relationship--29e639c6-4bcf-49c7-b637-3bce69b5e1f6", + "created": "2019-06-26T13:45:54.754Z", + "modified": "2019-06-26T13:46:06.887Z", + "relationship_type": "uses", + "source_ref": "campaign--b3e138d1-6ef7-4bdc-924a-285f588330d7", + "target_ref": "attack-pattern--389735f1-f21c-4208-b8f0-f8031e7169b8" + }, + { + "type": "relationship", + "id": "relationship--a5fa69e1-bcb4-43a6-b7d6-eda102017f77", + "created": "2019-06-26T13:42:34.268Z", + "modified": "2019-06-26T13:46:06.886Z", + "relationship_type": "attributed-to", + "source_ref": "report--a5fa69e1-bcb4-43a6-b7d6-eda102017f77", + "target_ref": "intrusion-set--f7fa709b-a740-4bdc-b48c-3ea69ed39cdf" + }, + { + "type": "relationship", + "id": "relationship--20138b9d-1aac-4a26-8654-a36b6bbf2bba", + "created": "2019-06-26T13:42:34.372Z", + "modified": "2019-06-26T13:46:06.887Z", + "relationship_type": "uses", + "source_ref": "campaign--b3e138d1-6ef7-4bdc-924a-285f588330d7", + "target_ref": "attack-pattern--20138b9d-1aac-4a26-8654-a36b6bbf2bba" + }, + { + "type": "report", + "id": "report--b3e138d1-6ef7-4bdc-924a-285f588330d7", + "created": "2019-06-26T13:42:34.372Z", + "modified": "2019-06-26T13:46:06.887Z", + "name": "Campaign 1 - Very Simple Playbook", + "published": "2019-06-26T13:46:06.887Z", + "object_refs": [ + "campaign--b3e138d1-6ef7-4bdc-924a-285f588330d7", + "intrusion-set--f7fa709b-a740-4bdc-b48c-3ea69ed39cdf", + "indicator--7d72d620-9720-4457-897d-7030d1a972df", + "attack-pattern--20138b9d-1aac-4a26-8654-a36b6bbf2bba", + "attack-pattern--389735f1-f21c-4208-b8f0-f8031e7169b8", + "relationship--c141c773-338c-4575-a26a-c0dc3f156ad3", + "relationship--5d399937-d736-413f-a6ec-f77cdf824032", + "relationship--29e639c6-4bcf-49c7-b637-3bce69b5e1f6", + "relationship--a5fa69e1-bcb4-43a6-b7d6-eda102017f77", + "relationship--20138b9d-1aac-4a26-8654-a36b6bbf2bba" + ], + "labels": [ + "campaign" + ] + }, + { + "type": "campaign", + "id": "campaign--b3e138d1-6ef7-4bdc-924a-285f588330d7", + "created": "2019-06-26T13:42:34.372Z", + "modified": "2019-06-26T13:46:06.887Z", + "name": "Campaign 1 - Very Simple Playbook", + "description": "", + "first_seen": "1970-01-01T00:00:00.000Z", + "last_seen": "1970-01-01T00:00:00.000Z" + } + ] +} \ No newline at end of file diff --git a/medallion/test/test_directory_backend.py b/medallion/test/test_directory_backend.py new file mode 100644 index 00000000..13b059a7 --- /dev/null +++ b/medallion/test/test_directory_backend.py @@ -0,0 +1,183 @@ +import copy +import json +import os +import uuid + +import six + +from medallion import test +from medallion.test.base_test import TaxiiTest +from medallion.views import MEDIA_TYPE_STIX_V20, MEDIA_TYPE_TAXII_V20 + + +class TestTAXIIServerWithDirectoryBackend(TaxiiTest): + """ + These tests assume that the './test/directory' contains one sub directory named 'trustgroup1' + """ + + type = "directory" + + @staticmethod + def load_json_response(response): + if isinstance(response, bytes): + response = response.decode() + io = six.StringIO(response) + return json.load(io) + + def cleanup(self): + p = os.path.join(self.configuration['backend']['path'], "trustgroup1") + + rm_files = [f for f in os.listdir(p) + if os.path.isfile(os.path.join(p, f)) and f.endswith(".json") and f != "very-simple-playbook.json"] + + for f in rm_files: + fp = os.path.join(p, f) + + os.remove(fp) + + def test_server_discovery(self): + r = self.client.get(test.DISCOVERY_EP, headers=self.auth) + + self.assertEqual(r.status_code, 200) + self.assertEqual(r.content_type, MEDIA_TYPE_TAXII_V20) + server_info = self.load_json_response(r.data) + assert server_info["title"] == "Indicators from the directory backend" + assert len(server_info["api_roots"]) == 1 + assert server_info["api_roots"][0] == "http://localhost:5000/trustgroup1/" + + def test_get_api_root_information(self): + r = self.client.get(test.API_ROOT_EP, headers=self.auth) + + self.assertEqual(r.status_code, 200) + self.assertEqual(r.content_type, MEDIA_TYPE_TAXII_V20) + api_root_metadata = self.load_json_response(r.data) + assert api_root_metadata["title"] == "Indicators from directory \'trustgroup1\'" + + def test_get_api_root_information_not_existent(self): + r = self.client.get("/trustgroup2/", headers=self.auth) + + self.assertEqual(r.status_code, 404) + + def test_get_collections(self): + r = self.client.get(test.COLLECTIONS_EP, headers=self.auth) + + self.assertEqual(r.status_code, 200) + self.assertEqual(r.content_type, MEDIA_TYPE_TAXII_V20) + collections_metadata = self.load_json_response(r.data) + collections_metadata = sorted(collections_metadata["collections"], key=lambda x: x["id"]) + collection_ids = [cm["id"] for cm in collections_metadata] + + assert len(collection_ids) == 1 + assert "46bb17fa-0af3-3446-a570-b55cdfdc7881" in collection_ids + + def test_get_collection(self): + r = self.client.get(test.GET_COLLECTION_EP_FOR_DIRECTORY_BACKEND, headers=self.auth) + + self.assertEqual(r.status_code, 200) + self.assertEqual(r.content_type, MEDIA_TYPE_TAXII_V20) + collections_metadata = self.load_json_response(r.data) + assert collections_metadata["media_types"][0] == "application/vnd.oasis.stix+json; version=2.0" + + def test_get_collection_not_existent(self): + r = self.client.get(test.NON_EXISTENT_COLLECTION_EP, headers=self.auth) + + self.assertEqual(r.status_code, 404) + + def test_get_objects(self): + r = self.client.get(test.GET_OBJECTS_FROM_DIRECTORY_BACKEND_EP + "?match[type]=indicator", headers=self.auth) + + self.assertEqual(r.status_code, 200) + self.assertEqual(r.content_type, MEDIA_TYPE_STIX_V20) + objs = self.load_json_response(r.data) + + self.assertEqual(len(objs["objects"]), 1) + expected_ids = set(["indicator--7d72d620-9720-4457-897d-7030d1a972df"]) + received_ids = set(obj["id"] for obj in objs["objects"]) + self.assertEqual(expected_ids, received_ids) + + def test_get_objects_not_existant(self): + r = self.client.get( + test.GET_OBJECTS_FROM_DIRECTORY_BACKEND_EP_NOT_EXISTANT + "?match[type]=indicator", headers=self.auth + ) + + self.assertEqual(r.status_code, 404) + + def test_get_object(self): + r = self.client.get( + test.GET_OBJECTS_FROM_DIRECTORY_BACKEND_EP + "indicator--7d72d620-9720-4457-897d-7030d1a972df/", + headers=self.auth, + ) + + self.assertEqual(r.status_code, 200) + self.assertEqual(r.content_type, MEDIA_TYPE_STIX_V20) + obj = self.load_json_response(r.data) + assert obj["objects"][0]["id"] == "indicator--7d72d620-9720-4457-897d-7030d1a972df" + + def test_get_object_not_existant(self): + r = self.client.get(test.GET_OBJECTS_FROM_DIRECTORY_BACKEND_EP_NOT_EXISTANT, headers=self.auth) + + self.assertEqual(r.status_code, 404) + + def test_add_objects(self): + # ------------- BEGIN: cleanup section ------------- # + self.cleanup() + # ------------- END: cleanup section ------------- # + + new_bundle = copy.deepcopy(self.API_OBJECTS_2) + new_id = "indicator--%s" % uuid.uuid4() + new_bundle["objects"][0]["id"] = new_id + + # ------------- BEGIN: add object section ------------- # + + post_header = copy.deepcopy(self.auth) + post_header["Content-Type"] = MEDIA_TYPE_STIX_V20 + post_header["Accept"] = MEDIA_TYPE_TAXII_V20 + + r_post = self.client.post( + test.ADD_OBJECTS_FOR_DIRECTORY_BACKEND_EP, + data=json.dumps(new_bundle), + headers=post_header, + ) + + self.load_json_response(r_post.data) + self.assertEqual(r_post.status_code, 202) + self.assertEqual(r_post.content_type, MEDIA_TYPE_TAXII_V20) + + # ------------- END: add object section ------------- # + # ------------- BEGIN: get object section ------------- # + + get_header = copy.deepcopy(self.auth) + get_header["Accept"] = MEDIA_TYPE_STIX_V20 + + r_get = self.client.get( + test.GET_OBJECTS_FROM_DIRECTORY_BACKEND_EP + "?match[id]=%s" % new_id, + headers=get_header, + ) + self.assertEqual(r_get.status_code, 200) + self.assertEqual(r_get.content_type, MEDIA_TYPE_STIX_V20) + + objs = self.load_json_response(r_get.data) + assert objs["objects"][0]["id"] == new_id + + # ------------- END: get object section ------------- # + # ------------- BEGIN: get objects section ------------- # + + # This test assumes that the only file existing is 'very-simple-playbook.json' + + get_header = copy.deepcopy(self.auth) + get_header["Accept"] = MEDIA_TYPE_STIX_V20 + + r_get = self.client.get( + test.GET_OBJECTS_FROM_DIRECTORY_BACKEND_EP + "?match[type]=indicator", headers=get_header + ) + + self.assertEqual(r_get.status_code, 200) + self.assertEqual(r_get.content_type, MEDIA_TYPE_STIX_V20) + + objs = self.load_json_response(r_get.data) + assert len(objs["objects"]) == 2 + + # ------------- END: get objects section ------------- # + # ------------- BEGIN: cleanup section ------------- # + self.cleanup() + # ------------- END: cleanup section ------------- # diff --git a/medallion/test/test_memory_backend.py b/medallion/test/test_memory_backend.py index 9e687afd..5a00cfef 100644 --- a/medallion/test/test_memory_backend.py +++ b/medallion/test/test_memory_backend.py @@ -7,12 +7,11 @@ from flask import current_app import six -from medallion import set_config, test +from medallion import set_backend_config, test, verify_basic_auth +from medallion.test.base_test import TaxiiTest from medallion.utils import common from medallion.views import MEDIA_TYPE_STIX_V20, MEDIA_TYPE_TAXII_V20 -from .base_test import TaxiiTest - class TestTAXIIWithNoTAXIISection(TaxiiTest): type = "no_taxii" @@ -25,7 +24,7 @@ class TestTAXIIWithNoAuthSection(TaxiiTest): type = "no_auth" def test_default_userpass_auth(self): - assert current_app.users_backend.get("user") == "pass" + assert verify_basic_auth("admin", "Password0") class TestTAXIIWithNoBackendSection(TaxiiTest): @@ -39,7 +38,7 @@ class TestTAXIIWithNoConfig(TaxiiTest): type = "memory_no_config" def test_default_userpass_config(self): - assert current_app.users_backend.get("user") == "pass" + assert verify_basic_auth("admin", "Password0") def test_server_discovery_backend(self): assert current_app.medallion_backend.data == {} @@ -406,7 +405,7 @@ def test_saving_data_file(self): # just for the memory backend configuration = copy.deepcopy(self.configuration) configuration["backend"]["filename"] = f.name - set_config(self.app, "backend", configuration) + set_backend_config(self.app, configuration["backend"]) r_get = self.client.get( test.GET_OBJECTS_EP + "?match[id]=%s" % new_id, diff --git a/medallion/test/test_mongodb_backend.py b/medallion/test/test_mongodb_backend.py index 20d98153..2fbc3618 100644 --- a/medallion/test/test_mongodb_backend.py +++ b/medallion/test/test_mongodb_backend.py @@ -5,12 +5,11 @@ import six from medallion import test +from medallion.test.base_test import TaxiiTest from medallion.test.generic_initialize_mongodb import connect_to_client from medallion.utils import common from medallion.views import MEDIA_TYPE_STIX_V20, MEDIA_TYPE_TAXII_V20 -from .base_test import TaxiiTest - class TestTAXIIServerWithMongoDBBackend(TaxiiTest): type = "mongo" diff --git a/medallion/test/test_views.py b/medallion/test/test_views.py index 3505db5f..909351f8 100644 --- a/medallion/test/test_views.py +++ b/medallion/test/test_views.py @@ -1,4 +1,4 @@ -import base64 +from base64 import b64encode import copy import json import sys @@ -7,8 +7,10 @@ import six -from medallion import (application_instance, register_blueprints, set_config, - test) +from medallion import create_app, test +from medallion.test import config +from medallion.test.base_test import TaxiiTest +from medallion.test.data.initialize_mongodb import reset_db from medallion.views import MEDIA_TYPE_STIX_V20, MEDIA_TYPE_TAXII_V20 if sys.version_info < (3, 3, 0): @@ -41,30 +43,17 @@ class TestTAXIIServerWithMockBackend(unittest.TestCase): def setUp(self): - self.app = application_instance - self.app_context = application_instance.app_context() + reset_db() + self.configuration = config.mongodb_config() + self.configuration['backend']['default_page_size'] = 20 + + self.app = create_app(self.configuration) + + self.app_context = self.app.app_context() self.app_context.push() - self.app.testing = True - register_blueprints(self.app) - self.configuration = { - "backend": { - "module": "medallion.backends.mongodb_backend", - "module_class": "MongoBackend", - "uri": "mongodb://localhost:27017/", - "default_page_size": 20, - }, - "users": { - "admin": "Password0", - }, - "taxii": { - "max_page_size": 20, - }, - } - self.client = application_instance.test_client() - set_config(self.app, "users", self.configuration) - set_config(self.app, "taxii", self.configuration) - encoded_auth = 'Basic ' + base64.b64encode(b"admin:Password0").decode("ascii") - self.auth = {'Authorization': encoded_auth} + + self.client = self.app.test_client() + self.auth = {'Authorization': 'Token abc123'} def tearDown(self): self.app_context.pop() @@ -76,7 +65,7 @@ def load_json_response(response): io = six.StringIO(response) return json.load(io) - @mock.patch('medallion.backends.base.Backend') + @mock.patch('medallion.backends.taxii.base.Backend') def test_responses_include_range_headers(self, mock_backend): """ This test confirms that the expected endpoints are returning the Accept-Ranges HTTP header as per section 3.4 of the specification """ @@ -117,7 +106,7 @@ def test_responses_include_range_headers(self, mock_backend): # ------------- END: test objects endpoint ------------- # - @mock.patch('medallion.backends.base.Backend') + @mock.patch('medallion.backends.taxii.base.Backend') def test_response_status_headers_for_large_responses(self, mock_backend): """ This test confirms that the expected endpoints are returning the Accept-Ranges and Content-Range headers as well as a HTTP 206 for large responses. Refer section 3.4.3 @@ -165,7 +154,7 @@ def test_response_status_headers_for_large_responses(self, mock_backend): # ------------- END: test large result set ------------- # - @mock.patch('medallion.backends.base.Backend') + @mock.patch('medallion.backends.taxii.base.Backend') def test_bad_range_request(self, mock_backend): """ This test should return a HTTP 416 for a range request that cannot be satisfied. Refer 3.4.2 in the TAXII specification. """ @@ -182,10 +171,10 @@ def test_bad_range_request(self, mock_backend): } r = self.client.get(test.GET_OBJECT_EP, headers=headers) - self.assertEqual(r.status_code, 416) + self.assertEqual(416, r.status_code) self.assertEqual(r.headers.get('Content-Range'), 'items */10') - @mock.patch('medallion.backends.base.Backend') + @mock.patch('medallion.backends.taxii.base.Backend') def test_invalid_range_request(self, mock_backend): """ This test should return a HTTP 400 with a message that the request contains a malformed range request header. """ @@ -202,9 +191,9 @@ def test_invalid_range_request(self, mock_backend): } r = self.client.get(test.GET_OBJECT_EP, headers=headers) - self.assertEqual(r.status_code, 400) + self.assertEqual(400, r.status_code) - @mock.patch('medallion.backends.base.Backend') + @mock.patch('medallion.backends.taxii.base.Backend') def test_content_range_header_empty_response(self, mock_backend): """ This test checks that the Content-Range header is correctly formed for queries that return an empty (zero record) response. """ @@ -219,5 +208,58 @@ def test_content_range_header_empty_response(self, mock_backend): } r = self.client.get(test.GET_OBJECT_EP, headers=headers) - self.assertEqual(r.status_code, 206) + self.assertEqual(206, r.status_code) self.assertEqual(r.headers.get('Content-Range'), 'items 0-0/0') + + +class TestAuth(TaxiiTest): + type = "memory" + + @classmethod + def setUpClass(cls): + cls.username, cls.password = "admin", "Password0" + + def test_auth_failure(self): + with self.app.test_client() as client: + response = client.get('/routes') + self.assertEqual(response.status_code, 401) + + def test_login(self): + with self.app.test_client() as client: + response = client.post(test.LOGIN, method='POST', + json={'username': self.username, + 'password': self.password}) + self.assertEqual(response.status_code, 200) + self.assertIn('access_token', response.json) + + response = client.get('/routes', + headers={'Authorization': 'JWT ' + response.json['access_token']}) + self.assertEqual(response.status_code, 200) + + def test_login_failure(self): + with self.app.test_client() as client: + response = client.post(test.LOGIN, + json={'username': self.username + 'x', + 'password': self.password + 'y'}) + self.assertEqual(response.status_code, 401) + + def test_api_key_auth_failure(self): + with self.app.test_client() as client: + response = client.get("/routes", + headers={ + 'Authorization': + 'Basic '.encode('utf-8') + b64encode("user:invalid".encode('utf-8')) + }) + self.assertEqual(response.headers.get('WWW-Authenticate'), + 'Basic realm="Authentication Required"') + + def test_basic_auth_failure(self): + with self.app.test_client() as client: + response = client.get("/routes", + headers={'Authorization': 'Token xxxxxxx'}) + self.assertEqual(response.headers.get('WWW-Authenticate'), + 'Token realm="Authentication Required"') + + +if __name__ == "__main__": + unittest.main() diff --git a/medallion/views/others/__init__.py b/medallion/views/others/__init__.py new file mode 100644 index 00000000..936719d4 --- /dev/null +++ b/medallion/views/others/__init__.py @@ -0,0 +1 @@ +"""Location for views that are not critical to demonstrate the TAXII Specification API Concepts """ diff --git a/medallion/views/others/auth.py b/medallion/views/others/auth.py new file mode 100644 index 00000000..afca4f27 --- /dev/null +++ b/medallion/views/others/auth.py @@ -0,0 +1,34 @@ +from flask import Blueprint, abort, current_app, jsonify, request +from werkzeug.security import check_password_hash + +from medallion import auth, jwt_encode + +auth_bp = Blueprint('auth', __name__) + + +@auth_bp.route('/login', methods=['POST']) +def login(): + auth_info = request.json + if not auth_info: + abort(400) + username, password = auth_info['username'], auth_info['password'] + password_hash = current_app.auth_backend.get_password_hash(username) + + if not password_hash or not check_password_hash(password_hash, password): + abort(401) + + return jsonify({'access_token': jwt_encode(username).decode('utf-8')}) + + +@auth_bp.route('/routes', methods=['GET']) +@auth.login_required +def routes(): + return jsonify([ + { + 'path': str(rule.rule), + 'arguments': list(rule.arguments), + 'defaults': rule.defaults, + 'methods': list(rule.methods) + } + for rule in current_app.url_map.iter_rules() + ]) diff --git a/medallion/views/others/healthcheck.py b/medallion/views/others/healthcheck.py new file mode 100644 index 00000000..a6dd68ec --- /dev/null +++ b/medallion/views/others/healthcheck.py @@ -0,0 +1,8 @@ +from flask import Blueprint, jsonify + +healthecheck_bp = Blueprint('healthcheck', __name__) + + +@healthecheck_bp.route('/ping', methods=['GET']) +def ping(): + return jsonify({"pong": True}) diff --git a/setup.py b/setup.py index 5cec437e..5e3e72eb 100644 --- a/setup.py +++ b/setup.py @@ -53,10 +53,13 @@ def get_long_description(): "Flask-HTTPAuth", "pytz", "six", + "pyjwt", ], entry_points={ "console_scripts": [ "medallion = medallion.scripts.run:main", + "medallion-generate-user-pass = medallion.scripts.generate_user_password:main", + "medallion-mongo-db-utils = medallion.scripts.auth_db_utils:main", ], }, extras_require={ diff --git a/tox.ini b/tox.ini index 8212e180..f37565bb 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,7 @@ deps = coverage responses pymongo + pyjwt commands = pytest --cov=medallion medallion/test/ --cov-report term-missing