From aa244762ae1b290fa27976bb672e987f00cfdcdc Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Tue, 26 Sep 2023 10:08:13 -0400 Subject: [PATCH 1/5] Update Python dependency versions --- requirements.txt | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9ea18290e..66d041e06 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,26 +1,25 @@ -aiohttp-jinja2==1.5.0 -aiohttp==3.8.4 -aiohttp_session==2.9.0 +aiohttp-jinja2==1.5.1 +aiohttp==3.8.5 +aiohttp_session==2.12.0 aiohttp-security==0.4.0 aiohttp-apispec==2.2.3 -jinja2==3.0.3 -pyyaml>=5.1 -cryptography>=3.2 -websockets>=10.3 -Sphinx==5.1.1 -docutils==0.16 # Broken bullet lists in sphinx_rtd_theme https://github.com/readthedocs/sphinx_rtd_theme/issues/1115 -sphinx_rtd_theme==0.4.3 -myst-parser==0.18.0 -marshmallow==3.5.1 -dirhash==0.2.0 -docker==4.2.0 -donut-shellcode==0.9.2 +jinja2==3.1.2 +pyyaml==6.0.1 +cryptography==41.0.3 +websockets==11.0.3 +Sphinx==7.2.5 +sphinx_rtd_theme==1.3.0 +myst-parser==2.0.0 +marshmallow==3.20.1 +dirhash==0.2.1 +docker==6.1.3 # builder +donut-shellcode==1.0.2 marshmallow-enum==1.5.1 -ldap3==2.8.1 +ldap3==2.9.1 lxml~=4.9.1 # debrief -reportlab==3.6.12 # debrief -svglib==1.0.1 # debrief -Markdown==3.3.3 # training -dnspython==2.1.0 -asyncssh==2.11.0 +reportlab==4.0.4 # debrief +svglib==1.5.1 # debrief +Markdown==3.4.4 # training +dnspython==2.4.2 +asyncssh==2.13.2 aioftp~=0.20.0 From 2073ab66751f8c59d3e9c1345545811a0f7f9ff9 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Tue, 26 Sep 2023 10:08:53 -0400 Subject: [PATCH 2/5] Fix websocket v11 bug preventing reads --- app/service/event_svc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/service/event_svc.py b/app/service/event_svc.py index 6d6935284..3cb3853fa 100644 --- a/app/service/event_svc.py +++ b/app/service/event_svc.py @@ -82,6 +82,7 @@ async def fire_event(self, exchange=None, queue=None, timestamp=True, **callback d = json.dumps(callback_kwargs) async with websockets.connect(uri) as websocket: asyncio.get_event_loop().create_task(self.handle_exceptions(websocket.send(d))) + await asyncio.sleep(0) # yield control to event loop class _Handle: From 37020e261a9521a2cb17fecbc06f61e8ed87788e Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Tue, 26 Sep 2023 11:00:49 -0400 Subject: [PATCH 3/5] Drop dependency of non-default plugin Users of the builder plugin will still need to install docker-py, as outlined in the plugin's ReadMe. --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 66d041e06..2b42ee9a5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,6 @@ sphinx_rtd_theme==1.3.0 myst-parser==2.0.0 marshmallow==3.20.1 dirhash==0.2.1 -docker==6.1.3 # builder donut-shellcode==1.0.2 marshmallow-enum==1.5.1 ldap3==2.9.1 From 12e7283dfa3230428f0486c7005b211726529f30 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 29 Sep 2023 12:53:43 -0400 Subject: [PATCH 4/5] Fix failing tests Sphinx 7.2 doesn't support Python 3.8, but we do. --- app/objects/c_source.py | 20 +++++++++----------- requirements.txt | 2 +- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/app/objects/c_source.py b/app/objects/c_source.py index f8dd33624..f0df44a0e 100644 --- a/app/objects/c_source.py +++ b/app/objects/c_source.py @@ -60,17 +60,15 @@ def _fix_loaded_object_origins(input_data): :param input_data: A 'source' dictionary :return: input_data with updated facts/relationships (patched in place) """ - if 'facts' in input_data: - for y in input_data['facts']: - y['origin_type'] = OriginType.IMPORTED.name - y['source'] = input_data['id'] - if 'relationships' in input_data: - for y in input_data['relationships']: - y['source']['origin_type'] = OriginType.IMPORTED.name - y['source']['source'] = input_data['id'] - if 'target' in y: - y['target']['origin_type'] = OriginType.IMPORTED.name - y['target']['source'] = input_data['id'] + for y in input_data.get('facts', []): + y['origin_type'] = OriginType.IMPORTED.name + y['source'] = input_data['id'] + for y in input_data.get('relationships', []): + y['source']['origin_type'] = OriginType.IMPORTED.name + y['source']['source'] = input_data['id'] + if y.get('target'): + y['target']['origin_type'] = OriginType.IMPORTED.name + y['target']['source'] = input_data['id'] class Source(FirstClassObjectInterface, BaseObject): diff --git a/requirements.txt b/requirements.txt index 2b42ee9a5..16a9577d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ jinja2==3.1.2 pyyaml==6.0.1 cryptography==41.0.3 websockets==11.0.3 -Sphinx==7.2.5 +Sphinx==7.1.2 sphinx_rtd_theme==1.3.0 myst-parser==2.0.0 marshmallow==3.20.1 From 3e0f877eca9b65dfdecf163a0f275f092ab68751 Mon Sep 17 00:00:00 2001 From: Chris Lenk Date: Fri, 29 Sep 2023 13:56:09 -0400 Subject: [PATCH 5/5] Rename missing->load_default for mashmallow 'missing' was deprecated. --- app/objects/c_ability.py | 26 +++++++++++------------ app/objects/c_adversary.py | 2 +- app/objects/c_operation.py | 2 +- app/objects/c_planner.py | 2 +- app/objects/c_source.py | 2 +- app/objects/secondclass/c_executor.py | 14 ++++++------ app/objects/secondclass/c_instruction.py | 2 +- app/objects/secondclass/c_link.py | 16 +++++++------- app/objects/secondclass/c_parserconfig.py | 4 ++-- app/objects/secondclass/c_result.py | 2 +- 10 files changed, 36 insertions(+), 36 deletions(-) diff --git a/app/objects/c_ability.py b/app/objects/c_ability.py index 1a90daff3..a26da0dad 100644 --- a/app/objects/c_ability.py +++ b/app/objects/c_ability.py @@ -13,21 +13,21 @@ class AbilitySchema(ma.Schema): ability_id = ma.fields.String() - tactic = ma.fields.String(missing=None) - technique_name = ma.fields.String(missing=None) - technique_id = ma.fields.String(missing=None) - name = ma.fields.String(missing=None) - description = ma.fields.String(missing=None) + tactic = ma.fields.String(load_default=None) + technique_name = ma.fields.String(load_default=None) + technique_id = ma.fields.String(load_default=None) + name = ma.fields.String(load_default=None) + description = ma.fields.String(load_default=None) executors = ma.fields.List(ma.fields.Nested(ExecutorSchema)) - requirements = ma.fields.List(ma.fields.Nested(RequirementSchema), missing=None) - privilege = ma.fields.String(missing=None) - repeatable = ma.fields.Bool(missing=None) - buckets = ma.fields.List(ma.fields.String(), missing=None) + requirements = ma.fields.List(ma.fields.Nested(RequirementSchema), load_default=None) + privilege = ma.fields.String(load_default=None) + repeatable = ma.fields.Bool(load_default=None) + buckets = ma.fields.List(ma.fields.String(), load_default=None) additional_info = ma.fields.Dict(keys=ma.fields.String(), values=ma.fields.String()) - access = ma.fields.Nested(AccessSchema, missing=None) - singleton = ma.fields.Bool(missing=None) - plugin = ma.fields.String(missing=None) - delete_payload = ma.fields.Bool(missing=None) + access = ma.fields.Nested(AccessSchema, load_default=None) + singleton = ma.fields.Bool(load_default=None) + plugin = ma.fields.String(load_default=None) + delete_payload = ma.fields.Bool(load_default=None) @ma.pre_load def fix_id(self, data, **_): diff --git a/app/objects/c_adversary.py b/app/objects/c_adversary.py index 32c5114b5..e2c4ccd6f 100644 --- a/app/objects/c_adversary.py +++ b/app/objects/c_adversary.py @@ -18,7 +18,7 @@ class AdversarySchema(ma.Schema): objective = ma.fields.String() tags = ma.fields.List(ma.fields.String(), allow_none=True) has_repeatable_abilities = ma.fields.Boolean(dump_only=True) - plugin = ma.fields.String(missing=None) + plugin = ma.fields.String(load_default=None) @ma.pre_load def fix_id(self, adversary, **_): diff --git a/app/objects/c_operation.py b/app/objects/c_operation.py index 647a3437e..99e981164 100644 --- a/app/objects/c_operation.py +++ b/app/objects/c_operation.py @@ -50,7 +50,7 @@ class OperationSchema(ma.Schema): visibility = ma.fields.Integer() objective = ma.fields.Nested(ObjectiveSchema()) use_learning_parsers = ma.fields.Boolean() - group = ma.fields.String(missing='') + group = ma.fields.String(load_default='') source = ma.fields.Nested(SourceSchema()) @ma.pre_load() diff --git a/app/objects/c_planner.py b/app/objects/c_planner.py index 15d6e062f..010c08752 100644 --- a/app/objects/c_planner.py +++ b/app/objects/c_planner.py @@ -16,7 +16,7 @@ class PlannerSchema(ma.Schema): stopping_conditions = ma.fields.List(ma.fields.Nested(FactSchema())) ignore_enforcement_modules = ma.fields.List(ma.fields.String()) allow_repeatable_abilities = ma.fields.Boolean() - plugin = ma.fields.String(missing=None) + plugin = ma.fields.String(load_default=None) @ma.post_load() def build_planner(self, data, **kwargs): diff --git a/app/objects/c_source.py b/app/objects/c_source.py index f0df44a0e..bcbcb451d 100644 --- a/app/objects/c_source.py +++ b/app/objects/c_source.py @@ -33,7 +33,7 @@ class SourceSchema(ma.Schema): rules = ma.fields.List(ma.fields.Nested(RuleSchema)) adjustments = ma.fields.List(ma.fields.Nested(AdjustmentSchema)) relationships = ma.fields.List(ma.fields.Nested(RelationshipSchema)) - plugin = ma.fields.String(missing=None) + plugin = ma.fields.String(load_default=None) @ma.pre_load def fix_adjustments(self, in_data, **_): diff --git a/app/objects/secondclass/c_executor.py b/app/objects/secondclass/c_executor.py index f31d3d289..870a921d2 100644 --- a/app/objects/secondclass/c_executor.py +++ b/app/objects/secondclass/c_executor.py @@ -7,15 +7,15 @@ class ExecutorSchema(ma.Schema): - name = ma.fields.String(missing=None) - platform = ma.fields.String(missing=None) - command = ma.fields.String(missing=None) - code = ma.fields.String(missing=None) - language = ma.fields.String(missing=None) - build_target = ma.fields.String(missing=None) + name = ma.fields.String(load_default=None) + platform = ma.fields.String(load_default=None) + command = ma.fields.String(load_default=None) + code = ma.fields.String(load_default=None) + language = ma.fields.String(load_default=None) + build_target = ma.fields.String(load_default=None) payloads = ma.fields.List(ma.fields.String()) uploads = ma.fields.List(ma.fields.String()) - timeout = ma.fields.Int(missing=60) + timeout = ma.fields.Int(load_default=60) parsers = ma.fields.List(ma.fields.Nested(ParserSchema())) cleanup = ma.fields.List(ma.fields.String()) variations = ma.fields.List(ma.fields.Nested(VariationSchema())) diff --git a/app/objects/secondclass/c_instruction.py b/app/objects/secondclass/c_instruction.py index 342d618a5..c2121b48d 100644 --- a/app/objects/secondclass/c_instruction.py +++ b/app/objects/secondclass/c_instruction.py @@ -12,7 +12,7 @@ class InstructionSchema(ma.Schema): payloads = ma.fields.List(ma.fields.String()) deadman = ma.fields.Boolean() uploads = ma.fields.List(ma.fields.Dict(keys=ma.fields.String(), values=ma.fields.String())) - delete_payload = ma.fields.Bool(missing=None) + delete_payload = ma.fields.Bool(load_default=None) @ma.post_load def build_instruction(self, data, **_): diff --git a/app/objects/secondclass/c_link.py b/app/objects/secondclass/c_link.py index 6b8218241..793dc34a0 100644 --- a/app/objects/secondclass/c_link.py +++ b/app/objects/secondclass/c_link.py @@ -25,15 +25,15 @@ class LinkSchema(ma.Schema): class Meta: unknown = ma.EXCLUDE - id = ma.fields.String(missing='') + id = ma.fields.String(load_default='') paw = ma.fields.String() command = ma.fields.String() plaintext_command = ma.fields.String() - status = ma.fields.Integer(missing=-3) - score = ma.fields.Integer(missing=0) - jitter = ma.fields.Integer(missing=0) + status = ma.fields.Integer(load_default=-3) + score = ma.fields.Integer(load_default=0) + jitter = ma.fields.Integer(load_default=0) decide = ma.fields.DateTime(format=BaseObject.TIME_FORMAT) - pin = ma.fields.Integer(missing=0) + pin = ma.fields.Integer(load_default=0) pid = ma.fields.String() facts = ma.fields.List(ma.fields.Nested(FactSchema())) relationships = ma.fields.List(ma.fields.Nested(RelationshipSchema())) @@ -43,12 +43,12 @@ class Meta: finish = ma.fields.String() ability = ma.fields.Nested(AbilitySchema()) executor = ma.fields.Nested(ExecutorSchema()) - cleanup = ma.fields.Integer(missing=0) + cleanup = ma.fields.Integer(load_default=0) visibility = ma.fields.Nested(VisibilitySchema()) - host = ma.fields.String(missing=None) + host = ma.fields.String(load_default=None) output = ma.fields.String() deadman = ma.fields.Boolean() - agent_reported_time = ma.fields.DateTime(format=BaseObject.TIME_FORMAT, missing=None) + agent_reported_time = ma.fields.DateTime(format=BaseObject.TIME_FORMAT, load_default=None) @ma.pre_load() def fix_ability(self, link, **_): diff --git a/app/objects/secondclass/c_parserconfig.py b/app/objects/secondclass/c_parserconfig.py index b67c5086e..030948b03 100644 --- a/app/objects/secondclass/c_parserconfig.py +++ b/app/objects/secondclass/c_parserconfig.py @@ -9,8 +9,8 @@ class Meta: unknown = ma.INCLUDE source = ma.fields.String() - edge = ma.fields.String(missing=None) - target = ma.fields.String(missing=None) + edge = ma.fields.String(load_default=None) + target = ma.fields.String(load_default=None) custom_parser_vals = ma.fields.Dict(keys=ma.fields.String(), values=ma.fields.String()) @ma.pre_load diff --git a/app/objects/secondclass/c_result.py b/app/objects/secondclass/c_result.py index b0790e42b..fa5b73bd3 100644 --- a/app/objects/secondclass/c_result.py +++ b/app/objects/secondclass/c_result.py @@ -10,7 +10,7 @@ class ResultSchema(ma.Schema): exit_code = ma.fields.String() pid = ma.fields.String() status = ma.fields.String() - agent_reported_time = ma.fields.DateTime(format=BaseObject.TIME_FORMAT, missing=None) + agent_reported_time = ma.fields.DateTime(format=BaseObject.TIME_FORMAT, load_default=None) @ma.post_load def build_result(self, data, **_):