Merge branch 'master' into bleepbop/VIRTS-4705/discarded-abilities-in…

…-steps

app/objects/c_ability.py

@@ -13,21 +13,21 @@
 
 class AbilitySchema(ma.Schema):
     ability_id = ma.fields.String()
-    tactic = ma.fields.String(missing=None)
-    technique_name = ma.fields.String(missing=None)
-    technique_id = ma.fields.String(missing=None)
-    name = ma.fields.String(missing=None)
-    description = ma.fields.String(missing=None)
+    tactic = ma.fields.String(load_default=None)
+    technique_name = ma.fields.String(load_default=None)
+    technique_id = ma.fields.String(load_default=None)
+    name = ma.fields.String(load_default=None)
+    description = ma.fields.String(load_default=None)
     executors = ma.fields.List(ma.fields.Nested(ExecutorSchema))
-    requirements = ma.fields.List(ma.fields.Nested(RequirementSchema), missing=None)
-    privilege = ma.fields.String(missing=None)
-    repeatable = ma.fields.Bool(missing=None)
-    buckets = ma.fields.List(ma.fields.String(), missing=None)
+    requirements = ma.fields.List(ma.fields.Nested(RequirementSchema), load_default=None)
+    privilege = ma.fields.String(load_default=None)
+    repeatable = ma.fields.Bool(load_default=None)
+    buckets = ma.fields.List(ma.fields.String(), load_default=None)
     additional_info = ma.fields.Dict(keys=ma.fields.String(), values=ma.fields.String())
-    access = ma.fields.Nested(AccessSchema, missing=None)
-    singleton = ma.fields.Bool(missing=None)
-    plugin = ma.fields.String(missing=None)
-    delete_payload = ma.fields.Bool(missing=None)
+    access = ma.fields.Nested(AccessSchema, load_default=None)
+    singleton = ma.fields.Bool(load_default=None)
+    plugin = ma.fields.String(load_default=None)
+    delete_payload = ma.fields.Bool(load_default=None)
 
     @ma.pre_load
     def fix_id(self, data, **_):

app/objects/c_adversary.py

@@ -18,7 +18,7 @@ class AdversarySchema(ma.Schema):
     objective = ma.fields.String()
     tags = ma.fields.List(ma.fields.String(), allow_none=True)
     has_repeatable_abilities = ma.fields.Boolean(dump_only=True)
-    plugin = ma.fields.String(missing=None)
+    plugin = ma.fields.String(load_default=None)
 
     @ma.pre_load
     def fix_id(self, adversary, **_):

app/objects/c_operation.py

@@ -50,7 +50,7 @@ class OperationSchema(ma.Schema):
     visibility = ma.fields.Integer()
     objective = ma.fields.Nested(ObjectiveSchema())
     use_learning_parsers = ma.fields.Boolean()
-    group = ma.fields.String(missing='')
+    group = ma.fields.String(load_default='')
     source = ma.fields.Nested(SourceSchema())
 
     @ma.pre_load()

app/objects/c_planner.py

@@ -16,7 +16,7 @@ class PlannerSchema(ma.Schema):
     stopping_conditions = ma.fields.List(ma.fields.Nested(FactSchema()))
     ignore_enforcement_modules = ma.fields.List(ma.fields.String())
     allow_repeatable_abilities = ma.fields.Boolean()
-    plugin = ma.fields.String(missing=None)
+    plugin = ma.fields.String(load_default=None)
 
     @ma.post_load()
     def build_planner(self, data, **kwargs):

app/objects/c_source.py

@@ -33,7 +33,7 @@ class SourceSchema(ma.Schema):
     rules = ma.fields.List(ma.fields.Nested(RuleSchema))
     adjustments = ma.fields.List(ma.fields.Nested(AdjustmentSchema))
     relationships = ma.fields.List(ma.fields.Nested(RelationshipSchema))
-    plugin = ma.fields.String(missing=None)
+    plugin = ma.fields.String(load_default=None)
 
     @ma.pre_load
     def fix_adjustments(self, in_data, **_):
@@ -60,17 +60,15 @@ def _fix_loaded_object_origins(input_data):
         :param input_data: A 'source' dictionary
         :return: input_data with updated facts/relationships (patched in place)
         """
-        if 'facts' in input_data:
-            for y in input_data['facts']:
-                y['origin_type'] = OriginType.IMPORTED.name
-                y['source'] = input_data['id']
-        if 'relationships' in input_data:
-            for y in input_data['relationships']:
-                y['source']['origin_type'] = OriginType.IMPORTED.name
-                y['source']['source'] = input_data['id']
-                if 'target' in y:
-                    y['target']['origin_type'] = OriginType.IMPORTED.name
-                    y['target']['source'] = input_data['id']
+        for y in input_data.get('facts', []):
+            y['origin_type'] = OriginType.IMPORTED.name
+            y['source'] = input_data['id']
+        for y in input_data.get('relationships', []):
+            y['source']['origin_type'] = OriginType.IMPORTED.name
+            y['source']['source'] = input_data['id']
+            if y.get('target'):
+                y['target']['origin_type'] = OriginType.IMPORTED.name
+                y['target']['source'] = input_data['id']
 
 
 class Source(FirstClassObjectInterface, BaseObject):

app/objects/secondclass/c_executor.py

@@ -7,15 +7,15 @@
 
 
 class ExecutorSchema(ma.Schema):
-    name = ma.fields.String(missing=None)
-    platform = ma.fields.String(missing=None)
-    command = ma.fields.String(missing=None)
-    code = ma.fields.String(missing=None)
-    language = ma.fields.String(missing=None)
-    build_target = ma.fields.String(missing=None)
+    name = ma.fields.String(load_default=None)
+    platform = ma.fields.String(load_default=None)
+    command = ma.fields.String(load_default=None)
+    code = ma.fields.String(load_default=None)
+    language = ma.fields.String(load_default=None)
+    build_target = ma.fields.String(load_default=None)
     payloads = ma.fields.List(ma.fields.String())
     uploads = ma.fields.List(ma.fields.String())
-    timeout = ma.fields.Int(missing=60)
+    timeout = ma.fields.Int(load_default=60)
     parsers = ma.fields.List(ma.fields.Nested(ParserSchema()))
     cleanup = ma.fields.List(ma.fields.String())
     variations = ma.fields.List(ma.fields.Nested(VariationSchema()))

app/objects/secondclass/c_instruction.py

@@ -12,7 +12,7 @@ class InstructionSchema(ma.Schema):
     payloads = ma.fields.List(ma.fields.String())
     deadman = ma.fields.Boolean()
     uploads = ma.fields.List(ma.fields.Dict(keys=ma.fields.String(), values=ma.fields.String()))
-    delete_payload = ma.fields.Bool(missing=None)
+    delete_payload = ma.fields.Bool(load_default=None)
 
     @ma.post_load
     def build_instruction(self, data, **_):

app/objects/secondclass/c_link.py

@@ -25,15 +25,15 @@ class LinkSchema(ma.Schema):
     class Meta:
         unknown = ma.EXCLUDE
 
-    id = ma.fields.String(missing='')
+    id = ma.fields.String(load_default='')
     paw = ma.fields.String()
     command = ma.fields.String()
     plaintext_command = ma.fields.String()
-    status = ma.fields.Integer(missing=-3)
-    score = ma.fields.Integer(missing=0)
-    jitter = ma.fields.Integer(missing=0)
+    status = ma.fields.Integer(load_default=-3)
+    score = ma.fields.Integer(load_default=0)
+    jitter = ma.fields.Integer(load_default=0)
     decide = ma.fields.DateTime(format=BaseObject.TIME_FORMAT)
-    pin = ma.fields.Integer(missing=0)
+    pin = ma.fields.Integer(load_default=0)
     pid = ma.fields.String()
     facts = ma.fields.List(ma.fields.Nested(FactSchema()))
     relationships = ma.fields.List(ma.fields.Nested(RelationshipSchema()))
@@ -43,12 +43,12 @@ class Meta:
     finish = ma.fields.String()
     ability = ma.fields.Nested(AbilitySchema())
     executor = ma.fields.Nested(ExecutorSchema())
-    cleanup = ma.fields.Integer(missing=0)
+    cleanup = ma.fields.Integer(load_default=0)
     visibility = ma.fields.Nested(VisibilitySchema())
-    host = ma.fields.String(missing=None)
+    host = ma.fields.String(load_default=None)
     output = ma.fields.String()
     deadman = ma.fields.Boolean()
-    agent_reported_time = ma.fields.DateTime(format=BaseObject.TIME_FORMAT, missing=None)
+    agent_reported_time = ma.fields.DateTime(format=BaseObject.TIME_FORMAT, load_default=None)
 
     @ma.pre_load()
     def fix_ability(self, link, **_):

app/objects/secondclass/c_parserconfig.py

@@ -9,8 +9,8 @@ class Meta:
         unknown = ma.INCLUDE
 
     source = ma.fields.String()
-    edge = ma.fields.String(missing=None)
-    target = ma.fields.String(missing=None)
+    edge = ma.fields.String(load_default=None)
+    target = ma.fields.String(load_default=None)
     custom_parser_vals = ma.fields.Dict(keys=ma.fields.String(), values=ma.fields.String())
 
     @ma.pre_load

app/objects/secondclass/c_result.py

@@ -10,7 +10,7 @@ class ResultSchema(ma.Schema):
     exit_code = ma.fields.String()
     pid = ma.fields.String()
     status = ma.fields.String()
-    agent_reported_time = ma.fields.DateTime(format=BaseObject.TIME_FORMAT, missing=None)
+    agent_reported_time = ma.fields.DateTime(format=BaseObject.TIME_FORMAT, load_default=None)
 
     @ma.post_load
     def build_result(self, data, **_):

app/service/event_svc.py

@@ -82,6 +82,7 @@ async def fire_event(self, exchange=None, queue=None, timestamp=True, **callback
         d = json.dumps(callback_kwargs)
         async with websockets.connect(uri) as websocket:
             asyncio.get_event_loop().create_task(self.handle_exceptions(websocket.send(d)))
+            await asyncio.sleep(0)  # yield control to event loop
 
 
 class _Handle:

requirements.txt

@@ -1,26 +1,24 @@
-aiohttp-jinja2==1.5.0
-aiohttp==3.8.4
-aiohttp_session==2.9.0
+aiohttp-jinja2==1.5.1
+aiohttp==3.8.5
+aiohttp_session==2.12.0
 aiohttp-security==0.4.0
 aiohttp-apispec==2.2.3
-jinja2==3.0.3
-pyyaml>=5.1
-cryptography>=3.2
-websockets>=10.3
-Sphinx==5.1.1
-docutils==0.16 # Broken bullet lists in sphinx_rtd_theme https://github.com/readthedocs/sphinx_rtd_theme/issues/1115
-sphinx_rtd_theme==0.4.3
-myst-parser==0.18.0
-marshmallow==3.5.1
-dirhash==0.2.0
-docker==4.2.0
-donut-shellcode==0.9.2
+jinja2==3.1.2
+pyyaml==6.0.1
+cryptography==41.0.3
+websockets==11.0.3
+Sphinx==7.1.2
+sphinx_rtd_theme==1.3.0
+myst-parser==2.0.0
+marshmallow==3.20.1
+dirhash==0.2.1
+donut-shellcode==1.0.2
 marshmallow-enum==1.5.1
-ldap3==2.8.1
+ldap3==2.9.1
 lxml~=4.9.1  # debrief
 reportlab==4.0.4  # debrief
-svglib==1.0.1  # debrief
-Markdown==3.3.3  # training
-dnspython==2.1.0
-asyncssh==2.11.0
+svglib==1.5.1  # debrief
+Markdown==3.4.4  # training
+dnspython==2.4.2
+asyncssh==2.13.2
 aioftp~=0.20.0