Merge pull request #2869 from mitre/clenk/update-workflow

Update GitHub Workflow

.github/workflows/quality.yml

@@ -29,14 +29,16 @@ jobs:
             toxenv: py310,style,coverage-ci
           - python-version: 3.11
             toxenv: py311,style,coverage-ci
+          - python-version: 3.12
+            toxenv: py312,style,coverage-ci
 
     steps:
-      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
         with:
           submodules: recursive
           fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
       - name: Setup python
-        uses: actions/setup-python@3542bca2639a428e1796aaa6a2ffef0c0f575566
+        uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
@@ -50,7 +52,7 @@ jobs:
       - name: Override Coverage Source Path for Sonar
         run: sed -i "s/<source>\/home\/runner\/work\/caldera\/caldera/<source>\/github\/workspace/g" /home/runner/work/caldera/caldera/coverage.xml
       - name: SonarCloud Scan
-        uses: SonarSource/sonarcloud-github-action@5875562561d22a34be0c657405578705a169af6c
+        uses: SonarSource/sonarcloud-github-action@49e6cd3b187936a73b8280d59ffd9da69df63ec9
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

.pre-commit-config.yaml

@@ -1,11 +1,11 @@
 repos:
     - repo: https://github.com/PyCQA/flake8
-      rev: 5.0.4
+      rev: 7.0.0
       hooks:
           - id: flake8
             additional_dependencies: [flake8-bugbear]
     - repo: https://github.com/PyCQA/bandit
-      rev: 1.7.4
+      rev: 1.7.7
       hooks:
         - id: bandit
           entry: bandit -ll --exclude=tests/ --skip=B303

app/api/packs/advanced.py

@@ -14,6 +14,7 @@ def __init__(self, services):
         self.rest_svc = services.get('rest_svc')
 
     async def enable(self):
+        self.app_svc.application.router._frozen = False
         self.app_svc.application.router.add_route('GET', '/advanced/sources', self._section_sources)
         self.app_svc.application.router.add_route('GET', '/advanced/objectives', self._section_objectives)
         self.app_svc.application.router.add_route('GET', '/advanced/planners', self._section_planners)

app/api/packs/campaign.py

@@ -16,6 +16,7 @@ def __init__(self, services):
         self.rest_svc = services.get('rest_svc')
 
     async def enable(self):
+        self.app_svc.application.router._frozen = False
         self.app_svc.application.router.add_route('GET', '/campaign/agents', self._section_agent)
         self.app_svc.application.router.add_route('GET', '/campaign/abilities', self._section_abilities)
         self.app_svc.application.router.add_route('GET', '/campaign/adversaries', self._section_profiles)

app/api/v2/managers/schedule_api_manager.py

@@ -30,6 +30,6 @@ def _merge_dictionaries(self, dict1, dict2):
         for key in dict1.keys():
             if key not in dict2:
                 dict2[key] = dict1[key]
-            elif type(dict1[key]) == dict:
+            elif isinstance(dict1[key], dict):
                 self._merge_dictionaries(dict1[key], dict2[key])
         return dict2

app/contacts/contact_dns.py

@@ -18,12 +18,17 @@ def __init__(self, services):
         self.contact_svc = services.get('contact_svc')
         self.domain = self.get_config('app.contact.dns.domain')
         self.handler = Handler(self.domain, services, self.name)
+        self.transport = None
 
     async def start(self):
         loop = asyncio.get_event_loop()
         dns = self.get_config('app.contact.dns.socket')
         addr, port = dns.split(':')
-        await loop.create_datagram_endpoint(lambda: self.handler, local_addr=(addr, port))
+        self.transport, _ = await loop.create_datagram_endpoint(lambda: self.handler, local_addr=(addr, port))
+
+    async def stop(self):
+        if self.transport:
+            self.transport.close()
 
 
 class DnsPacket:

app/contacts/contact_udp.py

@@ -13,12 +13,17 @@ def __init__(self, services):
         self.log = self.create_logger('contact_udp')
         self.contact_svc = services.get('contact_svc')
         self.handler = Handler(services)
+        self.transport = None
 
     async def start(self):
         loop = asyncio.get_event_loop()
         udp = self.get_config('app.contact.udp')
         addr, port = udp.split(':')
-        loop.create_task(loop.create_datagram_endpoint(lambda: self.handler, local_addr=(addr, port)))
+        self.transport, _ = await loop.create_task(loop.create_datagram_endpoint(lambda: self.handler, local_addr=(addr, port)))
+
+    async def stop(self):
+        if self.transport:
+            self.transport.close()
 
 
 class Handler(asyncio.DatagramProtocol):

app/learning/p_ip.py

@@ -22,6 +22,6 @@ def _is_valid_ip(raw_ip):
             if raw_ip in ['0.0.0.0', '127.0.0.1']:  # nosec
                 return False
             ip_address(raw_ip)
-        except BaseException:
+        except ValueError:
             return False
         return True

app/objects/c_ability.py

@@ -12,6 +12,10 @@
 
 
 class AbilitySchema(ma.Schema):
+
+    class Meta:
+        unknown = ma.EXCLUDE
+
     ability_id = ma.fields.String()
     tactic = ma.fields.String(load_default=None)
     technique_name = ma.fields.String(load_default=None)

app/objects/c_adversary.py

@@ -11,6 +11,9 @@
 
 class AdversarySchema(ma.Schema):
 
+    class Meta:
+        unknown = ma.EXCLUDE
+
     adversary_id = ma.fields.String()
     name = ma.fields.String()
     description = ma.fields.String()

app/objects/c_objective.py

@@ -9,6 +9,9 @@
 
 class ObjectiveSchema(ma.Schema):
 
+    class Meta:
+        unknown = ma.EXCLUDE
+
     id = ma.fields.String()
     name = ma.fields.String()
     description = ma.fields.String()

app/objects/c_operation.py

@@ -31,10 +31,14 @@ class InvalidOperationStateError(Exception):
 
 
 class OperationOutputRequestSchema(ma.Schema):
-    enable_agent_output = ma.fields.Boolean(default=False)
+    enable_agent_output = ma.fields.Boolean(dump_default=False)
 
 
 class OperationSchema(ma.Schema):
+
+    class Meta:
+        unknown = ma.EXCLUDE
+
     id = ma.fields.String()
     name = ma.fields.String(required=True)
     host_group = ma.fields.List(ma.fields.Nested(AgentSchema()), attribute='agents', dump_only=True)

app/objects/c_schedule.py

@@ -9,6 +9,9 @@
 
 class ScheduleSchema(ma.Schema):
 
+    class Meta:
+        unknown = ma.EXCLUDE
+
     id = ma.fields.String()
     schedule = ma.fields.Time(required=True)
     task = ma.fields.Nested(OperationSchema())

app/objects/c_source.py

@@ -39,7 +39,7 @@ class SourceSchema(ma.Schema):
     def fix_adjustments(self, in_data, **_):
         x = []
         raw_adjustments = in_data.pop('adjustments', {})
-        if raw_adjustments and type(raw_adjustments) == dict:
+        if raw_adjustments and isinstance(raw_adjustments, dict):
             for ability_id, adjustments in raw_adjustments.items():
                 for trait, block in adjustments.items():
                     for change in block:

app/objects/secondclass/c_link.py

@@ -39,7 +39,7 @@ class Meta:
     relationships = ma.fields.List(ma.fields.Nested(RelationshipSchema()))
     used = ma.fields.List(ma.fields.Nested(FactSchema()))
     unique = ma.fields.String()
-    collect = ma.fields.DateTime(format=BaseObject.TIME_FORMAT, default='')
+    collect = ma.fields.DateTime(format=BaseObject.TIME_FORMAT, dump_default='')
     finish = ma.fields.String()
     ability = ma.fields.Nested(AbilitySchema())
     executor = ma.fields.Nested(ExecutorSchema())

app/utility/base_world.py

@@ -4,7 +4,7 @@
 import yaml
 import logging
 import subprocess
-import distutils.version
+import packaging.version
 from base64 import b64encode, b64decode
 from datetime import datetime, timezone
 from importlib import import_module
@@ -126,7 +126,7 @@ def check_program_version(command, version, **kwargs):
 
         def compare_versions(version_string, minimum_version):
             version = parse_version(version_string)
-            return distutils.version.StrictVersion(version) >= distutils.version.StrictVersion(str(minimum_version))
+            return packaging.version.parse(version) >= packaging.version.parse(str(minimum_version))
 
         def parse_version(version_string, pattern=r'([0-9]+(?:\.[0-9]+)+)'):
             groups = re.search(pattern, version_string)

requirements.txt

@@ -1,11 +1,11 @@
 aiohttp-jinja2==1.5.1
-aiohttp==3.8.6
+aiohttp==3.9.3
 aiohttp_session==2.12.0
 aiohttp-security==0.4.0
-aiohttp-apispec==2.2.3
-jinja2==3.1.2
+aiohttp-apispec==3.0.0b2
+jinja2==3.1.3
 pyyaml==6.0.1
-cryptography==41.0.4
+cryptography==42.0.2
 websockets==11.0.3
 Sphinx==7.1.2
 sphinx_rtd_theme==1.3.0
@@ -22,3 +22,4 @@ Markdown==3.4.4  # training
 dnspython==2.4.2
 asyncssh==2.14.1
 aioftp~=0.20.0
+packaging==23.2

tests/api/v2/handlers/test_schedules_api.py

@@ -27,7 +27,7 @@ def expected_updated_schedule_dump(test_schedule, updated_schedule_payload):
 
     def _merge_dictionaries(dict1, dict2):
         for key in dict1.keys():
-            if type(dict1[key]) == dict:
+            if isinstance(dict1[key], dict):
                 _merge_dictionaries(dict1[key], dict2[key])
             else:
                 dict2[key] = dict1[key]

tests/planners/test_atomic.py

@@ -35,6 +35,9 @@ class LinkStub():
     def __init__(self, ability_id):
         self.ability = AbilityStub(ability_id)
 
+    def __eq__(self, other):
+        return self.ability.ability_id == other.ability.ability_id
+
 
 @pytest.fixture
 def atomic_planner():
@@ -61,8 +64,8 @@ def test_atomic_with_links_in_order(self, event_loop, atomic_planner):
 
         assert atomic_planner.operation.apply.call_count == 1
         assert atomic_planner.operation.wait_for_links_completion.call_count == 1
-        assert atomic_planner.operation.apply.called_with(LinkStub('ability_b'))
-        assert atomic_planner.operation.wait_for_links_completion.called_with([LinkStub('ability_b')])
+        atomic_planner.operation.apply.assert_called_with(LinkStub('ability_b'))
+        atomic_planner.operation.wait_for_links_completion.assert_called_with([LinkStub('ability_b')])
 
     def test_atomic_with_links_out_of_order(self, event_loop, atomic_planner):
 
@@ -77,8 +80,8 @@ def test_atomic_with_links_out_of_order(self, event_loop, atomic_planner):
 
         assert atomic_planner.operation.apply.call_count == 1
         assert atomic_planner.operation.wait_for_links_completion.call_count == 1
-        assert atomic_planner.operation.apply.called_with(LinkStub('ability_b'))
-        assert atomic_planner.operation.wait_for_links_completion.called_with([LinkStub('ability_b')])
+        atomic_planner.operation.apply.assert_called_with(LinkStub('ability_b'))
+        atomic_planner.operation.wait_for_links_completion.assert_called_with([LinkStub('ability_b')])
 
     def test_atomic_no_links(self, event_loop, atomic_planner):
 

tox.ini

@@ -6,7 +6,7 @@
 [tox]
 skipsdist = True
 envlist =
-    py{38,39,310,311}
+    py{38,39,310,311,312}
     style
     coverage
     safety