From ad1dc70c6153ac2bd17c83720ec1e2fcaf4daba7 Mon Sep 17 00:00:00 2001 From: invisig0th Date: Wed, 14 Feb 2024 16:24:59 -0500 Subject: [PATCH 01/11] Add received:from:ipv4/ipv6/fqdn to inet:email:message (#3564) --- synapse/models/inet.py | 9 +++++++++ synapse/tests/test_model_inet.py | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/synapse/models/inet.py b/synapse/models/inet.py index 36fa1e1e08..731cd24fc1 100644 --- a/synapse/models/inet.py +++ b/synapse/models/inet.py @@ -1560,6 +1560,15 @@ def getModelDefs(self): ('headers', ('array', {'type': 'inet:email:header'}), { 'doc': 'An array of email headers from the message.'}), + ('received:from:ipv4', ('inet:ipv4', {}), { + 'doc': 'The sending SMTP server IPv4, potentially from the Received: header.'}), + + ('received:from:ipv6', ('inet:ipv6', {}), { + 'doc': 'The sending SMTP server IPv6, potentially from the Received: header.'}), + + ('received:from:fqdn', ('inet:fqdn', {}), { + 'doc': 'The sending server FQDN, potentially from the Received: header.'}), + )), ('inet:email:header', {}, ( diff --git a/synapse/tests/test_model_inet.py b/synapse/tests/test_model_inet.py index 1d774a0f62..446a1c7239 100644 --- a/synapse/tests/test_model_inet.py +++ b/synapse/tests/test_model_inet.py @@ -2662,6 +2662,9 @@ async def test_model_inet_email_message(self): :headers=(('to', 'Visi Stark '),) :cc=(baz@faz.org, foo@bar.com, baz@faz.org) :bytes="*" + :received:from:ipv4=1.2.3.4 + :received:from:ipv6="::1" + :received:from:fqdn=smtp.vertex.link ] {[( inet:email:message:link=($node, https://www.vertex.link) :text=Vertex )]} @@ -2671,6 +2674,9 @@ async def test_model_inet_email_message(self): self.len(1, nodes) self.eq(nodes[0].get('cc'), ('baz@faz.org', 'foo@bar.com')) + self.eq(nodes[0].get('received:from:ipv6'), '::1') + self.eq(nodes[0].get('received:from:ipv4'), 0x01020304) + self.eq(nodes[0].get('received:from:fqdn'), 'smtp.vertex.link') self.len(1, await core.nodes('inet:email:message:to=woot@woot.com')) self.len(1, await core.nodes('inet:email:message:date=2015')) From 2a8065666c792f1564ed394f757aa5ef4ee0e988 Mon Sep 17 00:00:00 2001 From: Cisphyx Date: Wed, 14 Feb 2024 16:34:04 -0500 Subject: [PATCH 02/11] Add perm checks with default=True to $lib.bytes (SYN-6859) (#3563) Co-authored-by: invisig0th --- synapse/lib/stormtypes.py | 13 +++++++++++++ synapse/tests/test_lib_stormtypes.py | 23 +++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/synapse/lib/stormtypes.py b/synapse/lib/stormtypes.py index 9404cd9428..671d581561 100644 --- a/synapse/lib/stormtypes.py +++ b/synapse/lib/stormtypes.py @@ -2467,6 +2467,9 @@ def getObjLocals(self): } async def _libBytesUpload(self, genr): + + self.runt.confirm(('axon', 'upload'), default=True) + await self.runt.snap.core.getAxon() async with await self.runt.snap.core.axon.upload() as upload: async for byts in s_coro.agen(genr): @@ -2480,6 +2483,8 @@ async def _libBytesHas(self, sha256): if sha256 is None: return None + self.runt.confirm(('axon', 'has'), default=True) + await self.runt.snap.core.getAxon() todo = s_common.todo('has', s_common.uhex(sha256)) ret = await self.dyncall('axon', todo) @@ -2488,6 +2493,9 @@ async def _libBytesHas(self, sha256): @stormfunc(readonly=True) async def _libBytesSize(self, sha256): sha256 = await tostr(sha256) + + self.runt.confirm(('axon', 'has'), default=True) + await self.runt.snap.core.getAxon() todo = s_common.todo('size', s_common.uhex(sha256)) ret = await self.dyncall('axon', todo) @@ -2498,6 +2506,8 @@ async def _libBytesPut(self, byts): mesg = '$lib.bytes.put() requires a bytes argument' raise s_exc.BadArg(mesg=mesg) + self.runt.confirm(('axon', 'upload'), default=True) + await self.runt.snap.core.getAxon() todo = s_common.todo('put', byts) size, sha2 = await self.dyncall('axon', todo) @@ -2507,6 +2517,9 @@ async def _libBytesPut(self, byts): @stormfunc(readonly=True) async def _libBytesHashset(self, sha256): sha256 = await tostr(sha256) + + self.runt.confirm(('axon', 'has'), default=True) + await self.runt.snap.core.getAxon() todo = s_common.todo('hashset', s_common.uhex(sha256)) ret = await self.dyncall('axon', todo) diff --git a/synapse/tests/test_lib_stormtypes.py b/synapse/tests/test_lib_stormtypes.py index 1aebbcf229..93ad945197 100644 --- a/synapse/tests/test_lib_stormtypes.py +++ b/synapse/tests/test_lib_stormtypes.py @@ -3096,6 +3096,29 @@ async def test_storm_lib_bytes(self): retn = await core.callStorm('return($lib.bytes.upload($chunks))', opts=opts) self.eq((8, '9ed8ffd0a11e337e6e461358195ebf8ea2e12a82db44561ae5d9e638f6f922c4'), retn) + visi = await core.auth.addUser('visi') + await visi.addRule((False, ('axon', 'has'))) + + opts = {'user': visi.iden, 'vars': {'hash': asdfhash_h}} + with self.raises(s_exc.AuthDeny): + await core.callStorm('return($lib.bytes.has($hash))', opts=opts) + + with self.raises(s_exc.AuthDeny): + await core.callStorm('return($lib.bytes.size($hash))', opts=opts) + + with self.raises(s_exc.AuthDeny): + await core.callStorm('return($lib.bytes.hashset($hash))', opts=opts) + + await visi.addRule((False, ('axon', 'upload'))) + + opts = {'user': visi.iden, 'vars': {'byts': b'foo'}} + with self.raises(s_exc.AuthDeny): + await core.callStorm('return($lib.bytes.put($byts))', opts=opts) + + opts = {'user': visi.iden, 'vars': {'chunks': (b'visi', b'kewl')}} + with self.raises(s_exc.AuthDeny): + await core.callStorm('return($lib.bytes.upload($chunks))', opts=opts) + async def test_storm_lib_base64(self): async with self.getTestCore() as core: From 74878410db86c3f302b12b5e09b17daf2e039672 Mon Sep 17 00:00:00 2001 From: mikemoritz <57907149+mikemoritz@users.noreply.github.com> Date: Thu, 15 Feb 2024 08:23:08 -0800 Subject: [PATCH 03/11] Add rstorm onload waiters (SYN-6871) (#3567) --- synapse/lib/rstorm.py | 16 +++++++++ synapse/tests/files/rstorm/testsvc.py | 9 ++++- synapse/tests/files/stormpkg/testpkg.yaml | 4 +++ synapse/tests/test_lib_rstorm.py | 40 +++++++++++++++++++++-- 4 files changed, 66 insertions(+), 3 deletions(-) diff --git a/synapse/lib/rstorm.py b/synapse/lib/rstorm.py index b852530239..c2a75dcdfb 100644 --- a/synapse/lib/rstorm.py +++ b/synapse/lib/rstorm.py @@ -29,6 +29,7 @@ logger = logging.getLogger(__name__) +ONLOAD_TIMEOUT = int(os.getenv('SYNDEV_PKG_LOAD_TIMEOUT', 30)) # seconds class OutPutRst(s_output.OutPutStr): ''' @@ -413,8 +414,17 @@ async def _handleStormPkg(self, text): core = self._reqCore() pkg = s_genpkg.loadPkgProto(text) + + if pkg.get('onload') is not None: + waiter = core.waiter(1, 'core:pkg:onload:complete') + else: + waiter = None + await core.addStormPkg(pkg) + if waiter is not None and not await waiter.wait(timeout=ONLOAD_TIMEOUT): + raise s_exc.SynErr(mesg=f'Package onload failed to run for {pkg.get("name")}') + async def _handleStormPre(self, text): ''' Run a Storm query to prepare the Cortex without output. @@ -453,6 +463,9 @@ async def _handleStormSvc(self, text): svc = await self._getCell(ctor, conf=svcconf) + onloadcnt = len([p for p in svc.cellapi._storm_svc_pkgs if p.get('onload') is not None]) + waiter = core.waiter(onloadcnt, 'core:pkg:onload:complete') if onloadcnt else None + svc.dmon.share('svc', svc) root = await svc.auth.getUserByName('root') await root.setPasswd('root') @@ -463,6 +476,9 @@ async def _handleStormSvc(self, text): await core.nodes(f'service.add {svcname} {surl}') await core.nodes(f'$lib.service.wait({svcname})') + if waiter is not None and not await waiter.wait(timeout=ONLOAD_TIMEOUT): + raise s_exc.SynErr(mesg=f'Package onload failed to run for service {svcname}') + async def _handleStormFail(self, text): valu = json.loads(text) assert valu in (True, False), f'storm-fail must be a boolean: {text}' diff --git a/synapse/tests/files/rstorm/testsvc.py b/synapse/tests/files/rstorm/testsvc.py index e22cce55a3..55f4822d94 100644 --- a/synapse/tests/files/rstorm/testsvc.py +++ b/synapse/tests/files/rstorm/testsvc.py @@ -8,10 +8,17 @@ class TestsvcApi(s_cell.CellApi, s_stormsvc.StormSvc): { 'name': 'testsvc', 'version': (0, 0, 1), + 'onload': ''' + $lib.time.sleep($lib.globals.get(onload_sleep, 0)) + $lib.globals.set(testsvc, testsvc-done) + ''', 'commands': ( { 'name': 'testsvc.test', - 'storm': '$lib.print($lib.service.get($cmdconf.svciden).test())', + 'storm': ''' + $lib.print($lib.service.get($cmdconf.svciden).test()) + $lib.print($lib.globals.get(testsvc)) + ''', }, ) }, diff --git a/synapse/tests/files/stormpkg/testpkg.yaml b/synapse/tests/files/stormpkg/testpkg.yaml index e38236d940..c33f222bfc 100644 --- a/synapse/tests/files/stormpkg/testpkg.yaml +++ b/synapse/tests/files/stormpkg/testpkg.yaml @@ -31,6 +31,10 @@ modules: type: dict desc: A status dictionary. +onload: | + $lib.time.sleep($lib.globals.get(onload_sleep, 0)) + $lib.globals.set(testpkg, testpkg-done) + external_modules: - name: testext package: synapse.tests.files diff --git a/synapse/tests/test_lib_rstorm.py b/synapse/tests/test_lib_rstorm.py index 4ff2298fd0..6dfe641592 100644 --- a/synapse/tests/test_lib_rstorm.py +++ b/synapse/tests/test_lib_rstorm.py @@ -21,7 +21,7 @@ .. storm-pre:: [ inet:ipv6=0 ] .. storm-pkg:: synapse/tests/files/stormpkg/testpkg.yaml .. storm:: --hide-props testpkgcmd foo -.. storm:: --hide-query $lib.print(secret) +.. storm:: --hide-query $lib.print(secret) $lib.print($lib.globals.get(testpkg)) .. storm:: --hide-query file:bytes .. storm-svc:: synapse.tests.files.rstorm.testsvc.Testsvc test {"secret": "jupiter"} .. storm:: testsvc.test @@ -44,6 +44,7 @@ :: secret + testpkg-done :: @@ -53,6 +54,7 @@ > testsvc.test jupiter + testsvc-done ''' @@ -259,6 +261,17 @@ .. storm-cortex:: path.to.NewpCell ''' +pkg_onload_timeout = ''' +.. storm-cortex:: default +.. storm-pre:: $lib.globals.set(onload_sleep, 2) +.. storm-pkg:: synapse/tests/files/stormpkg/testpkg.yaml +''' + +svc_onload_timeout = ''' +.. storm-cortex:: default +.. storm-pre:: $lib.globals.set(onload_sleep, 2) +.. storm-svc:: synapse.tests.files.rstorm.testsvc.Testsvc test {"secret": "jupiter"} +''' async def get_rst_text(rstfile): async with await s_rstorm.StormRst.anit(rstfile) as rstorm: @@ -339,7 +352,7 @@ async def test_lib_rstorm(self): self.isin('inet:ipv4=5.6.7.8', text) # one mock at a time self.isin('it:dev:str=notjson', text) # one mock at a time - # multi reqest in 1 rstorm command + # multi request in 1 rstorm command path = s_common.genpath(dirn, 'http_multi.rst') with s_common.genfile(path) as fd: fd.write(multi_rst_in_http.encode()) @@ -496,6 +509,29 @@ async def test_lib_rstorm(self): with self.raises(s_exc.NoSuchCtor): await get_rst_text(path) + # onload failures + + try: + oldv = s_rstorm.ONLOAD_TIMEOUT + s_rstorm.ONLOAD_TIMEOUT = 0.1 + + path = s_common.genpath(dirn, 'pkg_onload_timeout.rst') + with s_common.genfile(path) as fd: + fd.write(pkg_onload_timeout.encode()) + with self.raises(s_exc.SynErr) as ectx: + await get_rst_text(path) + self.eq('Package onload failed to run for testpkg', ectx.exception.errinfo['mesg']) + + path = s_common.genpath(dirn, 'svc_onload_timeout.rst') + with s_common.genfile(path) as fd: + fd.write(svc_onload_timeout.encode()) + with self.raises(s_exc.SynErr) as ectx: + await get_rst_text(path) + self.eq('Package onload failed to run for service test', ectx.exception.errinfo['mesg']) + + finally: + s_rstorm.ONLOAD_TIMEOUT = oldv + async def test_rstorm_cli(self): with self.getTestDir() as dirn: From 85697612a618d4ac15c202f453852fb70b7c4e36 Mon Sep 17 00:00:00 2001 From: invisig0th Date: Thu, 15 Feb 2024 11:54:06 -0500 Subject: [PATCH 04/11] Fix bug where users may vote for their own merge (#3565) --- synapse/lib/stormtypes.py | 2 ++ synapse/lib/view.py | 17 +++++++++++++++-- synapse/tests/test_lib_stormtypes.py | 9 +++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/synapse/lib/stormtypes.py b/synapse/lib/stormtypes.py index 671d581561..4d0576abc6 100644 --- a/synapse/lib/stormtypes.py +++ b/synapse/lib/stormtypes.py @@ -7718,6 +7718,8 @@ async def setMergeVote(self, approved=True, comment=None): mesg = 'You are not a member of a role with voting privileges for this merge request.' raise s_exc.AuthDeny(mesg=mesg) + view.reqValidVoter(self.runt.user.iden) + vote = {'user': self.runt.user.iden, 'approved': await tobool(approved)} if comment is not None: diff --git a/synapse/lib/view.py b/synapse/lib/view.py index 457d9b54ae..8fdf93cd4e 100644 --- a/synapse/lib/view.py +++ b/synapse/lib/view.py @@ -268,15 +268,28 @@ async def setMergeVote(self, vote): vote['offset'] = await self.layers[0].getEditIndx() return await self._push('merge:vote:set', vote) + def reqValidVoter(self, useriden): + + merge = self.getMergeRequest() + if merge is None: + raise s_exc.BadState(mesg=f'View ({self.iden}) does not have a merge request.') + + if merge.get('creator') == useriden: + raise s_exc.AuthDeny(mesg='A user may not vote for their own merge request.') + @s_nexus.Pusher.onPush('merge:vote:set') async def _setMergeVote(self, vote): self.reqParentQuorum() s_schemas.reqValidVote(vote) - uidn = s_common.uhex(vote.get('user')) + useriden = vote.get('user') + + self.reqValidVoter(useriden) + + bidn = s_common.uhex(useriden) - self.core.slab.put(self.bidn + b'merge:vote' + uidn, s_msgpack.en(vote), db='view:meta') + self.core.slab.put(self.bidn + b'merge:vote' + bidn, s_msgpack.en(vote), db='view:meta') await self.core.feedBeholder('view:merge:vote:set', {'view': self.iden, 'vote': vote}) diff --git a/synapse/tests/test_lib_stormtypes.py b/synapse/tests/test_lib_stormtypes.py index 93ad945197..221e0ed60a 100644 --- a/synapse/tests/test_lib_stormtypes.py +++ b/synapse/tests/test_lib_stormtypes.py @@ -6416,6 +6416,9 @@ async def test_view_quorum(self): with self.raises(s_exc.SynErr): await core.callStorm('$lib.view.get().merge()', opts={'view': fork00}) + with self.raises(s_exc.BadState): + core.getView(fork00).reqValidVoter(visi.iden) + with self.raises(s_exc.AuthDeny): await core.callStorm('$lib.view.get().setMergeRequest()', opts={'user': visi.iden, 'view': fork00}) @@ -6425,12 +6428,18 @@ async def test_view_quorum(self): self.eq(merge['comment'], 'woot') self.eq(merge['creator'], core.auth.rootuser.iden) + with self.raises(s_exc.AuthDeny): + core.getView(fork00).reqValidVoter(root.iden) + merge = await core.callStorm('return($lib.view.get().getMergeRequest())', opts={'view': fork00}) self.nn(merge['iden']) self.nn(merge['created']) self.eq(merge['comment'], 'woot') self.eq(merge['creator'], core.auth.rootuser.iden) + with self.raises(s_exc.AuthDeny): + await core.callStorm('$lib.view.get().setMergeVote()', opts={'view': fork00}) + with self.raises(s_exc.AuthDeny): await core.callStorm('$lib.view.get().setMergeVote()', opts={'user': visi.iden, 'view': fork00}) From 33ea54d07a9e8edac58d655a3016b5afe142d1a2 Mon Sep 17 00:00:00 2001 From: Cisphyx Date: Thu, 15 Feb 2024 11:59:04 -0500 Subject: [PATCH 05/11] Adjust tag permissions check during merges (SYN-6830) (#3545) --- synapse/lib/layer.py | 19 ++++++++++++++++++- synapse/lib/storm.py | 16 ++++++++++++++++ synapse/tests/test_lib_view.py | 22 ++++++++++++++++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/synapse/lib/layer.py b/synapse/lib/layer.py index cb44304cb6..927ade2725 100644 --- a/synapse/lib/layer.py +++ b/synapse/lib/layer.py @@ -4384,9 +4384,14 @@ def getNodeEditPerms(nodeedits): ''' Yields (offs, perm) tuples that can be used in user.allowed() ''' + tags = [] + tagadds = [] for nodeoffs, (buid, form, edits) in enumerate(nodeedits): + tags.clear() + tagadds.clear() + for editoffs, (edit, info, _) in enumerate(edits): permoffs = (nodeoffs, editoffs) @@ -4408,7 +4413,11 @@ def getNodeEditPerms(nodeedits): continue if edit == EDIT_TAG_SET: - yield (permoffs, ('node', 'tag', 'add', *info[0].split('.'))) + if info[1] != (None, None): + tagadds.append(info[0]) + yield (permoffs, ('node', 'tag', 'add', *info[0].split('.'))) + else: + tags.append((len(info[0]), editoffs, info[0])) continue if edit == EDIT_TAG_DEL: @@ -4438,3 +4447,11 @@ def getNodeEditPerms(nodeedits): if edit == EDIT_EDGE_DEL: yield (permoffs, ('node', 'edge', 'del', info[0])) continue + + for _, editoffs, tag in sorted(tags, reverse=True): + look = tag + '.' + if any([tagadd.startswith(look) for tagadd in tagadds]): + continue + + yield ((nodeoffs, editoffs), ('node', 'tag', 'add', *tag.split('.'))) + tagadds.append(tag) diff --git a/synapse/lib/storm.py b/synapse/lib/storm.py index 5c4705e7ca..a0a9e0ccba 100644 --- a/synapse/lib/storm.py +++ b/synapse/lib/storm.py @@ -3787,7 +3787,23 @@ async def _checkNodePerms(self, node, sode, runt): runt.confirmPropDel(prop, layriden=layr0) runt.confirmPropSet(prop, layriden=layr1) + tags = [] + tagadds = [] for tag, valu in sode.get('tags', {}).items(): + if valu != (None, None): + tagadds.append(tag) + tagperm = tuple(tag.split('.')) + runt.confirm(('node', 'tag', 'del') + tagperm, gateiden=layr0) + runt.confirm(('node', 'tag', 'add') + tagperm, gateiden=layr1) + else: + tags.append((len(tag), tag)) + + for _, tag in sorted(tags, reverse=True): + look = tag + '.' + if any([tagadd.startswith(look) for tagadd in tagadds]): + continue + + tagadds.append(tag) tagperm = tuple(tag.split('.')) runt.confirm(('node', 'tag', 'del') + tagperm, gateiden=layr0) runt.confirm(('node', 'tag', 'add') + tagperm, gateiden=layr1) diff --git a/synapse/tests/test_lib_view.py b/synapse/tests/test_lib_view.py index b1d5ae0f11..c0f14ecc21 100644 --- a/synapse/tests/test_lib_view.py +++ b/synapse/tests/test_lib_view.py @@ -759,3 +759,25 @@ async def test_lib_view_merge_perms(self): self.nn(nodes[0].tagprops.get('seen')) self.nn(nodes[0].tagprops['seen'].get('score')) self.nn(nodes[0].nodedata.get('foo')) + + await core.delUserRule(useriden, (True, ('node', 'tag', 'add')), gateiden=baselayr) + + await core.addUserRule(useriden, (True, ('node', 'tag', 'add', 'rep', 'foo')), gateiden=baselayr) + + await core.nodes('test:str=foo [ -#seen +#rep.foo ]', opts=viewopts) + + await core.nodes('$lib.view.get().merge()', opts=viewopts) + nodes = await core.nodes('test:str=foo') + self.nn(nodes[0].get('#rep.foo')) + + await core.nodes('test:str=foo [ -#rep ]') + + await core.nodes('test:str=foo | merge --apply', opts=viewopts) + nodes = await core.nodes('test:str=foo') + self.nn(nodes[0].get('#rep.foo')) + + await core.nodes('test:str=foo [ -#rep ]') + await core.nodes('test:str=foo [ +#rep=now ]', opts=viewopts) + + with self.raises(s_exc.AuthDeny) as cm: + await core.nodes('$lib.view.get().merge()', opts=viewopts) From 0d46f0987ad892243f918c6b070e11864853439f Mon Sep 17 00:00:00 2001 From: Cisphyx Date: Thu, 15 Feb 2024 12:10:49 -0500 Subject: [PATCH 06/11] Include vars from the current and parent scopes when resolving STIX properties and relationships (#3571) Co-authored-by: invisig0th --- synapse/lib/stormlib/stix.py | 5 ++++- synapse/tests/test_lib_stormlib_stix.py | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/synapse/lib/stormlib/stix.py b/synapse/lib/stormlib/stix.py index e89f6d2dd9..0a8f22e16d 100644 --- a/synapse/lib/stormlib/stix.py +++ b/synapse/lib/stormlib/stix.py @@ -1405,7 +1405,10 @@ def size(self): async def _callStorm(self, text, node): - opts = {'vars': {'bundle': self}} + varz = self.runt.getScopeVars() + varz['bundle'] = self + + opts = {'vars': varz} query = await self.runt.snap.core.getStormQuery(text) async with self.runt.getCmdRuntime(query, opts=opts) as runt: diff --git a/synapse/tests/test_lib_stormlib_stix.py b/synapse/tests/test_lib_stormlib_stix.py index c4a090c536..4e7f89c558 100644 --- a/synapse/tests/test_lib_stormlib_stix.py +++ b/synapse/tests/test_lib_stormlib_stix.py @@ -428,6 +428,7 @@ async def test_stix_export_custom(self): "stix": { "vtx-mitigation": { "props": { + "desc": "return($desc)", "name": "{+:name return(:name)} return($node.repr())", "created": "return($lib.stix.export.timestamp(.created))", "modified": "return($lib.stix.export.timestamp(.created))", @@ -440,13 +441,14 @@ async def test_stix_export_custom(self): } [ risk:mitigation=c4f6dc09f1e1e6b7e7b05c9ce4186ce8 :name="patch stuff and do things" ] - + $desc = "scopevar" $bundle.add($node) fini { return($bundle) } ''') self.eq('vtx-mitigation--2df2a437-e372-468b-b989-d01753603659', bund['objects'][1]['id']) + self.eq('scopevar', bund['objects'][1]['desc']) self.eq('patch stuff and do things', bund['objects'][1]['name']) self.nn(bund['objects'][1]['created']) self.nn(bund['objects'][1]['modified']) From d00a7e5e62f8eafbff8f3f3405b3496a422417d7 Mon Sep 17 00:00:00 2001 From: vEpiphyte Date: Thu, 15 Feb 2024 20:40:03 -0500 Subject: [PATCH 07/11] Changelog for v2.162.0 (#3572) Co-authored-by: invisig0th Co-authored-by: Cisphyx --- CHANGELOG.rst | 153 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 152 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a7ecade2bd..0521d8f69f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,11 +1,162 @@ .. vim: set textwidth=79 --.. _changelog: +.. _changelog: ***************** Synapse Changelog ***************** +v2.162.0 - 2024-02-15 +===================== + +Model Changes +------------- +- Updates to the ``inet``, ``infotech``, ``ou``, ``proj``, and ``risk`` models. + (`#3549 `_) + (`#3551 `_) + (`#3564 `_) + + **New Properties** + + ``inet:email:message`` + The form had the following properties added to it: + + ``received:from:ipv4`` + The sending SMTP server IPv4, potentially from the Received: header. + + ``received:from:ipv6`` + The sending SMTP server IPv6, potentially from the Received: header. + + ``received:from:fqdn`` + The sending server FQDN, potentially from the Received: header. + + ``ou:oid:type`` + The form had the following property added to it: + + ``url`` + The official URL of the issuer. + + ``proj:project`` + The form had the following property added to it: + + ``type`` + The project type. + + ``risk:alert`` + The form had the following properties added to it: + + ``status`` + The status of the alert. + + ``assignee`` + The Synapse user who is assigned to investigate the alert. + + ``ext:assignee`` + The alert assignee contact information from an external system. + + ``risk:mitigation`` + The form had the following properties added to it: + + ``reporter`` + The organization reporting on the mitigation. + + ``reporter:name`` + The name of the organization reporting on the mitigation. + + ``tag`` + The tag used to annotate nodes which have the mitigation in place. + + **New Forms** + + ``proj:project:type:taxonomy`` + A type taxonomy for projects. + + **Deprecated Properties** + + ``it:mitre:attack:group`` + The ``it:mitre:attack:group`` form had the following property marked as deprecated: + + * ``tag`` + + ``it:mitre:attack:tactic`` + The ``it:mitre:attack:tactic`` form had the following property marked as deprecated: + + * ``tag`` + + ``it:mitre:attack:technique`` + The ``it:mitre:attack:technique`` form had the following property marked as deprecated: + + * ``tag`` + + ``it:mitre:attack:software`` + The ``it:mitre:attack:software`` form had the following property marked as deprecated: + + * ``tag`` + + ``it:mitre:attack:campaign`` + The ``it:mitre:attack:campaign`` form had the following property marked as deprecated: + + * ``tag`` + +Features and Enhancements +------------------------- +- Add Storm API methods for inspecting and manipulating dictionary objects + in Storm. These are ``$lib.dict.has()``, ``$lib.dict.keys()``, + ``$lib.dict.pop()``, ``$lib.dict.update()``, and ``$lib.dict.values()` + (`#3548 `_) +- Add a ``json()`` method to the ``str`` type in Storm to deserialize a string + as JSON data. + (`#3555 `_) +- Add an ``_ahainfo`` attribute to the ``Telepath.Proxy``, containing AHA + service name information if that is provided to the Dmon. + (`#3552 `_) +- Add permissions checks to ``$lib.bytes`` APIs using ``axon.has`` for APIs + that check for information about the Axon or metrics; and ``axon.upload`` + for APIs which put bytes in the Axon. These are checked with + ``default=True`` for backward compatibility. + (`#3563 `_) +- The rstorm ``storm-svc`` and ``storm-pkg`` directives now wait for any + ``onload`` handlers to complete. + (`#3567 `_) +- Update the Synapse Python package trove classifiers to list the platforms + we support using Synapse with. + (`#3557 `_) + +Bugfixes +-------- +- Fix a bug in the ``Cell.updateHttpSessInfo()`` API when the Cell does not + have the session in memory. + (`#3556 `_) +- Fix a bug where a user was allowed to vote for their own View merge request. + (`#3565 `_) +- Include Storm variables from the current and parent scopes when resolving + STIX properties and relationships. + (`#3571 `_) + +Improved Documentation +---------------------- +- Update the Storm automation documentation. Added additional information + about permissions used to manage automations. Added examples for + ``edge:add`` and ``edge:del`` triggers. Added examples for managing Macro + permissions. + (`#3547 `_) +- Update the Storm filtering and lifting documentation to add information + about using interfaces and wildcard values with those operations. + (`#3560 `_) +- Update the Synapse introduction to note that Synapse is not intended to + replace big-data or data-lake solutions. + (`#3553 `_) + +Deprecations +------------ +- The Storm function ``$lib.dict()`` has been deprecated, in favor of using + the ``({"key": "value"})`` style syntax for directly declaring a dictionary + in Storm. + (`#3548 `_) +- Writeback layer mirrors and upstream layer mirrors have been marked as + deprecated configuration options. + (`#3562 `_) + v2.161.0 - 2024-02-06 ===================== From 4f4fa04685adceb99beb700b2d51b0f99afe801b Mon Sep 17 00:00:00 2001 From: epiphyte Date: Fri, 16 Feb 2024 02:20:41 +0000 Subject: [PATCH 08/11] =?UTF-8?q?Bump=20version:=202.161.0=20=E2=86=92=202?= =?UTF-8?q?.162.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- pyproject.toml | 2 +- synapse/lib/version.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 0f9f84a02b..58c3fadc71 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.161.0 +current_version = 2.162.0 commit = True tag = True tag_message = diff --git a/pyproject.toml b/pyproject.toml index d8ab529cde..2442f2adfd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'synapse' -version = '2.161.0' +version = '2.162.0' authors = [ { name = 'The Vertex Project LLC', email = 'root@vertex.link'}, ] diff --git a/synapse/lib/version.py b/synapse/lib/version.py index d7a7da7fb7..d5f0787e39 100644 --- a/synapse/lib/version.py +++ b/synapse/lib/version.py @@ -223,6 +223,6 @@ def reqVersion(valu, reqver, ############################################################################## # The following are touched during the release process by bumpversion. # Do not modify these directly. -version = (2, 161, 0) +version = (2, 162, 0) verstring = '.'.join([str(x) for x in version]) commit = '' From aba2a373795621875d0923f53db0ba375b3bb303 Mon Sep 17 00:00:00 2001 From: blackout Date: Fri, 16 Feb 2024 09:14:16 -0500 Subject: [PATCH 09/11] Update CI for rerunning failed tests only (#3569) Update circleci.yaml to use `circleci tests run` instead of `circleci tests split` so the new(-ish) "rerun failed tests" feature works. This should hopefully save us a bunch of time (and $$$) by not having to rerun the entire test suite when one or two flaky tests fail. --------- Co-authored-by: Cisphyx --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 509eeaee40..f3ad46c90b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -59,7 +59,7 @@ commands: command: | . venv/bin/activate mkdir test-reports - circleci tests glob synapse/tests/test_*.py synapse/vendor/*/tests/test_*.py | circleci tests split --split-by=timings | xargs python3 -m pytest -v -s -rs --durations 6 --maxfail 6 -p no:logging --junitxml=test-reports/junit.xml -o junit_family=xunit1 ${COVERAGE_ARGS} + circleci tests glob synapse/tests/test_*.py synapse/vendor/*/tests/test_*.py | circleci tests run --split-by=timings --command "xargs python3 -m pytest -v -s -rs --durations 6 --maxfail 6 -p no:logging --junitxml=test-reports/junit.xml -o junit_family=xunit1 ${COVERAGE_ARGS}" test_steps_doc: description: "Documentation test steps" From b63a63cb2be26dcfdb52e5f966364d3e841e355f Mon Sep 17 00:00:00 2001 From: Cisphyx Date: Tue, 20 Feb 2024 07:53:13 -0500 Subject: [PATCH 10/11] Deprecate $lib.bytes (SYN-6861) (#3570) --- .../userguides/storm_adv_control.rstorm | 6 +- synapse/lib/stormtypes.py | 149 +++++++++++++++++- synapse/tests/test_cortex.py | 4 +- synapse/tests/test_lib_ast.py | 2 +- synapse/tests/test_lib_storm.py | 2 +- synapse/tests/test_lib_stormhttp.py | 2 +- synapse/tests/test_lib_stormtypes.py | 30 +++- 7 files changed, 179 insertions(+), 16 deletions(-) diff --git a/docs/synapse/userguides/storm_adv_control.rstorm b/docs/synapse/userguides/storm_adv_control.rstorm index 235c03cd7d..ebf0344c24 100644 --- a/docs/synapse/userguides/storm_adv_control.rstorm +++ b/docs/synapse/userguides/storm_adv_control.rstorm @@ -328,7 +328,7 @@ before attempting to download it. - if $lib.bytes.has(:sha256) { } + if $lib.axon.has(:sha256) { } else { | malware.download } @@ -336,9 +336,9 @@ before attempting to download it. The Storm query above: - takes an inbound ``file:bytes`` node; -- checks for the file in the Axon (:ref:`stormlibs-lib-bytes-has`) using the ``:sha256`` value of the inbound +- checks for the file in the Axon (:ref:`stormlibs-lib-axon-has`) using the ``:sha256`` value of the inbound file; -- if ``$lib.bytes.has(:sha256)`` returns ``true`` (i.e., we have the file), do nothing (``{ }``); +- if ``$lib.axon.has(:sha256)`` returns ``true`` (i.e., we have the file), do nothing (``{ }``); - otherwise call the ``malware.download`` service to attempt to download the file. **Note:** In the above example, ``malware.download`` is used as an example Storm command; it does not exist diff --git a/synapse/lib/stormtypes.py b/synapse/lib/stormtypes.py index 4d0576abc6..31dcadbc78 100644 --- a/synapse/lib/stormtypes.py +++ b/synapse/lib/stormtypes.py @@ -2121,6 +2121,82 @@ class LibAxon(Lib): ''', 'type': {'type': 'function', '_funcname': 'metrics', 'returns': {'type': 'dict', 'desc': 'A dictionary containing runtime data about the Axon.'}}}, + {'name': 'put', 'desc': ''' + Save the given bytes variable to the Axon the Cortex is configured to use. + + Examples: + Save a base64 encoded buffer to the Axon:: + + cli> storm $s='dGVzdA==' $buf=$lib.base64.decode($s) ($size, $sha256)=$lib.axon.put($buf) + $lib.print('size={size} sha256={sha256}', size=$size, sha256=$sha256) + + size=4 sha256=9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08''', + 'type': {'type': 'function', '_funcname': 'put', + 'args': ( + {'name': 'byts', 'type': 'bytes', 'desc': 'The bytes to save.', }, + ), + 'returns': {'type': 'list', 'desc': 'A tuple of the file size and sha256 value.', }}}, + {'name': 'has', 'desc': ''' + Check if the Axon the Cortex is configured to use has a given sha256 value. + + Examples: + Check if the Axon has a given file:: + + # This example assumes the Axon does have the bytes + cli> storm if $lib.axon.has(9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08) { + $lib.print("Has bytes") + } else { + $lib.print("Does not have bytes") + } + + Has bytes + ''', + 'type': {'type': 'function', '_funcname': 'has', + 'args': ( + {'name': 'sha256', 'type': 'str', 'desc': 'The sha256 value to check.', }, + ), + 'returns': {'type': 'boolean', 'desc': 'True if the Axon has the file, false if it does not.', }}}, + {'name': 'size', 'desc': ''' + Return the size of the bytes stored in the Axon for the given sha256. + + Examples: + Get the size for a file given a variable named ``$sha256``:: + + $size = $lib.axon.size($sha256) + ''', + 'type': {'type': 'function', '_funcname': 'size', + 'args': ( + {'name': 'sha256', 'type': 'str', 'desc': 'The sha256 value to check.', }, + ), + 'returns': {'type': ['int', 'null'], + 'desc': 'The size of the file or ``null`` if the file is not found.', }}}, + {'name': 'hashset', 'desc': ''' + Return additional hashes of the bytes stored in the Axon for the given sha256. + + Examples: + Get the md5 hash for a file given a variable named ``$sha256``:: + + $hashset = $lib.axon.hashset($sha256) + $md5 = $hashset.md5 + ''', + 'type': {'type': 'function', '_funcname': 'hashset', + 'args': ( + {'name': 'sha256', 'type': 'str', 'desc': 'The sha256 value to calculate hashes for.', }, + ), + 'returns': {'type': 'dict', 'desc': 'A dictionary of additional hashes.', }}}, + {'name': 'upload', 'desc': ''' + Upload a stream of bytes to the Axon as a file. + + Examples: + Upload bytes from a generator:: + + ($size, $sha256) = $lib.axon.upload($getBytesChunks()) + ''', + 'type': {'type': 'function', '_funcname': 'upload', + 'args': ( + {'name': 'genr', 'type': 'generator', 'desc': 'A generator which yields bytes.', }, + ), + 'returns': {'type': 'list', 'desc': 'A tuple of the file size and sha256 value.', }}}, ) _storm_lib_path = ('axon',) _storm_lib_perms = ( @@ -2148,6 +2224,11 @@ def getObjLocals(self): 'jsonlines': self.jsonlines, 'csvrows': self.csvrows, 'metrics': self.metrics, + 'put': self.put, + 'has': self.has, + 'size': self.size, + 'upload': self.upload, + 'hashset': self.hashset, } def strify(self, item): @@ -2372,10 +2453,62 @@ async def metrics(self): self.runt.confirm(('storm', 'lib', 'axon', 'has')) return await self.runt.snap.core.axon.metrics() + async def upload(self, genr): + + self.runt.confirm(('axon', 'upload')) + + await self.runt.snap.core.getAxon() + async with await self.runt.snap.core.axon.upload() as upload: + async for byts in s_coro.agen(genr): + await upload.write(byts) + size, sha256 = await upload.save() + return size, s_common.ehex(sha256) + + @stormfunc(readonly=True) + async def has(self, sha256): + sha256 = await tostr(sha256, noneok=True) + if sha256 is None: + return None + + self.runt.confirm(('axon', 'has')) + + await self.runt.snap.core.getAxon() + return await self.runt.snap.core.axon.has(s_common.uhex(sha256)) + + @stormfunc(readonly=True) + async def size(self, sha256): + sha256 = await tostr(sha256) + + self.runt.confirm(('axon', 'has')) + + await self.runt.snap.core.getAxon() + return await self.runt.snap.core.axon.size(s_common.uhex(sha256)) + + async def put(self, byts): + if not isinstance(byts, bytes): + mesg = '$lib.axon.put() requires a bytes argument' + raise s_exc.BadArg(mesg=mesg) + + self.runt.confirm(('axon', 'upload')) + + await self.runt.snap.core.getAxon() + size, sha256 = await self.runt.snap.core.axon.put(byts) + + return (size, s_common.ehex(sha256)) + + @stormfunc(readonly=True) + async def hashset(self, sha256): + sha256 = await tostr(sha256) + + self.runt.confirm(('axon', 'has')) + + await self.runt.snap.core.getAxon() + return await self.runt.snap.core.axon.hashset(s_common.uhex(sha256)) + @registry.registerLib class LibBytes(Lib): ''' - A Storm Library for interacting with bytes storage. + A Storm Library for interacting with bytes storage. This Library is deprecated; use ``$lib.axon.*`` instead. ''' _storm_locals = ( {'name': 'put', 'desc': ''' @@ -2467,6 +2600,8 @@ def getObjLocals(self): } async def _libBytesUpload(self, genr): + s_common.deprecated('$lib.bytes.upload()', curv='2.162.0') + await self.runt.warnonce('$lib.bytes.upload() is deprecated. Use $lib.axon.upload() instead.') self.runt.confirm(('axon', 'upload'), default=True) @@ -2479,6 +2614,9 @@ async def _libBytesUpload(self, genr): @stormfunc(readonly=True) async def _libBytesHas(self, sha256): + s_common.deprecated('$lib.bytes.has()', curv='2.162.0') + await self.runt.warnonce('$lib.bytes.has() is deprecated. Use $lib.axon.has() instead.') + sha256 = await tostr(sha256, noneok=True) if sha256 is None: return None @@ -2492,6 +2630,9 @@ async def _libBytesHas(self, sha256): @stormfunc(readonly=True) async def _libBytesSize(self, sha256): + s_common.deprecated('$lib.bytes.size()', curv='2.162.0') + await self.runt.warnonce('$lib.bytes.size() is deprecated. Use $lib.axon.size() instead.') + sha256 = await tostr(sha256) self.runt.confirm(('axon', 'has'), default=True) @@ -2502,6 +2643,9 @@ async def _libBytesSize(self, sha256): return ret async def _libBytesPut(self, byts): + s_common.deprecated('$lib.bytes.put()', curv='2.162.0') + await self.runt.warnonce('$lib.bytes.put() is deprecated. Use $lib.axon.put() instead.') + if not isinstance(byts, bytes): mesg = '$lib.bytes.put() requires a bytes argument' raise s_exc.BadArg(mesg=mesg) @@ -2516,6 +2660,9 @@ async def _libBytesPut(self, byts): @stormfunc(readonly=True) async def _libBytesHashset(self, sha256): + s_common.deprecated('$lib.bytes.hashset()', curv='2.162.0') + await self.runt.warnonce('$lib.bytes.hashset() is deprecated. Use $lib.axon.hashset() instead.') + sha256 = await tostr(sha256) self.runt.confirm(('axon', 'has'), default=True) diff --git a/synapse/tests/test_cortex.py b/synapse/tests/test_cortex.py index 5921484d6e..76c6ef5b1e 100644 --- a/synapse/tests/test_cortex.py +++ b/synapse/tests/test_cortex.py @@ -6312,11 +6312,11 @@ async def test_cortex_axon(self): # Use dyncalls, not direct object access. asdfhash_h = '2413fb3709b05939f04cf2e92f7d0897fc2596f9ad0b8a9ea855c7bfebaae892' - size, sha2 = await core.callStorm('return( $lib.bytes.put($buf) )', + size, sha2 = await core.callStorm('return( $lib.axon.put($buf) )', {'vars': {'buf': b'asdfasdf'}}) self.eq(size, 8) self.eq(sha2, asdfhash_h) - self.true(await core.callStorm('return( $lib.bytes.has($hash) )', + self.true(await core.callStorm('return( $lib.axon.has($hash) )', {'vars': {'hash': asdfhash_h}})) unset = False diff --git a/synapse/tests/test_lib_ast.py b/synapse/tests/test_lib_ast.py index 58c89bf0fd..85962e734b 100644 --- a/synapse/tests/test_lib_ast.py +++ b/synapse/tests/test_lib_ast.py @@ -2754,7 +2754,7 @@ async def test_ast_condeval(self): opts = {'vars': {'asdf': b'asdf'}} await core.nodes('[ file:bytes=$asdf ]', opts=opts) await core.axon.put(b'asdf') - self.len(1, await core.nodes('file:bytes +$lib.bytes.has(:sha256)')) + self.len(1, await core.nodes('file:bytes +$lib.axon.has(:sha256)')) async def test_ast_walkcond(self): diff --git a/synapse/tests/test_lib_storm.py b/synapse/tests/test_lib_storm.py index 1e20b9f090..9ec27673d3 100644 --- a/synapse/tests/test_lib_storm.py +++ b/synapse/tests/test_lib_storm.py @@ -4513,7 +4513,7 @@ async def test_lib_storm_delnode(self): visi = await core.auth.addUser('visi') await visi.addRule((True, ('node',))) - size, sha256 = await core.callStorm('return($lib.bytes.put($buf))', {'vars': {'buf': b'asdfasdf'}}) + size, sha256 = await core.callStorm('return($lib.axon.put($buf))', {'vars': {'buf': b'asdfasdf'}}) self.len(1, await core.nodes(f'[ file:bytes={sha256} ]')) diff --git a/synapse/tests/test_lib_stormhttp.py b/synapse/tests/test_lib_stormhttp.py index cc6c75ecbe..5ab18a5f35 100644 --- a/synapse/tests/test_lib_stormhttp.py +++ b/synapse/tests/test_lib_stormhttp.py @@ -466,7 +466,7 @@ async def test_storm_http_post_file(self): await root.setPasswd('root') text = ''' $url = $lib.str.format("https://root:root@127.0.0.1:{port}/api/v1/storm", port=$port) - $stormq = "($size, $sha2) = $lib.bytes.put($lib.base64.decode('dmVydGV4')) [ test:str = $sha2 ] [ test:int = $size ]" + $stormq = "($size, $sha2) = $lib.axon.put($lib.base64.decode('dmVydGV4')) [ test:str = $sha2 ] [ test:int = $size ]" $json = ({"query": $stormq}) $bytez = $lib.inet.http.post($url, json=$json, ssl_verify=$(0)) ''' diff --git a/synapse/tests/test_lib_stormtypes.py b/synapse/tests/test_lib_stormtypes.py index 221e0ed60a..1b65d56cb2 100644 --- a/synapse/tests/test_lib_stormtypes.py +++ b/synapse/tests/test_lib_stormtypes.py @@ -3008,11 +3008,16 @@ async def test_storm_lib_bytes(self): async with self.getTestCore() as core: + opts = {'vars': {'bytes': 10}} + with self.raises(s_exc.BadArg): - opts = {'vars': {'bytes': 10}} text = '($size, $sha2) = $lib.bytes.put($bytes)' nodes = await core.nodes(text, opts=opts) + with self.raises(s_exc.BadArg): + text = '($size, $sha2) = $lib.axon.put($bytes)' + nodes = await core.nodes(text, opts=opts) + asdf = b'asdfasdf' asdfset = s_hashset.HashSet() asdfset.update(asdf) @@ -3024,6 +3029,7 @@ async def test_storm_lib_bytes(self): ret = await core.callStorm('return($lib.bytes.has($hash))', {'vars': {'hash': asdfhash_h}}) self.false(ret) self.false(await core.callStorm('return($lib.bytes.has($lib.null))')) + self.false(await core.callStorm('return($lib.axon.has($lib.null))')) opts = {'vars': {'bytes': asdf}} text = '($size, $sha2) = $lib.bytes.put($bytes) [ test:int=$size test:str=$sha2 ]' @@ -3033,10 +3039,14 @@ async def test_storm_lib_bytes(self): opts = {'vars': {'sha256': asdfhash_h}} self.eq(8, await core.callStorm('return($lib.bytes.size($sha256))', opts=opts)) + self.eq(8, await core.callStorm('return($lib.axon.size($sha256))', opts=opts)) hashset = await core.callStorm('return($lib.bytes.hashset($sha256))', opts=opts) self.eq(hashset, hashes) + hashset = await core.callStorm('return($lib.axon.hashset($sha256))', opts=opts) + self.eq(hashset, hashes) + self.eq(nodes[0].ndef, ('test:int', 8)) self.eq(nodes[1].ndef, ('test:str', asdfhash_h)) @@ -3047,6 +3057,9 @@ async def test_storm_lib_bytes(self): ret = await core.callStorm('return($lib.bytes.has($hash))', {'vars': {'hash': asdfhash_h}}) self.true(ret) + ret = await core.callStorm('return($lib.axon.has($hash))', {'vars': {'hash': asdfhash_h}}) + self.true(ret) + # Allow bytes to be directly decoded as a string opts = {'vars': {'buf': 'hehe'.encode()}} nodes = await core.nodes('$valu=$buf.decode() [test:str=$valu]', opts) @@ -3096,6 +3109,9 @@ async def test_storm_lib_bytes(self): retn = await core.callStorm('return($lib.bytes.upload($chunks))', opts=opts) self.eq((8, '9ed8ffd0a11e337e6e461358195ebf8ea2e12a82db44561ae5d9e638f6f922c4'), retn) + retn = await core.callStorm('return($lib.axon.upload($chunks))', opts=opts) + self.eq((8, '9ed8ffd0a11e337e6e461358195ebf8ea2e12a82db44561ae5d9e638f6f922c4'), retn) + visi = await core.auth.addUser('visi') await visi.addRule((False, ('axon', 'has'))) @@ -3133,7 +3149,7 @@ async def test_storm_lib_base64(self): self.eq(nodes[0].ndef, ('test:str', 'Zm9vYmE_')) opts = {'vars': {'bytes': nodes[0].ndef[1]}} - text = '$lib.bytes.put($lib.base64.decode($bytes))' + text = '$lib.axon.put($lib.base64.decode($bytes))' nodes = await core.nodes(text, opts) key = binascii.unhexlify(hashlib.sha256(base64.urlsafe_b64decode(opts['vars']['bytes'])).hexdigest()) byts = b''.join([b async for b in core.axon.get(key)]) @@ -3147,7 +3163,7 @@ async def test_storm_lib_base64(self): self.eq(nodes[0].ndef, ('test:str', 'Zm9vYmE/')) opts = {'vars': {'bytes': nodes[0].ndef[1]}} - text = '$lib.bytes.put($lib.base64.decode($bytes, $(0)))' + text = '$lib.axon.put($lib.base64.decode($bytes, $(0)))' nodes = await core.nodes(text, opts) key = binascii.unhexlify(hashlib.sha256(base64.urlsafe_b64decode(opts['vars']['bytes'])).hexdigest()) byts = b''.join([b async for b in core.axon.get(key)]) @@ -5709,7 +5725,7 @@ async def test_storm_lib_axon(self): self.len(1, await core.callStorm('$x=$lib.list() for $i in $lib.axon.list() { $x.append($i) } return($x)')) - size, sha256 = await core.callStorm('return($lib.bytes.put($buf))', opts={'vars': {'buf': b'foo'}}) + size, sha256 = await core.callStorm('return($lib.axon.put($buf))', opts={'vars': {'buf': b'foo'}}) items = await core.callStorm('$x=$lib.list() for $i in $lib.axon.list() { $x.append($i) } return($x)') self.len(2, items) @@ -5731,9 +5747,9 @@ async def timeout(self): self.true(resp['ok']) opts = {'vars': {'linesbuf': linesbuf, 'jsonsbuf': jsonsbuf, 'asdfbuf': b'asdf'}} - asdfitem = await core.callStorm('return($lib.bytes.put($asdfbuf))', opts=opts) - linesitem = await core.callStorm('return($lib.bytes.put($linesbuf))', opts=opts) - jsonsitem = await core.callStorm('return($lib.bytes.put($jsonsbuf))', opts=opts) + asdfitem = await core.callStorm('return($lib.axon.put($asdfbuf))', opts=opts) + linesitem = await core.callStorm('return($lib.axon.put($linesbuf))', opts=opts) + jsonsitem = await core.callStorm('return($lib.axon.put($jsonsbuf))', opts=opts) opts = {'vars': {'sha256': asdfitem[1]}} self.eq(('asdf',), await core.callStorm(''' From 46aacd1ab7a509f703b8b2b4c5a75a5eada832da Mon Sep 17 00:00:00 2001 From: mikemoritz <57907149+mikemoritz@users.noreply.github.com> Date: Tue, 20 Feb 2024 04:54:03 -0800 Subject: [PATCH 11/11] Support mTLS in HTTP requests (SYN-5896) (#3566) --- synapse/axon.py | 88 ++++++++------- synapse/cortex.py | 4 + synapse/lib/cell.py | 63 +++++++++++ synapse/lib/oauth.py | 8 +- synapse/lib/schemas.py | 10 ++ synapse/lib/stormhttp.py | 80 +++++++++----- synapse/lib/stormtypes.py | 45 +++++++- synapse/tests/test_lib_stormhttp.py | 163 ++++++++++++++++++++++++++++ 8 files changed, 381 insertions(+), 80 deletions(-) diff --git a/synapse/axon.py b/synapse/axon.py index 4902ff532c..ae18e30ca0 100644 --- a/synapse/axon.py +++ b/synapse/axon.py @@ -593,7 +593,8 @@ async def dels(self, sha256s): await self._reqUserAllowed(('axon', 'del')) return await self.cell.dels(sha256s) - async def wget(self, url, params=None, headers=None, json=None, body=None, method='GET', ssl=True, timeout=None, proxy=None): + async def wget(self, url, params=None, headers=None, json=None, body=None, method='GET', + ssl=True, timeout=None, proxy=None, ssl_opts=None): ''' Stream a file download directly into the Axon. @@ -606,11 +607,20 @@ async def wget(self, url, params=None, headers=None, json=None, body=None, metho method (str): The HTTP method to use. ssl (bool): Perform SSL verification. timeout (int): The timeout of the request, in seconds. + ssl_opts (dict): Additional SSL/TLS options. Notes: The response body will be stored, regardless of the response code. The ``ok`` value in the reponse does not reflect that a status code, such as a 404, was encountered when retrieving the URL. + The ssl_opts dictionary may contain the following values:: + + { + 'verify': - Perform SSL/TLS verification. Is overridden by the ssl argument. + 'client_cert': - PEM encoded full chain certificate for use in mTLS. + 'client_key': - PEM encoded key for use in mTLS. Alternatively, can be included in client_cert. + } + The dictionary returned by this may contain the following values:: { @@ -640,18 +650,20 @@ async def wget(self, url, params=None, headers=None, json=None, body=None, metho dict: An information dictionary containing the results of the request. ''' await self._reqUserAllowed(('axon', 'wget')) - return await self.cell.wget(url, params=params, headers=headers, json=json, body=body, method=method, ssl=ssl, - timeout=timeout, proxy=proxy) + return await self.cell.wget(url, params=params, headers=headers, json=json, body=body, method=method, + ssl=ssl, timeout=timeout, proxy=proxy, ssl_opts=ssl_opts) - async def postfiles(self, fields, url, params=None, headers=None, method='POST', ssl=True, timeout=None, proxy=None): + async def postfiles(self, fields, url, params=None, headers=None, method='POST', + ssl=True, timeout=None, proxy=None, ssl_opts=None): await self._reqUserAllowed(('axon', 'wput')) - return await self.cell.postfiles(fields, url, params=params, headers=headers, - method=method, ssl=ssl, timeout=timeout, proxy=proxy) + return await self.cell.postfiles(fields, url, params=params, headers=headers, method=method, + ssl=ssl, timeout=timeout, proxy=proxy, ssl_opts=ssl_opts) - async def wput(self, sha256, url, params=None, headers=None, method='PUT', ssl=True, timeout=None, proxy=None): + async def wput(self, sha256, url, params=None, headers=None, method='PUT', + ssl=True, timeout=None, proxy=None, ssl_opts=None): await self._reqUserAllowed(('axon', 'wput')) - return await self.cell.wput(sha256, url, params=params, headers=headers, method=method, ssl=ssl, - timeout=timeout, proxy=proxy) + return await self.cell.wput(sha256, url, params=params, headers=headers, method=method, + ssl=ssl, timeout=timeout, proxy=proxy, ssl_opts=ssl_opts) async def metrics(self): ''' @@ -1434,7 +1446,8 @@ async def jsonlines(self, sha256, errors='ignore'): raise s_exc.BadJsonText(mesg=f'Bad json line encountered while processing {sha256}, ({e})', sha256=sha256) from None - async def postfiles(self, fields, url, params=None, headers=None, method='POST', ssl=True, timeout=None, proxy=None): + async def postfiles(self, fields, url, params=None, headers=None, method='POST', + ssl=True, timeout=None, proxy=None, ssl_opts=None): ''' Send files from the axon as fields in a multipart/form-data HTTP request. @@ -1447,6 +1460,7 @@ async def postfiles(self, fields, url, params=None, headers=None, method='POST', ssl (bool): Perform SSL verification. timeout (int): The timeout of the request, in seconds. proxy (bool|str|null): Use a specific proxy or disable proxy use. + ssl_opts (dict): Additional SSL/TLS options. Notes: The dictionaries in the fields list may contain the following values:: @@ -1460,6 +1474,14 @@ async def postfiles(self, fields, url, params=None, headers=None, method='POST', 'content_transfer_encoding': - Optional content-transfer-encoding header for the field. } + The ssl_opts dictionary may contain the following values:: + + { + 'verify': - Perform SSL/TLS verification. Is overridden by the ssl argument. + 'client_cert': - PEM encoded full chain certificate for use in mTLS. + 'client_key': - PEM encoded key for use in mTLS. Alternatively, can be included in client_cert. + } + The dictionary returned by this may contain the following values:: { @@ -1478,20 +1500,12 @@ async def postfiles(self, fields, url, params=None, headers=None, method='POST', if proxy is None: proxy = self.conf.get('http:proxy') - cadir = self.conf.get('tls:ca:dir') + ssl = self.getCachedSslCtx(opts=ssl_opts, verify=ssl) connector = None if proxy: connector = aiohttp_socks.ProxyConnector.from_url(proxy) - if ssl is False: - pass - elif cadir: - ssl = s_common.getSslCtx(cadir) - else: - # default aiohttp behavior - ssl = None - atimeout = aiohttp.ClientTimeout(total=timeout) async with aiohttp.ClientSession(connector=connector, timeout=atimeout) as sess: @@ -1556,27 +1570,19 @@ async def postfiles(self, fields, url, params=None, headers=None, method='POST', } async def wput(self, sha256, url, params=None, headers=None, method='PUT', ssl=True, timeout=None, - filename=None, filemime=None, proxy=None): + filename=None, filemime=None, proxy=None, ssl_opts=None): ''' Stream a blob from the axon as the body of an HTTP request. ''' if proxy is None: - prox = self.conf.get('http:proxy') + proxy = self.conf.get('http:proxy') - cadir = self.conf.get('tls:ca:dir') + ssl = self.getCachedSslCtx(opts=ssl_opts, verify=ssl) connector = None if proxy: connector = aiohttp_socks.ProxyConnector.from_url(proxy) - if ssl is False: - pass - elif cadir: - ssl = s_common.getSslCtx(cadir) - else: - # default aiohttp behavior - ssl = None - atimeout = aiohttp.ClientTimeout(total=timeout) async with aiohttp.ClientSession(connector=connector, timeout=atimeout) as sess: @@ -1638,7 +1644,8 @@ def _flatten_clientresponse(self, return info - async def wget(self, url, params=None, headers=None, json=None, body=None, method='GET', ssl=True, timeout=None, proxy=None): + async def wget(self, url, params=None, headers=None, json=None, body=None, method='GET', + ssl=True, timeout=None, proxy=None, ssl_opts=None): ''' Stream a file download directly into the Axon. @@ -1652,11 +1659,20 @@ async def wget(self, url, params=None, headers=None, json=None, body=None, metho ssl (bool): Perform SSL verification. timeout (int): The timeout of the request, in seconds. proxy (bool|str|null): Use a specific proxy or disable proxy use. + ssl_opts (dict): Additional SSL/TLS options. Notes: The response body will be stored, regardless of the response code. The ``ok`` value in the reponse does not reflect that a status code, such as a 404, was encountered when retrieving the URL. + The ssl_opts dictionary may contain the following values:: + + { + 'verify': - Perform SSL/TLS verification. Is overridden by the ssl argument. + 'client_cert': - PEM encoded full chain certificate for use in mTLS. + 'client_key': - PEM encoded key for use in mTLS. Alternatively, can be included in client_cert. + } + The dictionary returned by this may contain the following values:: { @@ -1690,7 +1706,7 @@ async def wget(self, url, params=None, headers=None, json=None, body=None, metho if proxy is None: proxy = self.conf.get('http:proxy') - cadir = self.conf.get('tls:ca:dir') + ssl = self.getCachedSslCtx(opts=ssl_opts, verify=ssl) connector = None if proxy: @@ -1698,14 +1714,6 @@ async def wget(self, url, params=None, headers=None, json=None, body=None, metho atimeout = aiohttp.ClientTimeout(total=timeout) - if ssl is False: - pass - elif cadir: - ssl = s_common.getSslCtx(cadir) - else: - # default aiohttp behavior - ssl = None - async with aiohttp.ClientSession(connector=connector, timeout=atimeout) as sess: try: diff --git a/synapse/cortex.py b/synapse/cortex.py index adf5b5544d..251cd969d7 100644 --- a/synapse/cortex.py +++ b/synapse/cortex.py @@ -3879,6 +3879,10 @@ async def _initCoreAxon(self): if proxyurl is not None: conf['http:proxy'] = proxyurl + cadir = self.conf.get('tls:ca:dir') + if cadir is not None: + conf['tls:ca:dir'] = cadir + self.axon = await s_axon.Axon.anit(path, conf=conf, parent=self) self.axoninfo = await self.axon.getCellInfo() self.axon.onfini(self.axready.clear) diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index 3ba4bdc44b..3ceef3c65d 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -1,5 +1,6 @@ import gc import os +import ssl import copy import time import fcntl @@ -12,6 +13,7 @@ import argparse import datetime import platform +import tempfile import functools import contextlib import multiprocessing @@ -32,6 +34,7 @@ import synapse.lib.coro as s_coro import synapse.lib.hive as s_hive import synapse.lib.link as s_link +import synapse.lib.cache as s_cache import synapse.lib.const as s_const import synapse.lib.nexus as s_nexus import synapse.lib.queue as s_queue @@ -58,6 +61,7 @@ logger = logging.getLogger(__name__) SLAB_MAP_SIZE = 128 * s_const.mebibyte +SSLCTX_CACHE_SIZE = 1000 ''' Base classes for the synapse "cell" microservice architecture. @@ -1151,6 +1155,8 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None): self.usermetadb = self.slab.initdb('user:meta') # useriden + -> dict valu self.rolemetadb = self.slab.initdb('role:meta') # roleiden + -> dict valu + self._sslctx_cache = s_cache.FixedCache(self._makeCachedSslCtx, size=SSLCTX_CACHE_SIZE) + self.hive = await self._initCellHive() # self.cellinfo, a HiveDict for general purpose persistent storage @@ -4342,3 +4348,60 @@ async def _onUserDelEvnt(self, evnt): self.slab.delete(key_iden, db=self.apikeydb) self.slab.delete(lkey, db=self.usermetadb) await asyncio.sleep(0) + + def _makeCachedSslCtx(self, opts): + + opts = dict(opts) + + cadir = self.conf.get('tls:ca:dir') + + if cadir is not None: + sslctx = s_common.getSslCtx(cadir, purpose=ssl.Purpose.SERVER_AUTH) + else: + sslctx = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH) + + if not opts['verify']: + sslctx.check_hostname = False + sslctx.verify_mode = ssl.CERT_NONE + + if not opts['client_cert']: + return sslctx + + client_cert = opts['client_cert'].encode() + + if opts['client_key']: + client_key = opts['client_key'].encode() + else: + client_key = None + client_key_path = None + + with self.getTempDir() as tmpdir: + + with tempfile.NamedTemporaryFile(dir=tmpdir, mode='wb', delete=False) as fh: + fh.write(client_cert) + client_cert_path = fh.name + + if client_key: + with tempfile.NamedTemporaryFile(dir=tmpdir, mode='wb', delete=False) as fh: + fh.write(client_key) + client_key_path = fh.name + + try: + sslctx.load_cert_chain(client_cert_path, keyfile=client_key_path) + except ssl.SSLError as e: + raise s_exc.BadArg(mesg=f'Error loading client cert: {str(e)}') from None + + return sslctx + + def getCachedSslCtx(self, opts=None, verify=None): + + if opts is None: + opts = {} + + if verify is not None: + opts['verify'] = verify + + opts = s_schemas.reqValidSslCtxOpts(opts) + + key = tuple(sorted(opts.items())) + return self._sslctx_cache.get(key) diff --git a/synapse/lib/oauth.py b/synapse/lib/oauth.py index 72c5591234..3a8fe9cf7a 100644 --- a/synapse/lib/oauth.py +++ b/synapse/lib/oauth.py @@ -230,13 +230,7 @@ async def _fetchOAuthToken(self, url, auth, formdata, ssl_verify=True, retries=1 timeout = aiohttp.ClientTimeout(total=DEFAULT_TIMEOUT) - cadir = self.conf.get('tls:ca:dir') - if ssl_verify is False: - ssl = False - elif cadir: - ssl = s_common.getSslCtx(cadir) - else: - ssl = None + ssl = self.getCachedSslCtx(verify=ssl_verify) async with aiohttp.ClientSession(timeout=timeout) as sess: diff --git a/synapse/lib/schemas.py b/synapse/lib/schemas.py index b46a9b28a2..0945d5f53c 100644 --- a/synapse/lib/schemas.py +++ b/synapse/lib/schemas.py @@ -263,3 +263,13 @@ ], } reqValidUserApiKeyDef = s_config.getJsValidator(_cellUserApiKeySchema) + +reqValidSslCtxOpts = s_config.getJsValidator({ + 'type': 'object', + 'properties': { + 'verify': {'type': 'boolean', 'default': True}, + 'client_cert': {'type': ['string', 'null'], 'default': None}, + 'client_key': {'type': ['string', 'null'], 'default': None}, + }, + 'additionalProperties': False, +}) diff --git a/synapse/lib/stormhttp.py b/synapse/lib/stormhttp.py index c79f295693..38db0d8a61 100644 --- a/synapse/lib/stormhttp.py +++ b/synapse/lib/stormhttp.py @@ -13,6 +13,7 @@ import synapse.lib.base as s_base import synapse.lib.msgpack as s_msgpack +import synapse.lib.version as s_version import synapse.lib.stormtypes as s_stormtypes @s_stormtypes.registry.registerType @@ -87,6 +88,14 @@ async def rx(self, timeout=None): class LibHttp(s_stormtypes.Lib): ''' A Storm Library exposing an HTTP client API. + + For APIs that accept an ssl_opts argument, the dictionary may contain the following values:: + + { + 'verify': - Perform SSL/TLS verification. Is overridden by the ssl_verify argument. + 'client_cert': - PEM encoded full chain certificate for use in mTLS. + 'client_key': - PEM encoded key for use in mTLS. Alternatively, can be included in client_cert. + } ''' _storm_locals = ( {'name': 'get', 'desc': 'Get the contents of a given URL.', @@ -105,6 +114,9 @@ class LibHttp(s_stormtypes.Lib): 'default': True}, {'name': 'proxy', 'type': ['bool', 'null', 'str'], 'desc': 'Set to a proxy URL string or $lib.false to disable proxy use.', 'default': None}, + {'name': 'ssl_opts', 'type': 'dict', + 'desc': 'Optional SSL/TLS options. See $lib.inet.http help for additional details.', + 'default': None}, ), 'returns': {'type': 'inet:http:resp', 'desc': 'The response object.'}}}, {'name': 'post', 'desc': 'Post data to a given URL.', @@ -134,6 +146,9 @@ class LibHttp(s_stormtypes.Lib): 'default': None}, {'name': 'proxy', 'type': ['bool', 'null', 'str'], 'desc': 'Set to a proxy URL string or $lib.false to disable proxy use.', 'default': None}, + {'name': 'ssl_opts', 'type': 'dict', + 'desc': 'Optional SSL/TLS options. See $lib.inet.http help for additional details.', + 'default': None}, ), 'returns': {'type': 'inet:http:resp', 'desc': 'The response object.'}}}, {'name': 'head', 'desc': 'Get the HEAD response for a URL.', @@ -153,6 +168,9 @@ class LibHttp(s_stormtypes.Lib): 'default': False}, {'name': 'proxy', 'type': ['bool', 'null', 'str'], 'desc': 'Set to a proxy URL string or $lib.false to disable proxy use.', 'default': None}, + {'name': 'ssl_opts', 'type': 'dict', + 'desc': 'Optional SSL/TLS options. See $lib.inet.http help for additional details.', + 'default': None}, ), 'returns': {'type': 'inet:http:resp', 'desc': 'The response object.'}}}, {'name': 'request', 'desc': 'Make an HTTP request using the given HTTP method to the url.', @@ -183,6 +201,9 @@ class LibHttp(s_stormtypes.Lib): 'default': None}, {'name': 'proxy', 'type': ['bool', 'null', 'str'], 'desc': 'Set to a proxy URL string or $lib.false to disable proxy use.', 'default': None}, + {'name': 'ssl_opts', 'type': 'dict', + 'desc': 'Optional SSL/TLS options. See $lib.inet.http help for additional details.', + 'default': None}, ), 'returns': {'type': 'inet:http:resp', 'desc': 'The response object.'} } @@ -201,6 +222,9 @@ class LibHttp(s_stormtypes.Lib): 'default': None}, {'name': 'proxy', 'type': ['bool', 'null', 'str'], 'desc': 'Set to a proxy URL string or $lib.false to disable proxy use.', 'default': None}, + {'name': 'ssl_opts', 'type': 'dict', + 'desc': 'Optional SSL/TLS options. See $lib.inet.http help for additional details.', + 'default': None}, ), 'returns': {'type': 'inet:http:socket', 'desc': 'A websocket object.'}}}, {'name': 'urlencode', 'desc': ''' @@ -290,28 +314,31 @@ async def codereason(self, code): return s_common.httpcodereason(code) async def _httpEasyHead(self, url, headers=None, ssl_verify=True, params=None, timeout=300, - allow_redirects=False, proxy=None): + allow_redirects=False, proxy=None, ssl_opts=None): return await self._httpRequest('HEAD', url, headers=headers, ssl_verify=ssl_verify, params=params, - timeout=timeout, allow_redirects=allow_redirects, proxy=proxy) + timeout=timeout, allow_redirects=allow_redirects, proxy=proxy, ssl_opts=ssl_opts) async def _httpEasyGet(self, url, headers=None, ssl_verify=True, params=None, timeout=300, - allow_redirects=True, proxy=None): + allow_redirects=True, proxy=None, ssl_opts=None): return await self._httpRequest('GET', url, headers=headers, ssl_verify=ssl_verify, params=params, - timeout=timeout, allow_redirects=allow_redirects, proxy=proxy) + timeout=timeout, allow_redirects=allow_redirects, proxy=proxy, ssl_opts=ssl_opts) async def _httpPost(self, url, headers=None, json=None, body=None, ssl_verify=True, - params=None, timeout=300, allow_redirects=True, fields=None, proxy=None): + params=None, timeout=300, allow_redirects=True, fields=None, proxy=None, ssl_opts=None): return await self._httpRequest('POST', url, headers=headers, json=json, body=body, ssl_verify=ssl_verify, params=params, timeout=timeout, - allow_redirects=allow_redirects, fields=fields, proxy=proxy) + allow_redirects=allow_redirects, fields=fields, proxy=proxy, ssl_opts=ssl_opts) - async def inetHttpConnect(self, url, headers=None, ssl_verify=True, timeout=300, params=None, proxy=None): + async def inetHttpConnect(self, url, headers=None, ssl_verify=True, timeout=300, + params=None, proxy=None, ssl_opts=None): url = await s_stormtypes.tostr(url) headers = await s_stormtypes.toprim(headers) timeout = await s_stormtypes.toint(timeout, noneok=True) params = await s_stormtypes.toprim(params) proxy = await s_stormtypes.toprim(proxy) + ssl_verify = await s_stormtypes.tobool(ssl_verify, noneok=True) + ssl_opts = await s_stormtypes.toprim(ssl_opts) headers = self.strify(headers) @@ -332,15 +359,7 @@ async def inetHttpConnect(self, url, headers=None, ssl_verify=True, timeout=300, if params: kwargs['params'] = params - cadir = self.runt.snap.core.conf.get('tls:ca:dir') - - if ssl_verify is False: - kwargs['ssl'] = False - elif cadir: - kwargs['ssl'] = s_common.getSslCtx(cadir) - else: - # default aiohttp behavior - kwargs['ssl'] = None + kwargs['ssl'] = self.runt.snap.core.getCachedSslCtx(opts=ssl_opts, verify=ssl_verify) try: sess = await sock.enter_context(aiohttp.ClientSession(connector=connector, timeout=timeout)) @@ -374,7 +393,7 @@ def _buildFormData(self, fields): async def _httpRequest(self, meth, url, headers=None, json=None, body=None, ssl_verify=True, params=None, timeout=300, allow_redirects=True, - fields=None, proxy=None): + fields=None, proxy=None, ssl_opts=None): meth = await s_stormtypes.tostr(meth) url = await s_stormtypes.tostr(url) json = await s_stormtypes.toprim(json) @@ -386,6 +405,7 @@ async def _httpRequest(self, meth, url, headers=None, json=None, body=None, ssl_verify = await s_stormtypes.tobool(ssl_verify, noneok=True) allow_redirects = await s_stormtypes.tobool(allow_redirects) proxy = await s_stormtypes.toprim(proxy) + ssl_opts = await s_stormtypes.toprim(ssl_opts) kwargs = {'allow_redirects': allow_redirects} if params: @@ -399,12 +419,24 @@ async def _httpRequest(self, meth, url, headers=None, json=None, body=None, if fields: if any(['sha256' in field for field in fields]): self.runt.confirm(('storm', 'lib', 'axon', 'wput')) + + kwargs = {} + axonvers = self.runt.snap.core.axoninfo['synapse']['version'] + if axonvers >= s_stormtypes.AXON_MINVERS_PROXY: + kwargs['proxy'] = proxy + + if ssl_opts is not None: + mesg = f'The ssl_opts argument requires an Axon Synapse version {s_stormtypes.AXON_MINVERS_SSLOPTS}, ' \ + f'but the Axon is running {axonvers}' + s_version.reqVersion(axonvers, s_stormtypes.AXON_MINVERS_SSLOPTS, mesg=mesg) + kwargs['ssl_opts'] = ssl_opts + axon = self.runt.snap.core.axon - info = await axon.postfiles(fields, url, headers=headers, params=params, - method=meth, ssl=ssl_verify, timeout=timeout, proxy=proxy) + info = await axon.postfiles(fields, url, headers=headers, params=params, method=meth, + ssl=ssl_verify, timeout=timeout, **kwargs) return HttpResp(info) - cadir = self.runt.snap.core.conf.get('tls:ca:dir') + kwargs['ssl'] = self.runt.snap.core.getCachedSslCtx(opts=ssl_opts, verify=ssl_verify) if proxy is None: proxy = await self.runt.snap.core.getConfOpt('http:proxy') @@ -413,14 +445,6 @@ async def _httpRequest(self, meth, url, headers=None, json=None, body=None, if proxy: connector = aiohttp_socks.ProxyConnector.from_url(proxy) - if ssl_verify is False: - kwargs['ssl'] = False - elif cadir: - kwargs['ssl'] = s_common.getSslCtx(cadir) - else: - # default aiohttp behavior - kwargs['ssl'] = None - timeout = aiohttp.ClientTimeout(total=timeout) async with aiohttp.ClientSession(connector=connector, timeout=timeout) as sess: diff --git a/synapse/lib/stormtypes.py b/synapse/lib/stormtypes.py index 31dcadbc78..b5acf0e828 100644 --- a/synapse/lib/stormtypes.py +++ b/synapse/lib/stormtypes.py @@ -34,11 +34,15 @@ import synapse.lib.msgpack as s_msgpack import synapse.lib.trigger as s_trigger import synapse.lib.urlhelp as s_urlhelp +import synapse.lib.version as s_version import synapse.lib.stormctrl as s_stormctrl import synapse.lib.provenance as s_provenance logger = logging.getLogger(__name__) +AXON_MINVERS_PROXY = (2, 97, 0) +AXON_MINVERS_SSLOPTS = '>=2.162.0' + class Undef: _storm_typename = 'undef' async def stormrepr(self): @@ -1916,6 +1920,14 @@ async def join(self, sepr, items): class LibAxon(Lib): ''' A Storm library for interacting with the Cortex's Axon. + + For APIs that accept an ssl_opts argument, the dictionary may contain the following values:: + + { + 'verify': - Perform SSL/TLS verification. Is overridden by the ssl argument. + 'client_cert': - PEM encoded full chain certificate for use in mTLS. + 'client_key': - PEM encoded key for use in mTLS. Alternatively, can be included in client_cert. + } ''' _storm_locals = ( {'name': 'wget', 'desc': """ @@ -1951,6 +1963,9 @@ class LibAxon(Lib): 'default': None}, {'name': 'proxy', 'type': ['bool', 'null', 'str'], 'desc': 'Set to a proxy URL string or $lib.false to disable proxy use.', 'default': None}, + {'name': 'ssl_opts', 'type': 'dict', + 'desc': 'Optional SSL/TLS options. See $lib.axon help for additional details.', + 'default': None}, ), 'returns': {'type': 'dict', 'desc': 'A status dictionary of metadata.'}}}, {'name': 'wput', 'desc': """ @@ -1971,6 +1986,9 @@ class LibAxon(Lib): 'default': None}, {'name': 'proxy', 'type': ['bool', 'null', 'str'], 'desc': 'Set to a proxy URL string or $lib.false to disable proxy use.', 'default': None}, + {'name': 'ssl_opts', 'type': 'dict', + 'desc': 'Optional SSL/TLS options. See $lib.axon help for additional details.', + 'default': None}, ), 'returns': {'type': 'dict', 'desc': 'A status dictionary of metadata.'}}}, {'name': 'urlfile', 'desc': ''' @@ -2288,7 +2306,8 @@ async def del_(self, sha256): axon = self.runt.snap.core.axon return await axon.del_(sha256b) - async def wget(self, url, headers=None, params=None, method='GET', json=None, body=None, ssl=True, timeout=None, proxy=None): + async def wget(self, url, headers=None, params=None, method='GET', json=None, body=None, + ssl=True, timeout=None, proxy=None, ssl_opts=None): if not self.runt.allowed(('axon', 'wget')): self.runt.confirm(('storm', 'lib', 'axon', 'wget')) @@ -2303,6 +2322,7 @@ async def wget(self, url, headers=None, params=None, method='GET', json=None, bo headers = await toprim(headers) timeout = await toprim(timeout) proxy = await toprim(proxy) + ssl_opts = await toprim(ssl_opts) if proxy is not None: self.runt.confirm(('storm', 'lib', 'inet', 'http', 'proxy')) @@ -2314,16 +2334,23 @@ async def wget(self, url, headers=None, params=None, method='GET', json=None, bo kwargs = {} axonvers = self.runt.snap.core.axoninfo['synapse']['version'] - if axonvers >= (2, 97, 0): + if axonvers >= AXON_MINVERS_PROXY: kwargs['proxy'] = proxy + if ssl_opts is not None: + mesg = f'The ssl_opts argument requires an Axon Synapse version {AXON_MINVERS_SSLOPTS}, ' \ + f'but the Axon is running {axonvers}' + s_version.reqVersion(axonvers, AXON_MINVERS_SSLOPTS, mesg=mesg) + kwargs['ssl_opts'] = ssl_opts + axon = self.runt.snap.core.axon resp = await axon.wget(url, headers=headers, params=params, method=method, ssl=ssl, body=body, json=json, timeout=timeout, **kwargs) resp['original_url'] = url return resp - async def wput(self, sha256, url, headers=None, params=None, method='PUT', ssl=True, timeout=None, proxy=None): + async def wput(self, sha256, url, headers=None, params=None, method='PUT', + ssl=True, timeout=None, proxy=None, ssl_opts=None): if not self.runt.allowed(('axon', 'wput')): self.runt.confirm(('storm', 'lib', 'axon', 'wput')) @@ -2337,6 +2364,7 @@ async def wput(self, sha256, url, headers=None, params=None, method='PUT', ssl=T params = await toprim(params) headers = await toprim(headers) timeout = await toprim(timeout) + ssl_opts = await toprim(ssl_opts) params = self.strify(params) headers = self.strify(headers) @@ -2349,10 +2377,17 @@ async def wput(self, sha256, url, headers=None, params=None, method='PUT', ssl=T kwargs = {} axonvers = self.runt.snap.core.axoninfo['synapse']['version'] - if axonvers >= (2, 97, 0): + if axonvers >= AXON_MINVERS_PROXY: kwargs['proxy'] = proxy - return await axon.wput(sha256byts, url, headers=headers, params=params, method=method, ssl=ssl, timeout=timeout, **kwargs) + if ssl_opts is not None: + mesg = f'The ssl_opts argument requires an Axon Synapse version {AXON_MINVERS_SSLOPTS}, ' \ + f'but the Axon is running {axonvers}' + s_version.reqVersion(axonvers, AXON_MINVERS_SSLOPTS, mesg=mesg) + kwargs['ssl_opts'] = ssl_opts + + return await axon.wput(sha256byts, url, headers=headers, params=params, method=method, + ssl=ssl, timeout=timeout, **kwargs) async def urlfile(self, *args, **kwargs): gateiden = self.runt.snap.wlyr.iden diff --git a/synapse/tests/test_lib_stormhttp.py b/synapse/tests/test_lib_stormhttp.py index 5ab18a5f35..703c9e1b9d 100644 --- a/synapse/tests/test_lib_stormhttp.py +++ b/synapse/tests/test_lib_stormhttp.py @@ -1,4 +1,5 @@ import os +import ssl import json import shutil @@ -672,3 +673,165 @@ async def test_storm_http_connect(self): with self.raises(s_stormctrl.StormExit) as cm: await core.callStorm(query, opts=opts) self.isin('connect to proxy 127.0.0.1:1', str(cm.exception)) + + async def test_storm_http_mtls(self): + + with self.getTestDir() as dirn: + + cdir = s_common.gendir(dirn, 'certs') + cadir = s_common.gendir(cdir, 'cas') + tdir = s_certdir.CertDir(cdir) + tdir.genCaCert('somelocalca') + tdir.genHostCert('localhost', signas='somelocalca') + + localkeyfp = tdir.getHostKeyPath('localhost') + localcertfp = tdir.getHostCertPath('localhost') + pkeypath = shutil.copyfile(localkeyfp, s_common.genpath(dirn, 'sslkey.pem')) + certpath = shutil.copyfile(localcertfp, s_common.genpath(dirn, 'sslcert.pem')) + + tlscadir = s_common.gendir(dirn, 'cadir') + cacertpath = shutil.copyfile(os.path.join(cadir, 'somelocalca.crt'), os.path.join(tlscadir, 'somelocalca.crt')) + + pkey, cert = tdir.genUserCert('someuser', signas='somelocalca') + user_pkey = tdir._pkeyToByts(pkey).decode() + user_cert = tdir._certToByts(cert).decode() + + user_fullchain = user_cert + s_common.getbytes(cacertpath).decode() + user_fullchain_key = user_fullchain + user_pkey + + conf = {'tls:ca:dir': tlscadir} + async with self.getTestCore(dirn=dirn, conf=conf) as core: + + sslctx = core.initSslCtx(certpath, pkeypath) + sslctx.load_verify_locations(cafile=cacertpath) + + addr, port = await core.addHttpsPort(0, sslctx=sslctx) + root = await core.auth.getUserByName('root') + await root.setPasswd('root') + + core.addHttpApi('/api/v0/test', s_test.HttpReflector, {'cell': core}) + core.addHttpApi('/test/ws', TstWebSock, {}) + + sslopts = {} + + opts = { + 'vars': { + 'url': f'https://root:root@localhost:{port}/api/v0/test', + 'ws': f'https://localhost:{port}/test/ws', + 'verify': True, + 'sslopts': sslopts, + }, + } + + q = 'return($lib.inet.http.get($url, ssl_verify=$verify, ssl_opts=$sslopts))' + + size, sha256 = await core.callStorm('return($lib.bytes.put($lib.base64.decode(Zm9v)))') + opts['vars']['sha256'] = sha256 + + # mtls required + + sslctx.verify_mode = ssl.CERT_REQUIRED + + ## no client cert provided + resp = await core.callStorm(q, opts=opts) + self.eq(-1, resp['code']) + self.isin('tlsv13 alert certificate required', resp['reason']) + + ## full chain cert w/key + sslopts['client_cert'] = user_fullchain_key + resp = await core.callStorm(q, opts=opts) + self.eq(200, resp['code']) + + ## separate cert and key + sslopts['client_cert'] = user_fullchain + sslopts['client_key'] = user_pkey + resp = await core.callStorm(q, opts=opts) + self.eq(200, resp['code']) + + ## sslctx's are cached + self.len(3, core._sslctx_cache) + resp = await core.callStorm(q, opts=opts) + self.eq(200, resp['code']) + self.len(3, core._sslctx_cache) + + ## remaining methods + self.eq(200, await core.callStorm('return($lib.inet.http.post($url, ssl_opts=$sslopts).code)', opts=opts)) + self.eq(200, await core.callStorm('return($lib.inet.http.head($url, ssl_opts=$sslopts).code)', opts=opts)) + self.eq(200, await core.callStorm('return($lib.inet.http.request(get, $url, ssl_opts=$sslopts).code)', opts=opts)) + + ## connect + ret = await core.callStorm(''' + ($ok, $sock) = $lib.inet.http.connect($ws, ssl_opts=$sslopts) + if (not $ok) { return(($ok, $sock)) } + ($ok, $mesg) = $sock.rx() + return(($ok, $mesg)) + ''', opts=opts) + self.true(ret[0]) + self.eq('woot', ret[1]['hi']) + + # Axon APIs + + axon_queries = { + 'postfile': ''' + $fields = ([{"name": "file", "sha256": $sha256}]) + return($lib.inet.http.post($url, fields=$fields, ssl_opts=$sslopts).code) + ''', + 'wget': 'return($lib.axon.wget($url, ssl_opts=$sslopts).code)', + 'wput': 'return($lib.axon.wput($sha256, $url, method=POST, ssl_opts=$sslopts).code)', + 'urlfile': 'yield $lib.axon.urlfile($url, ssl_opts=$sslopts)', + } + + ## version check fails + try: + oldv = core.axoninfo['synapse']['version'] + core.axoninfo['synapse']['version'] = (2, 161, 0) + await self.asyncraises(s_exc.BadVersion, core.callStorm(axon_queries['postfile'], opts=opts)) + await self.asyncraises(s_exc.BadVersion, core.callStorm(axon_queries['wget'], opts=opts)) + await self.asyncraises(s_exc.BadVersion, core.callStorm(axon_queries['wput'], opts=opts)) + await self.asyncraises(s_exc.BadVersion, core.nodes(axon_queries['urlfile'], opts=opts)) + finally: + core.axoninfo['synapse']['version'] = oldv + + ## version check succeeds + # todo: setting the synapse version can be removed once ssl_opts is released + try: + oldv = core.axoninfo['synapse']['version'] + core.axoninfo['synapse']['version'] = (oldv[0], oldv[1] + 1, oldv[2]) + self.eq(200, await core.callStorm(axon_queries['postfile'], opts=opts)) + self.eq(200, await core.callStorm(axon_queries['wget'], opts=opts)) + self.eq(200, await core.callStorm(axon_queries['wput'], opts=opts)) + self.len(1, await core.nodes(axon_queries['urlfile'], opts=opts)) + finally: + core.axoninfo['synapse']['version'] = oldv + + # verify arg precedence + + core.conf.pop('tls:ca:dir') + core._sslctx_cache.clear() + + ## fail w/o ca + resp = await core.callStorm(q, opts=opts) + self.eq(-1, resp['code']) + self.isin('self-signed certificate', resp['reason']) + + ## verify arg wins + opts['vars']['verify'] = False + sslopts['verify'] = True + resp = await core.callStorm(q, opts=opts) + self.eq(200, resp['code']) + + # bad opts + + ## schema violation + sslopts['newp'] = 'wut' + await self.asyncraises(s_exc.SchemaViolation, core.callStorm(q, opts=opts)) + sslopts.pop('newp') + + ## missing key + sslopts['client_cert'] = user_fullchain + sslopts['client_key'] = None + await self.asyncraises(s_exc.BadArg, core.callStorm(q, opts=opts)) + + ## bad cert + sslopts['client_cert'] = 'not-gonna-work' + await self.asyncraises(s_exc.BadArg, core.callStorm(q, opts=opts))