From 217c70e840ea162d143ca8c281333cb3b3a17235 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 29 Apr 2019 13:20:08 +0000 Subject: [PATCH 1/5] Initial pode repr helpers --- synapse/cmds/cortex.py | 22 ++++++++-------------- synapse/lib/node.py | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/synapse/cmds/cortex.py b/synapse/cmds/cortex.py index b014c3fd75..ca3ef4adbf 100644 --- a/synapse/cmds/cortex.py +++ b/synapse/cmds/cortex.py @@ -295,23 +295,20 @@ def _onNode(self, mesg): node = mesg[1] opts = node[1].pop('_opts', {}) - formname = node[0][0] - - formvalu = node[1].get('repr') - if formvalu is None: - formvalu = str(node[0][1]) if opts.get('raw'): self.printf(repr(node)) return + formname, formvalu = s_node.reprNdef(node) + self.printf(f'{formname}={formvalu}') if not opts.get('hide-props'): - for name, valu in sorted(node[1]['props'].items()): + for name in sorted(node[1]['props'].keys()): - valu = node[1]['reprs'].get(name, valu) + valu = s_node.reprProp(node, name) if name[0] != '.': name = ':' + name @@ -322,14 +319,11 @@ def _onNode(self, mesg): for tag in sorted(s_node.tags(node, leaf=True)): - valu = node[1]['tags'].get(tag) - if valu == (None, None): + valu = s_node.reprTag(node, tag) + if valu: + self.printf(f' #{tag} = {valu}') + else: self.printf(f' #{tag}') - continue - - mint = s_time.repr(valu[0]) - maxt = s_time.repr(valu[1]) - self.printf(f' #{tag} = ({mint}, {maxt})') def _onInit(self, mesg): pass diff --git a/synapse/lib/node.py b/synapse/lib/node.py index 5d08912e34..0427bf123c 100644 --- a/synapse/lib/node.py +++ b/synapse/lib/node.py @@ -630,7 +630,7 @@ def props(pode): ''' return pode[1]['props'].copy() -def prop(pode, prop): +def prop(pode, prop, use_repr=False): ''' Return the valu of a given property on the node. @@ -704,7 +704,7 @@ def tagged(pode, tag): def ndef(pode): ''' - Return a node definition (
, tuple from the node. + Return a node definition (,) tuple from the node. Args: node (tuple): A packed node. @@ -725,3 +725,31 @@ def iden(pode): str: The node iden. ''' return pode[1].get('iden') + +def reprNdef(pode): + ((form, valu), info) = pode + formvalu = info.get('repr') + if formvalu is None: + formvalu = str(valu) + return form, formvalu + +def reprProp(pode, prop): + opropvalu = pode[1].get('props').get(prop) + if opropvalu is None: + return None + propvalu = pode[1].get('reprs', {}).get(prop) + if propvalu is None: + return str(opropvalu) + return propvalu + +def reprTag(pode, tag): + tag = tag.lstrip('#') + valu = pode[1]['tags'].get(tag) + if valu is None: + return None + if valu == (None, None): + return '' + mint = s_time.repr(valu[0]) + maxt = s_time.repr(valu[1]) + valu = f'({mint}, {maxt})' + return valu From 405cd326e412062da1c6b41b2bf76df8cdcef52b Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 29 Apr 2019 17:44:35 +0000 Subject: [PATCH 2/5] wip --- synapse/cmds/cortex.py | 2 +- synapse/lib/node.py | 45 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/synapse/cmds/cortex.py b/synapse/cmds/cortex.py index ca3ef4adbf..ea1b0ed295 100644 --- a/synapse/cmds/cortex.py +++ b/synapse/cmds/cortex.py @@ -306,7 +306,7 @@ def _onNode(self, mesg): if not opts.get('hide-props'): - for name in sorted(node[1]['props'].keys()): + for name in sorted(s_node.props(node).keys()): valu = s_node.reprProp(node, name) diff --git a/synapse/lib/node.py b/synapse/lib/node.py index 0427bf123c..eddee000bd 100644 --- a/synapse/lib/node.py +++ b/synapse/lib/node.py @@ -707,7 +707,7 @@ def ndef(pode): Return a node definition (,) tuple from the node. Args: - node (tuple): A packed node. + pode (tuple): A packed node. Returns: ((str,obj)): The (,) tuple for the node @@ -727,6 +727,21 @@ def iden(pode): return pode[1].get('iden') def reprNdef(pode): + ''' + Get the ndef of the pode with a human readable value. + + Args: + pode (tuple): A packed node. + + Notes: + The human readable value is only available if the node came from + a storm query execution where the ``repr`` key was passed into + the ``opts`` argument with a True value. + + Returns: + (str, str): A tuple of form and the human readable value. + + ''' ((form, valu), info) = pode formvalu = info.get('repr') if formvalu is None: @@ -734,6 +749,20 @@ def reprNdef(pode): return form, formvalu def reprProp(pode, prop): + ''' + + Args: + pode (tuple): A packed node. + prop: + + Notes: + The human readable value is only available if the node came from + a storm query execution where the ``repr`` key was passed into + the ``opts`` argument with a True value. + + Returns: + str: The human readable property value. If the property is not present, returns None. + ''' opropvalu = pode[1].get('props').get(prop) if opropvalu is None: return None @@ -743,6 +772,20 @@ def reprProp(pode, prop): return propvalu def reprTag(pode, tag): + ''' + + Args: + pode (tuple): A packed node. + tag (str): The tag to get the value for. + + Notes: + The human readable value is only available if the node came from + a storm query execution where the ``repr`` key was passed into + the ``opts`` argument with a True value. + + Returns: + str: The human readable value for the tag. If the tag is not present, returns None. + ''' tag = tag.lstrip('#') valu = pode[1]['tags'].get(tag) if valu is None: From f51cfa6e03b388ef2cb54711c3de5c7347a00662 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 29 Apr 2019 18:29:55 +0000 Subject: [PATCH 3/5] Update tests / docstrings --- synapse/lib/node.py | 32 ++++++++++++++++++++++---------- synapse/tests/test_lib_node.py | 25 ++++++++++++++++++++++--- synapse/tests/utils.py | 1 + 3 files changed, 45 insertions(+), 13 deletions(-) diff --git a/synapse/lib/node.py b/synapse/lib/node.py index eddee000bd..ec78ba2e23 100644 --- a/synapse/lib/node.py +++ b/synapse/lib/node.py @@ -734,9 +734,9 @@ def reprNdef(pode): pode (tuple): A packed node. Notes: - The human readable value is only available if the node came from - a storm query execution where the ``repr`` key was passed into - the ``opts`` argument with a True value. + The human readable value is only available if the node came from a + storm query execution where the ``repr`` key was passed into the + ``opts`` argument with a True value. Returns: (str, str): A tuple of form and the human readable value. @@ -750,19 +750,28 @@ def reprNdef(pode): def reprProp(pode, prop): ''' + Get the human readable value for a secondary property from the pode. Args: pode (tuple): A packed node. prop: Notes: - The human readable value is only available if the node came from - a storm query execution where the ``repr`` key was passed into - the ``opts`` argument with a True value. + The human readable value is only available if the node came from a + storm query execution where the ``repr`` key was passed into the + ``opts`` argument with a True value. + + The prop argument may be the full property name (foo:bar:baz), relative + property name (:baz) , or the unadorned property name (baz). Returns: str: The human readable property value. If the property is not present, returns None. ''' + form = pode[0][0] + if prop.startswith(form): + prop = prop[len(form):] + if prop[0] == ':': + prop = prop[1:] opropvalu = pode[1].get('props').get(prop) if opropvalu is None: return None @@ -773,18 +782,21 @@ def reprProp(pode, prop): def reprTag(pode, tag): ''' + Get the human readable value for the tag timestamp from the pode. Args: pode (tuple): A packed node. tag (str): The tag to get the value for. Notes: - The human readable value is only available if the node came from - a storm query execution where the ``repr`` key was passed into - the ``opts`` argument with a True value. + The human readable value is only available if the node came from a + storm query execution where the ``repr`` key was passed into the + ``opts`` argument with a True value. + + If the tag does not have a timestamp, this returns a empty string. Returns: - str: The human readable value for the tag. If the tag is not present, returns None. + str: The human readable value for the tag. If the tag is not present, returns None. ''' tag = tag.lstrip('#') valu = pode[1]['tags'].get(tag) diff --git a/synapse/tests/test_lib_node.py b/synapse/tests/test_lib_node.py index 8496d8f0ab..423d5dea0f 100644 --- a/synapse/tests/test_lib_node.py +++ b/synapse/tests/test_lib_node.py @@ -173,7 +173,9 @@ async def test_tags(self): async def test_helpers(self): form = 'test:str' valu = 'cool' - props = {'tick': 12345} + props = {'tick': 12345, + 'hehe': 'hehe', + } tval = (None, None) async with self.getTestCore() as core: @@ -181,9 +183,16 @@ async def test_helpers(self): node = await snap.addNode(form, valu, props=props) await node.addTag('test.foo.bar.duck', tval) await node.addTag('test.foo.baz', tval) + await node.addTag('test.foo.time', ('2016', '2019')) pode = node.pack(dorepr=True) + node2 = await snap.addNode('test:int', '1234') + pode2 = node2.pack(dorepr=True) + self.eq(s_node.ndef(pode), ('test:str', 'cool')) + self.eq(s_node.reprNdef(pode), ('test:str', 'cool')) + self.eq(s_node.ndef(pode2), ('test:int', 1234)) + self.eq(s_node.reprNdef(pode2), ('test:int', '1234')) e = 'bf1198c5f28dae61d595434b0788dd6f7206b1e62d06b0798e012685f1abc85d' self.eq(s_node.iden(pode), e) @@ -193,14 +202,24 @@ async def test_helpers(self): self.true(s_node.tagged(pode, 'test.foo.bar.duck')) self.false(s_node.tagged(pode, 'test.foo.bar.newp')) - self.len(2, s_node.tags(pode, leaf=True)) - self.len(5, s_node.tags(pode)) + self.len(3, s_node.tags(pode, leaf=True)) + self.len(6, s_node.tags(pode)) + self.eq(s_node.reprTag(pode, '#test.foo.bar'), '') + self.eq(s_node.reprTag(pode, '#test.foo.time'), '(2016/01/01 00:00:00.000, 2019/01/01 00:00:00.000)') + self.none(s_node.reprTag(pode, 'test.foo.newp')) + self.eq(s_node.prop(pode, 'hehe'), 'hehe') self.eq(s_node.prop(pode, 'tick'), 12345) self.eq(s_node.prop(pode, ':tick'), 12345) self.eq(s_node.prop(pode, 'test:str:tick'), 12345) self.none(s_node.prop(pode, 'newp')) + self.eq(s_node.reprProp(pode, 'hehe'), 'hehe') + self.eq(s_node.reprProp(pode, 'tick'), '1970/01/01 00:00:12.345') + self.eq(s_node.reprProp(pode, ':tick'), '1970/01/01 00:00:12.345') + self.eq(s_node.reprProp(pode, 'test:str:tick'), '1970/01/01 00:00:12.345') + self.none(s_node.reprProp(pode, 'newp')) + props = s_node.props(pode) self.isin('.created', props) self.isin('tick', props) diff --git a/synapse/tests/utils.py b/synapse/tests/utils.py index 9c6a51254a..27669619be 100644 --- a/synapse/tests/utils.py +++ b/synapse/tests/utils.py @@ -223,6 +223,7 @@ def indx(self, norm): ('bar', ('ndef', {}), {}), ('baz', ('nodeprop', {}), {}), ('tick', ('test:time', {}), {}), + ('hehe', ('str', {}), {}), )), ('test:migr', {}, ( From 72a26abde27b0a7ca18743889f3c2cebad49597e Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 29 Apr 2019 18:32:34 +0000 Subject: [PATCH 4/5] Changelog --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1ab9823848..62368699b8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,7 @@ Features and Enhancements - Add POST support to the ``/api/v1/model/norm`` HTTP API endpoint. (`#1207 `_) - Add ``getPropNorm()`` and ``getTypeNorm()`` Telepath API endpoints to the Cortex and CoreApi. (`#1207 `_) - Add list ``length()`` and ``index()`` methods to Storm types. (`#1208 `_) +- Add helper functions to ``synapse.lib.node`` for extracting repr values from packed nodes. (`#1212 `_) Bugfixes -------- From 69b1f4ed36b36e860e1b9a7bce5a7b715dbdadcd Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 29 Apr 2019 18:38:53 +0000 Subject: [PATCH 5/5] Remove unused arg --- synapse/lib/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/lib/node.py b/synapse/lib/node.py index ec78ba2e23..1fc359f67f 100644 --- a/synapse/lib/node.py +++ b/synapse/lib/node.py @@ -630,7 +630,7 @@ def props(pode): ''' return pode[1]['props'].copy() -def prop(pode, prop, use_repr=False): +def prop(pode, prop): ''' Return the valu of a given property on the node.