initial economic model for tracking purchases (#1177)

* wip

* acct bill/payment

* tweaks for a purchase of multiple items

* more tweaks

* remove jupyter notebook fail on new form

* remove jupyter notebook fail on model changes

* model tweaks from discussion and more tests

* docs/speeeling tweaks

* limit MII and IIN value range

docs/synapse/userguides/storm_ref_model_introspect.ipynb

@@ -89,7 +89,7 @@
     "q = 'syn:type'\n",
     "print(q)\n",
     "# Execute the query and test\n",
-    "podes = await core.eval(q, num=231, cmdr=False)"
+    "podes = await core.eval(q, cmdr=False)"
    ]
   },
   {
@@ -104,7 +104,7 @@
     "q = 'syn:form'\n",
     "print(q)\n",
     "# Execute the query and test\n",
-    "podes = await core.eval(q, num=188, cmdr=False)"
+    "podes = await core.eval(q, cmdr=False)"
    ]
   },
   {
@@ -119,7 +119,7 @@
     "q = 'syn:prop'\n",
     "print(q)\n",
     "# Execute the query and test\n",
-    "podes = await core.eval(q, num=1389, cmdr=False)"
+    "podes = await core.eval(q, cmdr=False)"
    ]
   },
   {
@@ -141,7 +141,7 @@
     "q = 'syn:type:subof = str'\n",
     "print(q)\n",
     "# Execute the query and test\n",
-    "podes = await core.eval(q, num=37, cmdr=False)"
+    "podes = await core.eval(q, cmdr=False)"
    ]
   },
   {

synapse/lib/modules.py

@@ -20,4 +20,5 @@
     'synapse.models.gov.cn.GovCnModule',
     'synapse.models.gov.us.GovUsModule',
     'synapse.models.gov.intl.GovIntlModule',
+    'synapse.models.economic.EconModule',
 )

synapse/lib/types.py

@@ -1250,6 +1250,7 @@ def indxByPref(self, valu):
 
     def _normPyStr(self, valu):
 
+        info = {}
         norm = str(valu)
 
         if self.opts['lower']:
@@ -1267,10 +1268,16 @@ def _normPyStr(self, valu):
                                         mesg='Value not in enums')
 
         if self.regex is not None:
-            if self.regex.match(norm) is None:
+
+            match = self.regex.match(norm)
+            if match is None:
                 raise s_exc.BadTypeValu(name=self.name, valu=valu, mesg='regex does not match')
 
-        return norm, {}
+            subs = match.groupdict()
+            if subs:
+                info['subs'] = subs
+
+        return norm, info
 
 class Tag(StrBase):
 

synapse/models/economic.py

@@ -0,0 +1,131 @@
+import synapse.lib.module as s_module
+
+class EconModule(s_module.CoreModule):
+
+    def getModelDefs(self):
+        return (('econ', {
+
+            'types': (
+
+                ('econ:pay:cvv', ('str', {'regex': '^[0-9]{1,6}$'}), {
+                    'doc': 'A Card Verification Value (CVV).'}),
+
+                ('econ:pay:pin', ('str', {'regex': '^[0-9]{3,6}$'}), {
+                    'doc': 'A Personal Identification Number.'}),
+
+                ('econ:pay:mii', ('int', {'min': 0, 'max': 9}), {
+                    'doc': 'A Major Industry Identifier (MII).'}),
+
+                ('econ:pay:pan', ('str', {'regex': '^(?<iin>(?<mii>[0-9]{1})[0-9]{5})[0-9]{1,13}$'}), {
+                    'doc': 'A Primary Account Number (PAN) or card number.'}),
+
+                ('econ:pay:iin', ('int', {'min': 0, 'max': 999999}), {
+                    'doc': 'An Issuer Id Number (IIN).'}),
+
+                ('econ:pay:card', ('guid', {}), {
+                    'doc': 'A single payment card.'}),
+
+                ('econ:purchase', ('guid', {}), {
+                    'doc': 'A purchase event.'}),
+
+                ('econ:acquired', ('comp', {'fields': (('purchase', 'econ:purchase'), ('item', 'ndef'))}), {
+                    'doc': 'A relationship between a purchase event and a purchased item.'}),
+
+                ('econ:acct:payment', ('guid', {}), {
+                    'doc': 'A payment moving currency from one monetary instrument to another.'}),
+
+                # TODO currency / monetary units / crypto currency
+                # econ:acct:bill
+                # econ:goods econ:services
+
+                # econ:bank:us:aba:rtn ( ABA Routing Number )
+                # econ:bank:us:account = (econ:bank:us:aba:rtn, acct)
+                # econ:bank:swift:...
+            ),
+
+            'forms': (
+
+                ('econ:pay:iin', {}, (
+
+                    ('org', ('ou:org', {}), {
+                        'doc': 'The issuer organization.'}),
+
+                    ('name', ('str', {'lower': True}), {
+                        'doc': 'The registered name of the issuer.'}),
+                )),
+
+                ('econ:pay:card', {}, (
+
+                    ('pan', ('econ:pay:pan', {}), {
+                        'doc': 'The payment card number.'}),
+
+                    ('pan:mii', ('econ:pay:mii', {}), {
+                        'doc': 'The payment card MII.'}),
+
+                    ('pan:iin', ('econ:pay:iin', {}), {
+                        'doc': 'The payment card IIN.'}),
+
+                    ('name', ('ps:name', {}), {
+                        'doc': 'The name as it appears on the card.'}),
+
+                    ('expr', ('time', {}), {
+                        'doc': 'The expiration date for the card.'}),
+
+                    ('cvv', ('econ:pay:cvv', {}), {
+                        'doc': 'The Card Verification Value on the card.'}),
+
+                    ('pin', ('econ:pay:pin', {}), {
+                        'doc': 'The Personal Identification Number on the card.'}),
+                )),
+
+                ('econ:purchase', {}, (
+
+                    ('by:contact', ('ps:contact', {}), {
+                        'doc': 'The contact information used to make the purchase.'}),
+
+                    ('from:contact', ('ps:contact', {}), {
+                        'doc': 'The contact information used to sell the item.'}),
+
+                    ('time', ('time', {}), {
+                        'doc': 'The time of the purchase.'}),
+
+                    ('place', ('geo:place', {}), {
+                        'doc': 'The place where the purchase took place.'}),
+
+                    ('paid', ('bool', {}), {
+                        'doc': 'Set to True if the purchase has been paid in full.'}),
+
+                    ('paid:time', ('time', {}), {
+                        'doc': 'The point in time where the purchase was paid in full.'}),
+
+                    # TODO price
+                )),
+
+                ('econ:acquired', {}, (
+                    ('purchase', ('econ:purchase', {}), {
+                        'doc': 'The purchase event which acquired an item.'}),
+                    ('item', ('ndef', {}), {
+                        'doc': 'A reference to the item that was acquired.'}),
+                    ('item:form', ('str', {}), {
+                        'doc': 'The form of item purchased.'}),
+                )),
+
+                ('econ:acct:payment', {}, (
+
+                    ('from:pay:card', ('econ:pay:card', {}), {
+                        'doc': 'The payment card making the payment.'}),
+
+                    ('to:contact', ('ps:contact', {}), {
+                        'doc': 'Contact information for the person/org being paid.'}),
+
+                    ('from:contact', ('ps:contact', {}), {
+                        'doc': 'Contact information for the person/org being paid.'}),
+
+                    ('time', ('time', {}), {
+                        'doc': 'The time the payment was processed.'}),
+
+                    ('purchase', ('econ:purchase', {}), {
+                        'doc': 'The purchase which the payment was paying for.'}),
+                )),
+            ),
+        }),)

synapse/tests/test_model_economic.py

@@ -0,0 +1,74 @@
+import synapse.tests.utils as s_utils
+
+import synapse.common as s_common
+
+class EconTest(s_utils.SynTest):
+
+    async def test_model_econ(self):
+
+        async with self.getTestCore() as core:
+
+            # test card number 4024007150779444
+            card = (await core.nodes('[ econ:pay:card="*" :expr=201802 :name="Bob Smith" :cvv=123 :pin=1234 :pan=4024007150779444 ]'))[0]
+            self.eq('bob smith', card.get('name'))
+            self.eq(1517443200000, card.get('expr'))
+            self.eq('4024007150779444', card.get('pan'))
+            self.eq(4, card.get('pan:mii'))
+            self.eq(402400, card.get('pan:iin'))
+
+            place = s_common.guid()
+            bycont = s_common.guid()
+            fromcont = s_common.guid()
+
+            text = f'''[
+                econ:purchase="*"
+
+                    :by:contact={bycont}
+                    :from:contact={fromcont}
+
+                    :time=20180202
+                    :place={place}
+
+                    :paid=true
+                    :paid:time=20180202
+            ]'''
+
+            perc = (await core.nodes(text))[0]
+
+            self.eq(bycont, perc.get('by:contact'))
+            self.eq(fromcont, perc.get('from:contact'))
+
+            self.eq(True, perc.get('paid'))
+            self.eq(1517529600000, perc.get('paid:time'))
+
+            self.eq(1517529600000, perc.get('time'))
+            self.eq(place, perc.get('place'))
+
+            self.len(1, await core.nodes('econ:purchase -> geo:place'))
+            self.len(2, await core.nodes('econ:purchase -> ps:contact | uniq'))
+
+            acqu = (await core.nodes(f'[ econ:acquired=({perc.ndef[1]}, (inet:fqdn,vertex.link)) ]'))[0]
+            self.eq(perc.ndef[1], acqu.get('purchase'))
+
+            self.len(1, await core.nodes('econ:acquired:item:form=inet:fqdn'))
+
+            self.eq(('inet:fqdn', 'vertex.link'), acqu.get('item'))
+
+            text = f'''[
+                econ:acct:payment="*"
+
+                    :to:contact={bycont}
+                    :from:contact={fromcont}
+
+                    :from:pay:card={card.ndef[1]}
+
+                    :time=20180202
+                    :purchase={perc.ndef[1]}
+            ]'''
+            await core.nodes(text)
+
+            self.len(1, await core.nodes('econ:acct:payment +:time@=(2017,2019) +{-> econ:pay:card +:name="bob smith"}'))
+
+            self.len(1, await core.nodes('econ:acct:payment -> econ:purchase'))
+            self.len(1, await core.nodes('econ:acct:payment -> econ:pay:card'))
+            self.len(2, await core.nodes('econ:acct:payment -> ps:contact | uniq'))