Skip to content

Commit

Permalink
Reduce unit test memory usage (#2152)
Browse files Browse the repository at this point in the history
* Revert CI parallelism under the idea we have a handle on the unit test issue
* Add a decorator memoizemethod
  • Loading branch information
Nic Watson authored Apr 9, 2021
1 parent 1dae53e commit fca63a7
Show file tree
Hide file tree
Showing 9 changed files with 34 additions and 13 deletions.
4 changes: 2 additions & 2 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -511,7 +511,7 @@ jobs:
- test_steps_osx

python37:
parallelism: 16
parallelism: 8
docker:
- image: circleci/python:3.7.9
environment:
Expand All @@ -527,7 +527,7 @@ jobs:
- test_steps_python

python38:
parallelism: 16
parallelism: 8
docker:
- image: circleci/python:3.8
environment:
Expand Down
2 changes: 1 addition & 1 deletion synapse/cortex.py
Original file line number Diff line number Diff line change
Expand Up @@ -4163,7 +4163,7 @@ async def eval(self, text, opts=None):
async def stormlist(self, text, opts=None):
return [m async for m in self.storm(text, opts=opts)]

@s_cache.memoize(size=10000)
@s_cache.memoizemethod(size=10000)
def getStormQuery(self, text, mode='storm'):
'''
Parse storm query text and return a Query object.
Expand Down
1 change: 0 additions & 1 deletion synapse/lib/ast.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,6 @@ def format(self, depth=0):
yield item

def init(self, core):
self.core = core
[k.init(core) for k in self.kids]
self.prepare()

Expand Down
22 changes: 22 additions & 0 deletions synapse/lib/cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
A few speed optimized (lockless) cache helpers. Use carefully.
'''
import asyncio
import weakref
import functools
import collections

Expand All @@ -13,6 +14,27 @@
def memoize(size=16384):
return functools.lru_cache(maxsize=size)

# From https://stackoverflow.com/a/33672499/6518334
def memoizemethod(size=16384):
'''
A version of memoize that doesn't cause GC cycles when applied to a method.
'''
def decorator(func):
@functools.wraps(func)
def wrapped_func(self, *args, **kwargs):
# We're storing the wrapped method inside the instance. If we had
# a strong reference to self the instance would never die.
self_weak = weakref.ref(self)

@functools.wraps(func)
@functools.lru_cache(maxsize=size)
def cached_method(*args, **kwargs):
return func(self_weak(), *args, **kwargs)
setattr(self, func.__name__, cached_method)
return cached_method(*args, **kwargs)
return wrapped_func
return decorator

class FixedCache:

def __init__(self, callback, size=10000):
Expand Down
4 changes: 2 additions & 2 deletions synapse/lib/layer.py
Original file line number Diff line number Diff line change
Expand Up @@ -1490,14 +1490,14 @@ async def _onLayrFini(self):
async def getFormCounts(self):
return self.formcounts.pack()

@s_cache.memoize()
@s_cache.memoizemethod()
def getPropAbrv(self, form, prop):
return self.propabrv.bytsToAbrv(s_msgpack.en((form, prop)))

def setPropAbrv(self, form, prop):
return self.propabrv.setBytsToAbrv(s_msgpack.en((form, prop)))

@s_cache.memoize()
@s_cache.memoizemethod()
def getTagPropAbrv(self, *args):
return self.tagpropabrv.bytsToAbrv(s_msgpack.en(args))

Expand Down
4 changes: 2 additions & 2 deletions synapse/lib/lmdbslab.py
Original file line number Diff line number Diff line change
Expand Up @@ -165,15 +165,15 @@ def __init__(self, slab, name):
if item is not None:
self.offs = s_common.int64un(item[0]) + 1

@s_cache.memoize()
@s_cache.memoizemethod()
def abrvToByts(self, abrv):
byts = self.slab.get(abrv, db=self.abrv2name)
if byts is None:
raise s_exc.NoSuchAbrv

return byts

@s_cache.memoize()
@s_cache.memoizemethod()
def bytsToAbrv(self, byts):
abrv = self.slab.get(byts, db=self.name2abrv)
if abrv is None:
Expand Down
2 changes: 1 addition & 1 deletion synapse/lib/snap.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ def scrub(self, pode):

return pode

@s_cache.memoize()
@s_cache.memoizemethod()
def _isTagInc(self, tag):
if tag in self.inctags:
return True
Expand Down
6 changes: 3 additions & 3 deletions synapse/lib/stormlib/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -325,19 +325,19 @@ def getObjLocals(self):
'tagprop': self._methTagProp,
}

@s_cache.memoize(size=100)
@s_cache.memoizemethod(size=100)
async def _methType(self, name):
type_ = self.model.type(name)
if type_ is not None:
return ModelType(type_)

@s_cache.memoize(size=100)
@s_cache.memoizemethod(size=100)
async def _methProp(self, name):
prop = self.model.prop(name)
if prop is not None:
return ModelProp(prop)

@s_cache.memoize(size=100)
@s_cache.memoizemethod(size=100)
async def _methForm(self, name):
form = self.model.form(name)
if form is not None:
Expand Down
2 changes: 1 addition & 1 deletion synapse/lib/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -1212,7 +1212,7 @@ def _normPyStr(self, valu):
norm = '.'.join(norms)
return norm, {}

@s_cache.memoize()
@s_cache.memoizemethod()
def stems(self, valu):
norm, info = self.norm(valu)
parts = norm.split('.')
Expand Down

0 comments on commit fca63a7

Please sign in to comment.