diff --git a/include/sourmash.h b/include/sourmash.h index 72c9b39916..8f9bbe01fb 100644 --- a/include/sourmash.h +++ b/include/sourmash.h @@ -208,7 +208,7 @@ uintptr_t nodegraph_ntables(Nodegraph *ptr); void nodegraph_save(Nodegraph *ptr, const char *filename); -uint8_t *nodegraph_to_buffer(Nodegraph *ptr, uintptr_t *size); +uint8_t *nodegraph_to_buffer(Nodegraph *ptr, uint8_t compression, uintptr_t *size); void nodegraph_buffer_free(uint8_t *ptr, uintptr_t insize); @@ -267,7 +267,7 @@ Signature **signatures_load_path(const char *ptr, const char *select_moltype, uintptr_t *size); -SourmashStr signatures_save_buffer(Signature **ptr, uintptr_t size); +uint8_t *signatures_save_buffer(Signature **ptr, uintptr_t size, uint8_t compression, uintptr_t *osize); char sourmash_aa_to_dayhoff(char aa); diff --git a/setup.py b/setup.py index 6a9355ed95..c09c93079e 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,7 @@ def build_native(spec): 'sourmash = sourmash.__main__:main' ] }, - "install_requires": ["screed>=0.9", "cffi>=1.14.0", 'numpy', + "install_requires": ["screed>=0.9", "cffi>=1.14.0", "enum34", 'numpy', 'matplotlib', 'scipy', "deprecation>=2.0.6"], "setup_requires": [ "setuptools>=38.6.0", diff --git a/sourmash/_compat.py b/sourmash/_compat.py index 86b4e97f98..90f7afabf2 100644 --- a/sourmash/_compat.py +++ b/sourmash/_compat.py @@ -1,3 +1,4 @@ +import abc import sys @@ -14,6 +15,7 @@ def implements_to_string(cls): cls.__unicode__ = cls.__str__ cls.__str__ = lambda x: x.__unicode__().encode('utf-8') return cls + ABC = abc.ABCMeta(str('ABC'), (object,), {'__slots__': ()}) else: text_type = str int_types = (int,) @@ -22,3 +24,4 @@ def implements_to_string(cls): itervalues = lambda x: x.values() NUL = 0 implements_to_string = lambda x: x + from abc import ABC diff --git a/sourmash/commands.py b/sourmash/commands.py index 2f335351c1..7ca0d92813 100644 --- a/sourmash/commands.py +++ b/sourmash/commands.py @@ -359,7 +359,8 @@ def index(args): nums = set() scaleds = set() for f in inp_files: - notify('\r...reading from {} ({} signatures so far)', f, n, end='') + if n % 100 == 0: + notify('\r...reading from {} ({} signatures so far)', f, n, end='') siglist = sig.load_signatures(f, ksize=args.ksize, select_moltype=moltype) diff --git a/sourmash/exceptions.py b/sourmash/exceptions.py index 1a55ffbda6..6f73a59593 100644 --- a/sourmash/exceptions.py +++ b/sourmash/exceptions.py @@ -22,24 +22,47 @@ def __str__(self): return rv +class IndexNotSupported(SourmashError): + def __init__(self): + SourmashError.__init__(self, "This index format is not supported in this version of sourmash") + + +def _make_error(error_name, base=SourmashError, code=None): + class Exc(base): + pass + + Exc.__name__ = Exc.__qualname__ = error_name + if code is not None: + Exc.code = code + globals()[Exc.__name__] = Exc + __all__.append(Exc.__name__) + return Exc + + +def _get_error_base(error_name): + pieces = error_name.split("Error", 1) + if len(pieces) == 2 and pieces[0] and pieces[1]: + base_error_name = pieces[0] + "Error" + base_class = globals().get(base_error_name) + if base_class is None: + base_class = _make_error(base_error_name) + return base_class + return SourmashError + + def _make_exceptions(): for attr in dir(lib): if not attr.startswith('SOURMASH_ERROR_CODE_'): continue - class Exc(SourmashError): - pass - code = getattr(lib, attr) if code == 1104: exceptions_by_code[code] = ValueError elif code < 100 or code > 10000: - Exc.__name__ = attr[20:].title().replace('_', '') - Exc.code = getattr(lib, attr) - globals()[Exc.__name__] = Exc - Exc.code = code - exceptions_by_code[code] = Exc - __all__.append(Exc.__name__) + error_name = attr[20:].title().replace("_", "") + base = _get_error_base(error_name) + exc = _make_error(error_name, base=base, code=getattr(lib, attr)) + exceptions_by_code[exc.code] = exc else: exceptions_by_code[code] = ValueError diff --git a/sourmash/index.py b/sourmash/index.py index 3ebf136dc3..2a9eb8faef 100644 --- a/sourmash/index.py +++ b/sourmash/index.py @@ -1,11 +1,10 @@ "An Abstract Base Class for collections of signatures." from __future__ import division -from abc import ABCMeta, abstractmethod +from abc import abstractmethod from collections import namedtuple -# compatible with Python 2 *and* 3: -ABC = ABCMeta("ABC", (object,), {"__slots__": ()}) +from ._compat import ABC class Index(ABC): diff --git a/sourmash/nodegraph.py b/sourmash/nodegraph.py index 5c859b76b5..c865a3c7c3 100644 --- a/sourmash/nodegraph.py +++ b/sourmash/nodegraph.py @@ -31,9 +31,9 @@ def from_buffer(buf): def save(self, filename): self._methodcall(lib.nodegraph_save, to_bytes(filename)) - def to_bytes(self): + def to_bytes(self, compression=1): size = ffi.new("uintptr_t *") - rawbuf = self._methodcall(lib.nodegraph_to_buffer, size) + rawbuf = self._methodcall(lib.nodegraph_to_buffer, compression, size) size = size[0] rawbuf = ffi.gc(rawbuf, lambda o: lib.nodegraph_buffer_free(o, size), size) diff --git a/sourmash/sbt.py b/sourmash/sbt.py index 9f4aa2047d..32311b7e0e 100644 --- a/sourmash/sbt.py +++ b/sourmash/sbt.py @@ -55,10 +55,12 @@ def search_transcript(node, seq, threshold): import os from random import randint, random import sys +from tempfile import NamedTemporaryFile from deprecation import deprecated -from .sbt_storage import FSStorage, TarStorage, IPFSStorage, RedisStorage +from .exceptions import IndexNotSupported +from .sbt_storage import FSStorage, TarStorage, IPFSStorage, RedisStorage, ZipStorage from .logging import error, notify, debug from .index import Index from .nodegraph import Nodegraph, extract_nodegraph_info, calc_expected_collisions @@ -68,6 +70,7 @@ def search_transcript(node, seq, threshold): 'FSStorage': FSStorage, 'IPFSStorage': IPFSStorage, 'RedisStorage': RedisStorage, + 'ZipStorage': ZipStorage, } NodePos = namedtuple("NodePos", ["pos", "node"]) @@ -484,101 +487,44 @@ def save(self, path, storage=None, sparseness=0.0, structure_only=False): str full path to the new SBT description """ - version = 4 - - if path.endswith('.sbt.json'): - path = path[:-9] - fn = os.path.abspath(path + '.sbt.json') - - if storage is None: - # default storage - location = os.path.dirname(fn) - subdir = '.sbt.{}'.format(os.path.basename(path)) - - storage = FSStorage(location, subdir) - fn = os.path.join(location, fn) - - backend = [k for (k, v) in STORAGES.items() if v == type(storage)][0] - info = {} info['d'] = self.d - info['version'] = version - info['storage'] = { - 'backend': backend, - 'args': storage.init_args() - } - info['factory'] = { - 'class': GraphFactory.__name__, - 'args': self.factory.init_args() - } - - nodes = {} - total_nodes = len(self) - for n, (i, node) in enumerate(self): - if node is None: - continue - - if isinstance(node, Node): - if random() - sparseness <= 0: - continue - - data = { - # TODO: start using md5sum instead? - 'filename': os.path.basename(node.name), - 'name': node.name - } - - try: - node.metadata.pop('max_n_below') - except (AttributeError, KeyError): - pass - - data['metadata'] = node.metadata - - if structure_only is False: - # trigger data loading before saving to the new place - node.data - - node.storage = storage - - data['filename'] = node.save(data['filename']) - - node.storage = storage - data['filename'] = node.save(data['filename']) - nodes[i] = data - - notify("{} of {} nodes saved".format(n+1, total_nodes), end='\r') - - notify("\nFinished saving nodes, now saving SBT json file.") - info['nodes'] = nodes - with open(fn, 'w') as fp: - json.dump(info, fp) - - return fn - - def _save_v5(self, path, storage=None, sparseness=0.0, structure_only=False): - version = 5 - - if path.endswith('.sbt.json'): - path = path[:-9] - fn = os.path.abspath(path + '.sbt.json') + info['version'] = 6 + info["index_type"] = self.__class__.__name__ # TODO: check + + # choose between ZipStorage and FS (file system/directory) storage. + if path.endswith(".sbt.zip"): + kind = "Zip" + storage = ZipStorage(path) + backend = "FSStorage" + name = os.path.basename(path[:-8]) + subdir = '.sbt.{}'.format(name) + storage_args = FSStorage("", subdir).init_args() + storage.save(subdir + "/", b"") + index_filename = os.path.abspath(path) + else: + kind = "FS" + name = os.path.basename(path) + if path.endswith('.sbt.json'): + name = name[:-9] + index_filename = os.path.abspath(path) + else: + index_filename = os.path.abspath(path + '.sbt.json') - if storage is None: - # default storage - location = os.path.dirname(fn) - subdir = '.sbt.{}'.format(os.path.basename(path)) + if storage is None: + # default storage + location = os.path.dirname(index_filename) + subdir = '.sbt.{}'.format(name) - storage = FSStorage(location, subdir) - fn = os.path.join(location, fn) + storage = FSStorage(location, subdir) + index_filename = os.path.join(location, index_filename) - backend = [k for (k, v) in STORAGES.items() if v == type(storage)][0] + backend = [k for (k, v) in STORAGES.items() if v == type(storage)][0] + storage_args = storage.init_args() - info = {} - info['d'] = self.d - info['version'] = version info['storage'] = { 'backend': backend, - 'args': storage.init_args() + 'args': storage_args } info['factory'] = { 'class': GraphFactory.__name__, @@ -615,24 +561,37 @@ def _save_v5(self, path, storage=None, sparseness=0.0, structure_only=False): node.storage = storage - data['filename'] = node.save(data['filename']) + if kind == "Zip": + node.save(os.path.join(subdir, data['filename'])) + elif kind == "FS": + data['filename'] = node.save(data['filename']) - node.storage = storage - data['filename'] = node.save(data['filename']) if isinstance(node, Node): nodes[i] = data else: leaves[i] = data - notify("{} of {} nodes saved".format(n+1, total_nodes), end='\r') + if n % 100 == 0: + notify("{} of {} nodes saved".format(n+1, total_nodes), end='\r') notify("\nFinished saving nodes, now saving SBT json file.") info['nodes'] = nodes - info['leaves'] = leaves - with open(fn, 'w') as fp: - json.dump(info, fp) + info['signatures'] = leaves + + if kind == "Zip": + tree_data = json.dumps(info).encode("utf-8") + save_path = "{}.sbt.json".format(name) + storage.save(save_path, tree_data) + storage.close() + + elif kind == "FS": + with open(index_filename, 'w') as fp: + json.dump(info, fp) + + notify("\nFinished saving SBT, available at {0}\n".format(index_filename)) + + return path - return fn @classmethod def load(cls, location, leaf_loader=None, storage=None, print_version_warning=True): @@ -653,37 +612,77 @@ def load(cls, location, leaf_loader=None, storage=None, print_version_warning=Tr SBT the SBT tree built from the description. """ - dirname = os.path.dirname(os.path.abspath(location)) - sbt_name = os.path.basename(location) - if sbt_name.endswith('.sbt.json'): - sbt_name = sbt_name[:-9] + tempfile = None + sbt_name = None + tree_data = None - loaders = { - 1: cls._load_v1, - 2: cls._load_v2, - 3: cls._load_v3, - 4: cls._load_v4, - 5: cls._load_v5, - } + if storage is None and ZipStorage.can_open(location): + storage = ZipStorage(location) - if leaf_loader is None: - leaf_loader = Leaf.load + sbts = storage.list_sbts() + if len(sbts) != 1: + print("no SBT, or too many SBTs!") + else: + tree_data = storage.load(sbts[0]) + + tempfile = NamedTemporaryFile() + + tempfile.write(tree_data) + tempfile.flush() + + dirname = os.path.dirname(tempfile.name) + sbt_name = os.path.basename(tempfile.name) + + if sbt_name is None: + dirname = os.path.dirname(os.path.abspath(location)) + sbt_name = os.path.basename(location) + if sbt_name.endswith('.sbt.json'): + sbt_name = sbt_name[:-9] sbt_fn = os.path.join(dirname, sbt_name) - if not sbt_fn.endswith('.sbt.json'): + if not sbt_fn.endswith('.sbt.json') and tempfile is None: sbt_fn += '.sbt.json' with open(sbt_fn) as fp: jnodes = json.load(fp) + if tempfile is not None: + tempfile.close() + version = 1 if isinstance(jnodes, Mapping): version = jnodes['version'] + if leaf_loader is None: + leaf_loader = Leaf.load + + loaders = { + 1: cls._load_v1, + 2: cls._load_v2, + 3: cls._load_v3, + 4: cls._load_v4, + 5: cls._load_v5, + 6: cls._load_v6, + } + + try: + loader = loaders[version] + except KeyError: + raise IndexNotSupported() + + #if version >= 6: + # if jnodes.get("index_type", "SBT") == "LocalizedSBT": + # loaders[6] = LocalizedSBT._load_v6 + if version < 3 and storage is None: storage = FSStorage(dirname, '.sbt.{}'.format(sbt_name)) + elif storage is None: + klass = STORAGES[jnodes['storage']['backend']] + if jnodes['storage']['backend'] == "FSStorage": + storage = FSStorage(dirname, jnodes['storage']['args']['path']) + elif storage is None: + storage = klass(**jnodes['storage']['args']) - return loaders[version](jnodes, leaf_loader, dirname, storage, - print_version_warning) + return loader(jnodes, leaf_loader, dirname, storage, print_version_warning) @staticmethod def _load_v1(jnodes, leaf_loader, dirname, storage, print_version_warning=True): @@ -760,12 +759,6 @@ def _load_v3(cls, info, leaf_loader, dirname, storage, print_version_warning=Tru sbt_nodes = {} sbt_leaves = {} - klass = STORAGES[info['storage']['backend']] - if info['storage']['backend'] == "FSStorage": - storage = FSStorage(dirname, info['storage']['args']['path']) - elif storage is None: - storage = klass(**info['storage']['args']) - factory = GraphFactory(*info['factory']['args']) max_node = 0 @@ -807,12 +800,6 @@ def _load_v4(cls, info, leaf_loader, dirname, storage, print_version_warning=Tru sbt_nodes = {} sbt_leaves = {} - klass = STORAGES[info['storage']['backend']] - if info['storage']['backend'] == "FSStorage": - storage = FSStorage(dirname, info['storage']['args']['path']) - elif storage is None: - storage = klass(**info['storage']['args']) - factory = GraphFactory(*info['factory']['args']) max_node = 0 @@ -848,11 +835,53 @@ def _load_v5(cls, info, leaf_loader, dirname, storage, print_version_warning=Tru sbt_nodes = {} sbt_leaves = {} - klass = STORAGES[info['storage']['backend']] - if info['storage']['backend'] == "FSStorage": - storage = FSStorage(dirname, info['storage']['args']['path']) - elif storage is None: - storage = klass(**info['storage']['args']) + if storage is None: + klass = STORAGES[info['storage']['backend']] + if info['storage']['backend'] == "FSStorage": + storage = FSStorage(dirname, info['storage']['args']['path']) + elif storage is None: + storage = klass(**info['storage']['args']) + + factory = GraphFactory(*info['factory']['args']) + + max_node = 0 + for k, node in nodes.items(): + node['factory'] = factory + sbt_node = Node.load(node, storage) + + sbt_nodes[k] = sbt_node + max_node = max(max_node, k) + + for k, node in leaves.items(): + sbt_leaf = leaf_loader(node, storage) + sbt_leaves[k] = sbt_leaf + max_node = max(max_node, k) + + tree = cls(factory, d=info['d'], storage=storage) + tree._nodes = sbt_nodes + tree._leaves = sbt_leaves + tree._missing_nodes = {i for i in range(max_node) + if i not in sbt_nodes and i not in sbt_leaves} + + return tree + + @classmethod + def _load_v6(cls, info, leaf_loader, dirname, storage, print_version_warning=True): + nodes = {int(k): v for (k, v) in info['nodes'].items()} + leaves = {int(k): v for (k, v) in info['signatures'].items()} + + if not leaves: + raise ValueError("Empty tree!") + + sbt_nodes = {} + sbt_leaves = {} + + if storage is None: + klass = STORAGES[info['storage']['backend']] + if info['storage']['backend'] == "FSStorage": + storage = FSStorage(dirname, info['storage']['args']['path']) + elif storage is None: + storage = klass(**info['storage']['args']) factory = GraphFactory(*info['factory']['args']) @@ -1067,7 +1096,7 @@ def __str__(self): fpr=calc_expected_collisions(self.data, True, 1.1)) def save(self, path): - buf = self.data.to_bytes() + buf = self.data.to_bytes(compression=1) return self.storage.save(path, buf) @property @@ -1140,7 +1169,7 @@ def unload(self): self._data = None def save(self, path): - buf = self.data.to_bytes() + buf = self.data.to_bytes(compression=1) return self.storage.save(path, buf) def update(self, parent): diff --git a/sourmash/sbt_storage.py b/sourmash/sbt_storage.py index b2dd435934..e173b2747f 100644 --- a/sourmash/sbt_storage.py +++ b/sourmash/sbt_storage.py @@ -3,12 +3,16 @@ import abc from io import BytesIO import os +import shutil +import sys import tarfile +from tempfile import NamedTemporaryFile +import zipfile +from ._compat import ABC -class Storage(abc.ABCMeta(str('ABC'), (object,), {'__slots__': ()})): - # this weird baseclass is compatible with Python 2 *and* 3, - # we can remove once we support only py3.4+ + +class Storage(ABC): @abc.abstractmethod def save(self, path, content): @@ -25,8 +29,14 @@ def __enter__(self): return self def __exit__(self, type, value, traceback): + self.close() + + def close(self): pass + def can_open(self, location): + return False + class FSStorage(Storage): @@ -63,7 +73,7 @@ def __init__(self, path=None): if path is None: # TODO: Open a temporary file? - pass + pass # CTB: should raise an exception, no? self.path = os.path.abspath(path) @@ -97,6 +107,147 @@ def __exit__(self, type, value, traceback): self.tarfile.close() +class ZipStorage(Storage): + + def __init__(self, path): + self.path = os.path.abspath(path) + + dirname = os.path.dirname(self.path) + if not os.path.exists(dirname): + os.makedirs(dirname) + + self.bufferzip = None + + # Turns out we can't delete/modify an entry in a zipfile easily, + # so we need to check some things: + if not os.path.exists(self.path): + # If the file doesn't exist open it in write mode. + self.zipfile = zipfile.ZipFile(path, mode='w', + compression=zipfile.ZIP_STORED) + else: + # If it exists, open it in read mode and prepare a buffer for + # new/duplicated items. During close() there are checks to see + # how the original file needs to be updated (append new items, + # deal with duplicates, and so on) + self.zipfile = zipfile.ZipFile(path, 'r') + self.bufferzip = zipfile.ZipFile(BytesIO(), mode="w") + + self.subdir = None + subdirs = [f for f in self.zipfile.namelist() if f.endswith("/")] + if len(subdirs) == 1: + self.subdir = subdirs[0] + + def _save_to_zf(self, zf, path, content): + # we repeat these steps for self.zipfile and self.bufferzip, + # so better to have an auxiliary method + try: + info = zf.getinfo(path) + + with zf.open(info, mode='r') as entry: + if entry.read() == content: + # if new content == entry content, skip writing + return + else: + # Trying to write new content, raise error + raise ValueError("This will insert duplicated entries") + except KeyError: + # entry not there yet, write a new one + zf.writestr(path, content) + + def save(self, path, content): + # First try to save to self.zipfile, if it is not writable + # or would introduce duplicates then try to save it in the buffer + try: + self._save_to_zf(self.zipfile, path, content) + except (ValueError, RuntimeError): + # Can't write in the zipfile, write in buffer instead + if self.bufferzip: + self._save_to_zf(self.bufferzip, path, content) + else: + # Throw error, can't write the data + raise ValueError("can't write data") + + return path + + def _load_from_zf(self, zf, path): + # we repeat these steps for self.zipfile and self.bufferzip, + # so better to have an auxiliary method + try: + return zf.read(path) + except KeyError: + path = os.path.join(self.subdir, path) + return zf.read(path) + + def load(self, path): + try: + return self._load_from_zf(self.zipfile, path) + except KeyError: + return self._load_from_zf(self.bufferzip, path) + + def init_args(self): + return {'path': self.path} + + def close(self): + # This is a bit complicated, but we have to deal with new data + # (if the original zipfile is read-only) and possible duplicates. + + if self.bufferzip is None: + # The easy case: just close the zipfile, nothing else to do + self.zipfile.close() + else: + # The complicated one. Need to consider: + # - Is there data in the buffer? + # - If there is, is any of it + # * duplicated? + # * new data? + buffer_names = set(self.bufferzip.namelist()) + zf_names = set(self.zipfile.namelist()) + if buffer_names: + new_data = buffer_names - zf_names + duplicated = buffer_names.intersection(zf_names) + + if duplicated: + # bad news, need to create new file... + # create a temporary file to write the final version, + # which will be copied to the right place later. + tempfile = NamedTemporaryFile() + final_file = zipfile.ZipFile(tempfile, mode="w") + all_data = buffer_names.union(zf_names) + + for item in all_data: + if item in duplicated or item in buffer_names: + # we prioritize writing data from the buffer to the + # final file + final_file.writestr(item, self.bufferzip.read(item)) + else: + # it is only in the zipfile, so write from it + final_file.writestr(item, self.zipfile.read(item)) + + # close the files, remove the old one and copy the final + # file to the right place. + self.zipfile.close() + final_file.close() + os.unlink(self.path) + shutil.move(tempfile.name, self.path) + + elif new_data: + # Since there is no duplicated data, we can + # reopen self.zipfile in append mode and write the new data + self.zipfile.close() + zf = zipfile.ZipFile(self.path, mode='a') + for item in new_data: + zf.writestr(item, self.bufferzip.read(item)) + # finally, close the buffer and release memory + self.bufferzip.close() + + @staticmethod + def can_open(location): + return zipfile.is_zipfile(location) + + def list_sbts(self): + return [f for f in self.zipfile.namelist() if f.endswith(".sbt.json")] + + class IPFSStorage(Storage): def __init__(self, pin_on_add=True, **kwargs): @@ -106,7 +257,6 @@ def __init__(self, pin_on_add=True, **kwargs): self.api = ipfshttpclient.connect(**self.ipfs_args) def save(self, path, content): - # api.add_bytes(b"Mary had a little lamb") new_obj = self.api.add_bytes(content) if self.pin_on_add: self.api.pin.add(new_obj) @@ -117,6 +267,9 @@ def save(self, path, content): # like putting all the generated objects inside the same dir. # Check this call using the files API for an example. # api.files_write("/test/file", io.BytesIO(b"hi"), create=True) + # + # This is also required to bring the IPFSStorage closer to what the + # ZipStorage is doing now. def load(self, path): return self.api.cat(path) diff --git a/sourmash/sbtmh.py b/sourmash/sbtmh.py index bcd3983b6e..ed22021059 100644 --- a/sourmash/sbtmh.py +++ b/sourmash/sbtmh.py @@ -1,7 +1,7 @@ from __future__ import print_function from __future__ import division -from io import BytesIO, TextIOWrapper +from io import BytesIO import sys from .sbt import Leaf, SBT, GraphFactory @@ -47,11 +47,8 @@ def save(self, path): # content...) self.data - buf = BytesIO() - with TextIOWrapper(buf) as out: - signature.save_signatures([self.data], out) - out.flush() - return self.storage.save(path, buf.getvalue()) + buf = signature.save_signatures([self.data], compression=1) + return self.storage.save(path, buf) def update(self, parent): mh = self.data.minhash diff --git a/sourmash/signature.py b/sourmash/signature.py index 2205a7dfac..6244a05d94 100644 --- a/sourmash/signature.py +++ b/sourmash/signature.py @@ -7,17 +7,26 @@ import sys import os import weakref +from enum import Enum from .logging import error from . import MinHash from ._minhash import to_bytes from ._lowlevel import ffi, lib from .utils import RustObject, rustcall, decode_str +from ._compat import PY2 SIGNATURE_VERSION = 0.4 +class SigInput(Enum): + FILE_LIKE = 1 + PATH = 2 + BUFFER = 3 + UNKNOWN = 4 + + class SourmashSignature(RustObject): "Main class for signature information." @@ -185,6 +194,43 @@ def __reduce__(self): ) +def _detect_input_type(data): + """\ + Determine how to load input from `data`. Returns SigInput enum. + + Checks for: + - Python file-like objects + - JSON text (uncompressed sigs) + - Compressed memory buffers + - filename + """ + if hasattr(data, "fileno") or hasattr(data, "mode"): # file-like object + return SigInput.FILE_LIKE + elif hasattr(data, "find"): # check if it is uncompressed sig + try: + if data.find("sourmash_signature") > 0: + return SigInput.BUFFER + elif PY2: + try: + if data.startswith(b'\x1F\x8B'): # gzip compressed + return SigInput.BUFFER + except UnicodeDecodeError: + pass + except TypeError: + if data.find(b"sourmash_signature") > 0: + return SigInput.BUFFER + elif data.startswith(b'\x1F\x8B'): # gzip compressed + return SigInput.BUFFER + + try: + if os.path.exists(data): # filename + return SigInput.PATH + except (ValueError, TypeError): # No idea... + return SigInput.UNKNOWN + + return SigInput.UNKNOWN + + def load_signatures( data, ksize=None, select_moltype=None, ignore_md5sum=False, do_raise=False, quiet=False @@ -195,36 +241,14 @@ def load_signatures( Note, the order is not necessarily the same as what is in the source file. """ - if ksize: + if ksize is not None: ksize = int(ksize) + else: + ksize = 0 if not data: return - is_fp = False - is_filename = False - is_fobj = False - if hasattr(data, "fileno"): - is_fp = True - elif os.path.exists(data): # filename - is_filename = True - elif hasattr(data, "mode"): # file object-like - is_fobj = True - if "t" in data.mode: # need to reopen handler as binary - if sys.version_info >= (3,): - data = data.buffer - elif hasattr(data, "find") and data.find("sourmash_signature") > 0: - # json string containing the data - if hasattr(data, "encode"): - data = data.encode("utf-8") - else: - if do_raise: - raise ValueError("Can't parse data. No such file or invalid data.") - return - - if ksize is None: - ksize = 0 - if select_moltype is None: select_moltype = ffi.NULL else: @@ -233,11 +257,26 @@ def load_signatures( except AttributeError: pass + input_type = _detect_input_type(data) + if input_type == SigInput.UNKNOWN: + if not quiet: + error("Error in parsing signature; quitting. Cannot open file or invalid signature") + return + size = ffi.new("uintptr_t *") try: - # JSON format - if is_filename: + if input_type == SigInput.FILE_LIKE: + if hasattr(data, "mode") and "t" in data.mode: # need to reopen handler as binary + if sys.version_info >= (3,): + data = data.buffer + + buf = data.read() + data.close() + data = buf + input_type = SigInput.BUFFER + + elif input_type == SigInput.PATH: sigs_ptr = rustcall( lib.signatures_load_path, data.encode("utf-8"), @@ -246,24 +285,10 @@ def load_signatures( select_moltype, size, ) - else: - if is_fp or is_fobj: - # TODO: we still can't pass a file-like object to rust... - try: - buf = data.read() - is_fp = False - is_fobj = False - data.close() - data = buf - except AttributeError: - pass - if hasattr(data, "encode"): - data = data.encode("utf-8") - - # TODO: use ffi.cast in the future? - # fp_c = ffi.cast("FILE *", data) - # sigs_ptr = rustcall(lib.signatures_load_file, fp_c, ignore_md5sum, size) + if input_type == SigInput.BUFFER: + if hasattr(data, "encode") and not PY2: + data = data.encode("utf-8") sigs_ptr = rustcall( lib.signatures_load_buffer, @@ -275,7 +300,7 @@ def load_signatures( size, ) - size = ffi.unpack(size, 1)[0] + size = size[0] sigs = [] for i in range(size): @@ -291,9 +316,6 @@ def load_signatures( error("Exception: {}", str(e)) if do_raise: raise - finally: - if is_fp or is_fobj: - data.close() def load_one_signature(data, ksize=None, select_moltype=None, ignore_md5sum=False): @@ -314,9 +336,11 @@ def load_one_signature(data, ksize=None, select_moltype=None, ignore_md5sum=Fals raise ValueError("expected to load exactly one signature") -def save_signatures(siglist, fp=None): +def save_signatures(siglist, fp=None, compression=0): "Save multiple signatures into a JSON string (or into file handle 'fp')" attached_refs = weakref.WeakKeyDictionary() + + # get list of rust objects collected = [] for obj in siglist: rv = obj._get_objptr() @@ -324,16 +348,25 @@ def save_signatures(siglist, fp=None): collected.append(rv) siglist_c = ffi.new("Signature*[]", collected) - if fp is None: - buf = rustcall(lib.signatures_save_buffer, siglist_c, len(collected)) - return decode_str(buf, free=True) + size = ffi.new("uintptr_t *") + + # save signature into a string (potentially compressed) + rawbuf = rustcall(lib.signatures_save_buffer, siglist_c, len(collected), + compression, size) + size = size[0] + + # associate a finalizer with rawbuf so that it gets freed + buf = ffi.gc(rawbuf, lambda o: lib.nodegraph_buffer_free(o, size), size) + if compression: + result = ffi.buffer(buf, size)[:] else: - # fp_c = ffi.cast("FILE *", fp) - # buf = rustcall(lib.signatures_save_file, siglist_c, len(collected), fp_c) - buf = rustcall(lib.signatures_save_buffer, siglist_c, len(collected)) - result = decode_str(buf, free=True) - try: + result = ffi.string(buf, size) + + if fp is None: # return string + return result + else: + try: # write to file fp.write(result) except TypeError: - fp.write(result.encode('utf-8')) + fp.write(result.decode('utf-8')) return None diff --git a/src/core/benches/nodegraph.rs b/src/core/benches/nodegraph.rs index af7112dd1e..43e266c579 100644 --- a/src/core/benches/nodegraph.rs +++ b/src/core/benches/nodegraph.rs @@ -13,7 +13,8 @@ fn save_load(c: &mut Criterion) { let mut f = File::open("../../tests/test-data/.sbt.v3/internal.0").unwrap(); let _ = f.read_to_end(&mut data); - let mut group = c.benchmark_group("save_load"); + let mut group = c.benchmark_group("nodegraph"); + group.sample_size(10); let mut reader = Cursor::new(data.clone()); let ng = Nodegraph::from_reader(&mut reader).unwrap(); @@ -32,6 +33,20 @@ fn save_load(c: &mut Criterion) { ng.save_to_writer(&mut writer).unwrap(); }); }); + + group.bench_function("save compressed nodegraph", |b| { + b.iter(|| { + let mut buf = Vec::new(); + let mut writer = niffler::get_writer( + Box::new(&mut buf), + niffler::compression::Format::Gzip, + niffler::compression::Level::Five, + ) + .unwrap(); + + ng.save_to_writer(&mut writer).unwrap(); + }); + }); } criterion_group!(nodegraph, save_load); diff --git a/src/core/src/errors.rs b/src/core/src/errors.rs index 8e9ba04c7f..e4b5f8e26a 100644 --- a/src/core/src/errors.rs +++ b/src/core/src/errors.rs @@ -41,6 +41,9 @@ pub enum SourmashError { #[fail(display = "Error from deserialization")] SerdeError, + + #[fail(display = "I/O Error")] + IOError, } #[repr(u32)] @@ -102,6 +105,7 @@ impl SourmashErrorCode { SourmashErrorCode::InvalidHashFunction } SourmashError::SerdeError => SourmashErrorCode::SerdeError, + SourmashError::IOError => SourmashErrorCode::Io, }; } } diff --git a/src/core/src/ffi/nodegraph.rs b/src/core/src/ffi/nodegraph.rs index 233aae0350..969d06dbd6 100644 --- a/src/core/src/ffi/nodegraph.rs +++ b/src/core/src/ffi/nodegraph.rs @@ -240,16 +240,37 @@ unsafe fn nodegraph_save(ptr: *mut Nodegraph, filename: *const c_char) -> Result } ffi_fn! { -unsafe fn nodegraph_to_buffer(ptr: *mut Nodegraph, size: *mut usize) -> Result<*const u8> { +unsafe fn nodegraph_to_buffer(ptr: *mut Nodegraph, compression: u8, size: *mut usize) -> Result<*const u8> { let ng = { assert!(!ptr.is_null()); &mut *ptr }; - let mut st: Vec = Vec::new(); - ng.save_to_writer(&mut st)?; + let mut buffer = vec![]; + { + let mut writer = if compression > 0 { + let level = match compression { + 1 => niffler::compression::Level::One, + 2 => niffler::compression::Level::Two, + 3 => niffler::compression::Level::Three, + 4 => niffler::compression::Level::Four, + 5 => niffler::compression::Level::Five, + 6 => niffler::compression::Level::Six, + 7 => niffler::compression::Level::Seven, + 8 => niffler::compression::Level::Eight, + _ => niffler::compression::Level::Nine, + }; + + niffler::get_writer(Box::new(&mut buffer), + niffler::compression::Format::Gzip, + level)? + } else { + Box::new(&mut buffer) + }; + ng.save_to_writer(&mut writer)?; + } - let b = st.into_boxed_slice(); + let b = buffer.into_boxed_slice(); *size = b.len(); Ok(Box::into_raw(b) as *const u8) diff --git a/src/core/src/ffi/signature.rs b/src/core/src/ffi/signature.rs index 9ac5773281..0373bf6a90 100644 --- a/src/core/src/ffi/signature.rs +++ b/src/core/src/ffi/signature.rs @@ -4,9 +4,8 @@ use std::io; use std::os::raw::c_char; use std::slice; -use serde_json; - use crate::cmd::ComputeParameters; +use crate::errors::SourmashError; use crate::ffi::utils::SourmashStr; use crate::signature::Signature; use crate::sketch::minhash::{HashFunctions, KmerMinHash}; @@ -265,15 +264,42 @@ unsafe fn signature_get_mhs(ptr: *mut Signature, size: *mut usize) -> Result<*mu } ffi_fn! { -unsafe fn signatures_save_buffer(ptr: *mut *mut Signature, size: usize) -> Result { +unsafe fn signatures_save_buffer(ptr: *mut *mut Signature, size: usize, compression: u8, osize: *mut usize) -> Result<*const u8> { let sigs = { assert!(!ptr.is_null()); slice::from_raw_parts(ptr, size) }; let rsigs: Vec<&Signature> = sigs.iter().map(|x| x.as_ref().unwrap()).collect(); - let st = serde_json::to_string(&rsigs)?; - Ok(SourmashStr::from_string(st)) + + let mut buffer = vec![]; + { + let mut writer = if compression > 0 { + let level = match compression { + 1 => niffler::compression::Level::One, + 2 => niffler::compression::Level::Two, + 3 => niffler::compression::Level::Three, + 4 => niffler::compression::Level::Four, + 5 => niffler::compression::Level::Five, + 6 => niffler::compression::Level::Six, + 7 => niffler::compression::Level::Seven, + 8 => niffler::compression::Level::Eight, + _ => niffler::compression::Level::Nine, + }; + + niffler::get_writer(Box::new(&mut buffer), + niffler::compression::Format::Gzip, + level)? + } else { + Box::new(&mut buffer) + }; + serde_json::to_writer(&mut writer, &rsigs)?; + } + + let b = buffer.into_boxed_slice(); + *osize = b.len(); + + Ok(Box::into_raw(b) as *const u8) } } @@ -302,7 +328,7 @@ unsafe fn signatures_load_path(ptr: *const c_char, x => Some(x) }; - let (mut input, _) = niffler::from_path(buf.to_str()?)?; + let (mut input, _) = niffler::from_path(buf.to_str()?).map_err(|_| SourmashError::IOError)?; let filtered_sigs = Signature::load_signatures(&mut input, k, moltype, None)?; let ptr_sigs: Vec<*mut Signature> = filtered_sigs.into_iter().map(|x| { diff --git a/src/core/src/index/sbt/mod.rs b/src/core/src/index/sbt/mod.rs index 22ffab8453..e52f2d807a 100644 --- a/src/core/src/index/sbt/mod.rs +++ b/src/core/src/index/sbt/mod.rs @@ -154,6 +154,7 @@ where let mut st: FSStorage = match sinfo { SBTInfo::V4(ref sbt) => (&sbt.storage.args).into(), SBTInfo::V5(ref sbt) => (&sbt.storage.args).into(), + SBTInfo::V6(ref sbt) => (&sbt.storage.args).into(), }; st.set_base(path.as_ref().to_str().unwrap()); let storage: Rc = Rc::new(st); @@ -161,14 +162,49 @@ where let d = match sinfo { SBTInfo::V4(ref sbt) => sbt.d, SBTInfo::V5(ref sbt) => sbt.d, + SBTInfo::V6(ref sbt) => sbt.d, }; let factory = match sinfo { SBTInfo::V4(ref sbt) => sbt.factory.clone(), SBTInfo::V5(ref sbt) => sbt.factory.clone(), + SBTInfo::V6(ref sbt) => sbt.factory.clone(), }; let (nodes, leaves) = match sinfo { + SBTInfo::V6(sbt) => { + let nodes = sbt + .nodes + .into_iter() + .map(|(n, l)| { + ( + n, + Node::builder() + .filename(l.filename) + .name(l.name) + .metadata(l.metadata) + .storage(Some(Rc::clone(&storage))) + .build(), + ) + }) + .collect(); + let leaves = sbt + .signatures + .into_iter() + .map(|(n, l)| { + ( + n, + SigStore::builder() + .filename(l.filename) + .name(l.name) + .metadata(l.metadata) + .storage(Some(Rc::clone(&storage))) + .build(), + ) + }) + .collect(); + (nodes, leaves) + } SBTInfo::V5(sbt) => { let nodes = sbt .nodes @@ -633,9 +669,20 @@ struct SBTInfoV5 { leaves: HashMap, } +#[derive(Serialize, Deserialize)] +struct SBTInfoV6 { + d: u32, + version: u32, + storage: StorageInfo, + factory: Factory, + nodes: HashMap, + signatures: HashMap, +} + #[derive(Deserialize)] #[serde(untagged)] enum SBTInfo { + V6(SBTInfoV6), V5(SBTInfoV5), V4(SBTInfoV4), } diff --git a/src/core/src/signature.rs b/src/core/src/signature.rs index ed8d1bd09d..6c7429a9d2 100644 --- a/src/core/src/signature.rs +++ b/src/core/src/signature.rs @@ -161,6 +161,8 @@ impl Signature { where R: io::Read, { + let (rdr, _format) = niffler::get_reader(Box::new(rdr))?; + let sigs: Vec = serde_json::from_reader(rdr)?; Ok(sigs) } diff --git a/src/core/src/sketch/nodegraph.rs b/src/core/src/sketch/nodegraph.rs index b1c8a22161..d9a587c251 100644 --- a/src/core/src/sketch/nodegraph.rs +++ b/src/core/src/sketch/nodegraph.rs @@ -6,7 +6,6 @@ use std::slice; use byteorder::{BigEndian, ByteOrder, LittleEndian, ReadBytesExt, WriteBytesExt}; use failure::Error; use fixedbitset::FixedBitSet; -use primal_check; use crate::index::sbt::Update; use crate::sketch::minhash::KmerMinHash; diff --git a/tests/test-data/genome-s10+s11.sig.gz b/tests/test-data/genome-s10+s11.sig.gz new file mode 100644 index 0000000000..d0796f62ed Binary files /dev/null and b/tests/test-data/genome-s10+s11.sig.gz differ diff --git a/tests/test-data/subset.sbt.json b/tests/test-data/subset.sbt.json index c11fb9908f..9bcd7a9ef3 100644 --- a/tests/test-data/subset.sbt.json +++ b/tests/test-data/subset.sbt.json @@ -1 +1 @@ -{"storage": {"args": {"path": ".sbt.subset"}, "backend": "FSStorage"}, "version": 5, "d": 2, "nodes": {"0": {"name": "internal.0", "metadata": {"min_n_below": 314}, "filename": "internal.0"}, "1": {"name": "internal.1", "metadata": {"min_n_below": 314}, "filename": "internal.1"}, "2": {"name": "internal.2", "metadata": {"min_n_below": 870}, "filename": "internal.2"}, "3": {"name": "internal.3", "metadata": {"min_n_below": 314}, "filename": "internal.3"}, "4": {"name": "internal.4", "metadata": {"min_n_below": 652}, "filename": "internal.4"}, "5": {"name": "internal.5", "metadata": {"min_n_below": 870}, "filename": "internal.5"}, "6": {"name": "internal.6", "metadata": {"min_n_below": 970}, "filename": "internal.6"}, "7": {"name": "internal.7", "metadata": {"min_n_below": 522}, "filename": "internal.7"}, "8": {"name": "internal.8", "metadata": {"min_n_below": 314}, "filename": "internal.8"}, "9": {"name": "internal.9", "metadata": {"min_n_below": 1378}, "filename": "internal.9"}, "10": {"name": "internal.10", "metadata": {"min_n_below": 652}, "filename": "internal.10"}, "11": {"name": "internal.11", "metadata": {"min_n_below": 1385}, "filename": "internal.11"}, "12": {"name": "internal.12", "metadata": {"min_n_below": 870}, "filename": "internal.12"}, "13": {"name": "internal.13", "metadata": {"min_n_below": 1064}, "filename": "internal.13"}, "14": {"name": "internal.14", "metadata": {"min_n_below": 970}, "filename": "internal.14"}, "15": {"name": "internal.15", "metadata": {"min_n_below": 522}, "filename": "internal.15"}, "16": {"name": "internal.16", "metadata": {"min_n_below": 1032}, "filename": "internal.16"}, "17": {"name": "internal.17", "metadata": {"min_n_below": 314}, "filename": "internal.17"}, "18": {"name": "internal.18", "metadata": {"min_n_below": 424}, "filename": "internal.18"}, "19": {"name": "internal.19", "metadata": {"min_n_below": 1378}, "filename": "internal.19"}, "20": {"name": "internal.20", "metadata": {"min_n_below": 1509}, "filename": "internal.20"}, "21": {"name": "internal.21", "metadata": {"min_n_below": 993}, "filename": "internal.21"}, "22": {"name": "internal.22", "metadata": {"min_n_below": 652}, "filename": "internal.22"}, "23": {"name": "internal.23", "metadata": {"min_n_below": 1385}, "filename": "internal.23"}, "24": {"name": "internal.24", "metadata": {"min_n_below": 2159}, "filename": "internal.24"}, "25": {"name": "internal.25", "metadata": {"min_n_below": 870}, "filename": "internal.25"}, "26": {"name": "internal.26", "metadata": {"min_n_below": 1109}, "filename": "internal.26"}, "27": {"name": "internal.27", "metadata": {"min_n_below": 1166}, "filename": "internal.27"}, "28": {"name": "internal.28", "metadata": {"min_n_below": 1064}, "filename": "internal.28"}, "29": {"name": "internal.29", "metadata": {"min_n_below": 970}, "filename": "internal.29"}, "30": {"name": "internal.30", "metadata": {"min_n_below": 2215}, "filename": "internal.30"}, "31": {"name": "internal.31", "metadata": {"min_n_below": 522}, "filename": "internal.31"}, "32": {"name": "internal.32", "metadata": {"min_n_below": 1946}, "filename": "internal.32"}, "33": {"name": "internal.33", "metadata": {"min_n_below": 1158}, "filename": "internal.33"}, "34": {"name": "internal.34", "metadata": {"min_n_below": 1032}, "filename": "internal.34"}, "35": {"name": "internal.35", "metadata": {"min_n_below": 314}, "filename": "internal.35"}, "36": {"name": "internal.36", "metadata": {"min_n_below": 1396}, "filename": "internal.36"}, "37": {"name": "internal.37", "metadata": {"min_n_below": 424}, "filename": "internal.37"}, "38": {"name": "internal.38", "metadata": {"min_n_below": 1406}, "filename": "internal.38"}, "39": {"name": "internal.39", "metadata": {"min_n_below": 1378}, "filename": "internal.39"}, "40": {"name": "internal.40", "metadata": {"min_n_below": 1390}, "filename": "internal.40"}, "41": {"name": "internal.41", "metadata": {"min_n_below": 1509}, "filename": "internal.41"}, "42": {"name": "internal.42", "metadata": {"min_n_below": 1588}, "filename": "internal.42"}, "43": {"name": "internal.43", "metadata": {"min_n_below": 993}, "filename": "internal.43"}, "44": {"name": "internal.44", "metadata": {"min_n_below": 1027}, "filename": "internal.44"}, "45": {"name": "internal.45", "metadata": {"min_n_below": 652}, "filename": "internal.45"}, "46": {"name": "internal.46", "metadata": {"min_n_below": 1403}, "filename": "internal.46"}, "47": {"name": "internal.47", "metadata": {"min_n_below": 1422}, "filename": "internal.47"}, "48": {"name": "internal.48", "metadata": {"min_n_below": 1385}, "filename": "internal.48"}, "49": {"name": "internal.49", "metadata": {"min_n_below": 3270}, "filename": "internal.49"}, "50": {"name": "internal.50", "metadata": {"min_n_below": 2159}, "filename": "internal.50"}, "51": {"name": "internal.51", "metadata": {"min_n_below": 870}, "filename": "internal.51"}, "52": {"name": "internal.52", "metadata": {"min_n_below": 1766}, "filename": "internal.52"}, "53": {"name": "internal.53", "metadata": {"min_n_below": 1109}, "filename": "internal.53"}, "54": {"name": "internal.54", "metadata": {"min_n_below": 1482}, "filename": "internal.54"}, "55": {"name": "internal.55", "metadata": {"min_n_below": 1166}, "filename": "internal.55"}, "56": {"name": "internal.56", "metadata": {"min_n_below": 1422}, "filename": "internal.56"}, "57": {"name": "internal.57", "metadata": {"min_n_below": 1064}, "filename": "internal.57"}, "58": {"name": "internal.58", "metadata": {"min_n_below": 1081}, "filename": "internal.58"}, "59": {"name": "internal.59", "metadata": {"min_n_below": 970}, "filename": "internal.59"}, "60": {"name": "internal.60", "metadata": {"min_n_below": 1679}, "filename": "internal.60"}, "61": {"name": "internal.61", "metadata": {"min_n_below": 2215}, "filename": "internal.61"}, "62": {"name": "internal.62", "metadata": {"min_n_below": 2330}, "filename": "internal.62"}, "63": {"name": "internal.63", "metadata": {"min_n_below": 927}, "filename": "internal.63"}, "64": {"name": "internal.64", "metadata": {"min_n_below": 522}, "filename": "internal.64"}, "65": {"name": "internal.65", "metadata": {"min_n_below": 2159}, "filename": "internal.65"}, "66": {"name": "internal.66", "metadata": {"min_n_below": 1946}, "filename": "internal.66"}, "67": {"name": "internal.67", "metadata": {"min_n_below": 2427}, "filename": "internal.67"}, "68": {"name": "internal.68", "metadata": {"min_n_below": 1158}, "filename": "internal.68"}, "69": {"name": "internal.69", "metadata": {"min_n_below": 1032}, "filename": "internal.69"}, "70": {"name": "internal.70", "metadata": {"min_n_below": 1104}, "filename": "internal.70"}, "71": {"name": "internal.71", "metadata": {"min_n_below": 314}, "filename": "internal.71"}, "72": {"name": "internal.72", "metadata": {"min_n_below": 1410}, "filename": "internal.72"}, "73": {"name": "internal.73", "metadata": {"min_n_below": 1396}, "filename": "internal.73"}, "74": {"name": "internal.74", "metadata": {"min_n_below": 1618}, "filename": "internal.74"}, "75": {"name": "internal.75", "metadata": {"min_n_below": 1921}, "filename": "internal.75"}, "76": {"name": "internal.76", "metadata": {"min_n_below": 424}, "filename": "internal.76"}, "77": {"name": "internal.77", "metadata": {"min_n_below": 1406}, "filename": "internal.77"}, "78": {"name": "internal.78", "metadata": {"min_n_below": 2161}, "filename": "internal.78"}, "79": {"name": "internal.79", "metadata": {"min_n_below": 1378}, "filename": "internal.79"}, "80": {"name": "internal.80", "metadata": {"min_n_below": 3023}, "filename": "internal.80"}, "81": {"name": "internal.81", "metadata": {"min_n_below": 1428}, "filename": "internal.81"}, "82": {"name": "internal.82", "metadata": {"min_n_below": 1390}, "filename": "internal.82"}, "83": {"name": "internal.83", "metadata": {"min_n_below": 1509}, "filename": "internal.83"}, "84": {"name": "internal.84", "metadata": {"min_n_below": 2165}, "filename": "internal.84"}, "85": {"name": "internal.85", "metadata": {"min_n_below": 1588}, "filename": "internal.85"}, "86": {"name": "internal.86", "metadata": {"min_n_below": 1888}, "filename": "internal.86"}, "87": {"name": "internal.87", "metadata": {"min_n_below": 993}, "filename": "internal.87"}, "88": {"name": "internal.88", "metadata": {"min_n_below": 1002}, "filename": "internal.88"}, "89": {"name": "internal.89", "metadata": {"min_n_below": 1027}, "filename": "internal.89"}, "90": {"name": "internal.90", "metadata": {"min_n_below": 1037}, "filename": "internal.90"}, "91": {"name": "internal.91", "metadata": {"min_n_below": 1973}, "filename": "internal.91"}, "92": {"name": "internal.92", "metadata": {"min_n_below": 652}, "filename": "internal.92"}, "93": {"name": "internal.93", "metadata": {"min_n_below": 1403}, "filename": "internal.93"}, "94": {"name": "internal.94", "metadata": {"min_n_below": 2910}, "filename": "internal.94"}, "95": {"name": "internal.95", "metadata": {"min_n_below": 1433}, "filename": "internal.95"}, "96": {"name": "internal.96", "metadata": {"min_n_below": 1422}, "filename": "internal.96"}, "97": {"name": "internal.97", "metadata": {"min_n_below": 1395}, "filename": "internal.97"}, "98": {"name": "internal.98", "metadata": {"min_n_below": 1385}, "filename": "internal.98"}}, "factory": {"class": "GraphFactory", "args": [1, 100000, 4]}, "leaves": {"99": {"name": "a573351886e921b7204064508010fde0", "metadata": "a573351886e921b7204064508010fde0", "filename": "a573351886e921b7204064508010fde0"}, "100": {"name": "da939f28327bdd52576eec2c516a9ad3", "metadata": "da939f28327bdd52576eec2c516a9ad3", "filename": "da939f28327bdd52576eec2c516a9ad3"}, "101": {"name": "0418b8351b86bb41c8224d9d15474614", "metadata": "0418b8351b86bb41c8224d9d15474614", "filename": "0418b8351b86bb41c8224d9d15474614"}, "102": {"name": "978c4674f45437d4e84d0a0fc11c424b", "metadata": "978c4674f45437d4e84d0a0fc11c424b", "filename": "978c4674f45437d4e84d0a0fc11c424b"}, "103": {"name": "edbfe6ac404682ef420377507d52ca4d", "metadata": "edbfe6ac404682ef420377507d52ca4d", "filename": "edbfe6ac404682ef420377507d52ca4d"}, "104": {"name": "56b9e7bddb830ebe3faaf9923322b51e", "metadata": "56b9e7bddb830ebe3faaf9923322b51e", "filename": "56b9e7bddb830ebe3faaf9923322b51e"}, "105": {"name": "802dc6bf5787992180db6ca259313edf", "metadata": "802dc6bf5787992180db6ca259313edf", "filename": "802dc6bf5787992180db6ca259313edf"}, "106": {"name": "95fd78a08b85bfc006555fe7cbd7e3de", "metadata": "95fd78a08b85bfc006555fe7cbd7e3de", "filename": "95fd78a08b85bfc006555fe7cbd7e3de"}, "107": {"name": "81d3020367359d305b0e21933faece1b", "metadata": "81d3020367359d305b0e21933faece1b", "filename": "81d3020367359d305b0e21933faece1b"}, "108": {"name": "0f6508728e178731f3884f59dc7ff3c3", "metadata": "0f6508728e178731f3884f59dc7ff3c3", "filename": "0f6508728e178731f3884f59dc7ff3c3"}, "109": {"name": "857129e817ea9f95c22893d652c98a23", "metadata": "857129e817ea9f95c22893d652c98a23", "filename": "857129e817ea9f95c22893d652c98a23"}, "110": {"name": "43637330e78d1518b4d8f7603fdb6899", "metadata": "43637330e78d1518b4d8f7603fdb6899", "filename": "43637330e78d1518b4d8f7603fdb6899"}, "111": {"name": "849cf867719c1edecd4ec5cf2fb01a32", "metadata": "849cf867719c1edecd4ec5cf2fb01a32", "filename": "849cf867719c1edecd4ec5cf2fb01a32"}, "112": {"name": "f13c7b1cf4a5280dcc066d583c09bf78", "metadata": "f13c7b1cf4a5280dcc066d583c09bf78", "filename": "f13c7b1cf4a5280dcc066d583c09bf78"}, "113": {"name": "e2057ff9c72f6820163717d7ca69bf55", "metadata": "e2057ff9c72f6820163717d7ca69bf55", "filename": "e2057ff9c72f6820163717d7ca69bf55"}, "114": {"name": "d719ca7fca663697fd096fb757b89920", "metadata": "d719ca7fca663697fd096fb757b89920", "filename": "d719ca7fca663697fd096fb757b89920"}, "115": {"name": "5d19af7826508594c035efe76443138c", "metadata": "5d19af7826508594c035efe76443138c", "filename": "5d19af7826508594c035efe76443138c"}, "116": {"name": "e1595bc3bec4961c61a5304c1df226f0", "metadata": "e1595bc3bec4961c61a5304c1df226f0", "filename": "e1595bc3bec4961c61a5304c1df226f0"}, "117": {"name": "43fcce1eeeebc8d40162d5247202aef8", "metadata": "43fcce1eeeebc8d40162d5247202aef8", "filename": "43fcce1eeeebc8d40162d5247202aef8"}, "118": {"name": "09cc8e435e5570a5ba3b086bed8c831f", "metadata": "09cc8e435e5570a5ba3b086bed8c831f", "filename": "09cc8e435e5570a5ba3b086bed8c831f"}, "119": {"name": "1131a68ec746703c8c4a2bd13557bf6a", "metadata": "1131a68ec746703c8c4a2bd13557bf6a", "filename": "1131a68ec746703c8c4a2bd13557bf6a"}, "120": {"name": "9d3cbd4300d2ca17479b351d152ea677", "metadata": "9d3cbd4300d2ca17479b351d152ea677", "filename": "9d3cbd4300d2ca17479b351d152ea677"}, "121": {"name": "85ac23fb1585cbe148318b4947a702b7", "metadata": "85ac23fb1585cbe148318b4947a702b7", "filename": "85ac23fb1585cbe148318b4947a702b7"}, "122": {"name": "e0de0a27b7e73c2def0370febf06b399", "metadata": "e0de0a27b7e73c2def0370febf06b399", "filename": "e0de0a27b7e73c2def0370febf06b399"}, "123": {"name": "c787b5e7ee5160e73735755b872c0a41", "metadata": "c787b5e7ee5160e73735755b872c0a41", "filename": "c787b5e7ee5160e73735755b872c0a41"}, "124": {"name": "ecc266c69b70073c8b0f6682498d0675", "metadata": "ecc266c69b70073c8b0f6682498d0675", "filename": "ecc266c69b70073c8b0f6682498d0675"}, "125": {"name": "6381609c5cdcd210887f5add4bab44d9", "metadata": "6381609c5cdcd210887f5add4bab44d9", "filename": "6381609c5cdcd210887f5add4bab44d9"}, "126": {"name": "175e67b69e1833bdee0859a3a99495b1", "metadata": "175e67b69e1833bdee0859a3a99495b1", "filename": "175e67b69e1833bdee0859a3a99495b1"}, "127": {"name": "8f9573dec32c97fe824c06edaa16c979", "metadata": "8f9573dec32c97fe824c06edaa16c979", "filename": "8f9573dec32c97fe824c06edaa16c979"}, "128": {"name": "c4bcae8b1308ed48980134bb4d83f804", "metadata": "c4bcae8b1308ed48980134bb4d83f804", "filename": "c4bcae8b1308ed48980134bb4d83f804"}, "129": {"name": "9893ff8c4def73d5566baa3541b0a806", "metadata": "9893ff8c4def73d5566baa3541b0a806", "filename": "9893ff8c4def73d5566baa3541b0a806"}, "130": {"name": "de7c51531b29594960229029a8eb6bd3", "metadata": "de7c51531b29594960229029a8eb6bd3", "filename": "de7c51531b29594960229029a8eb6bd3"}, "131": {"name": "01bb2bac3849b82dee57a6ecf8725432", "metadata": "01bb2bac3849b82dee57a6ecf8725432", "filename": "01bb2bac3849b82dee57a6ecf8725432"}, "132": {"name": "736138545b801b99385439231a69ab77", "metadata": "736138545b801b99385439231a69ab77", "filename": "736138545b801b99385439231a69ab77"}, "133": {"name": "c2e8e26fb3377705376b1d6d434e1233", "metadata": "c2e8e26fb3377705376b1d6d434e1233", "filename": "c2e8e26fb3377705376b1d6d434e1233"}, "134": {"name": "a893a18ca62dd4e2bcb3c9aaf9c1957b", "metadata": "a893a18ca62dd4e2bcb3c9aaf9c1957b", "filename": "a893a18ca62dd4e2bcb3c9aaf9c1957b"}, "135": {"name": "b3068619013f7ce1bddaa4a67305a571", "metadata": "b3068619013f7ce1bddaa4a67305a571", "filename": "b3068619013f7ce1bddaa4a67305a571"}, "136": {"name": "7d859a1c0be4c97c9e1c6c8272950772", "metadata": "7d859a1c0be4c97c9e1c6c8272950772", "filename": "7d859a1c0be4c97c9e1c6c8272950772"}, "137": {"name": "3e18cd65bcc35fa150185fea0f162549", "metadata": "3e18cd65bcc35fa150185fea0f162549", "filename": "3e18cd65bcc35fa150185fea0f162549"}, "138": {"name": "7e22d796abc7a6c49dadb94c4dae16b3", "metadata": "7e22d796abc7a6c49dadb94c4dae16b3", "filename": "7e22d796abc7a6c49dadb94c4dae16b3"}, "139": {"name": "a20adff6c30627dc22e74d57d7f9b7db", "metadata": "a20adff6c30627dc22e74d57d7f9b7db", "filename": "a20adff6c30627dc22e74d57d7f9b7db"}, "140": {"name": "7574dd67e829221b9ae553692c2d5258", "metadata": "7574dd67e829221b9ae553692c2d5258", "filename": "7574dd67e829221b9ae553692c2d5258"}, "141": {"name": "b476935e55355147395b4d4ac5c7fc8d", "metadata": "b476935e55355147395b4d4ac5c7fc8d", "filename": "b476935e55355147395b4d4ac5c7fc8d"}, "142": {"name": "3988811e454e96213af488031d84eed9", "metadata": "3988811e454e96213af488031d84eed9", "filename": "3988811e454e96213af488031d84eed9"}, "143": {"name": "8963a3ad29f88d2b9869c51ac102ddf6", "metadata": "8963a3ad29f88d2b9869c51ac102ddf6", "filename": "8963a3ad29f88d2b9869c51ac102ddf6"}, "144": {"name": "8bdbd61dd2e99b7716968338bcfe9660", "metadata": "8bdbd61dd2e99b7716968338bcfe9660", "filename": "8bdbd61dd2e99b7716968338bcfe9660"}, "145": {"name": "4f1bf83d739fa4d420ceb0aae9403400", "metadata": "4f1bf83d739fa4d420ceb0aae9403400", "filename": "4f1bf83d739fa4d420ceb0aae9403400"}, "146": {"name": "8276440e3138e74a1c91b4d91f21072b", "metadata": "8276440e3138e74a1c91b4d91f21072b", "filename": "8276440e3138e74a1c91b4d91f21072b"}, "147": {"name": "205c4b9623e7331a907f293865925dfa", "metadata": "205c4b9623e7331a907f293865925dfa", "filename": "205c4b9623e7331a907f293865925dfa"}, "148": {"name": "b23715f6943b9a84bd64315d27cc8bd5", "metadata": "b23715f6943b9a84bd64315d27cc8bd5", "filename": "b23715f6943b9a84bd64315d27cc8bd5"}, "149": {"name": "a04ede5f60924719bfaabef59c1de821", "metadata": "a04ede5f60924719bfaabef59c1de821", "filename": "a04ede5f60924719bfaabef59c1de821"}, "150": {"name": "c6487fc895704808bfbd28461eb5406d", "metadata": "c6487fc895704808bfbd28461eb5406d", "filename": "c6487fc895704808bfbd28461eb5406d"}, "151": {"name": "2b28f083395d8f5e7b88c1899ff4a212", "metadata": "2b28f083395d8f5e7b88c1899ff4a212", "filename": "2b28f083395d8f5e7b88c1899ff4a212"}, "152": {"name": "2ac8b6220c4d44ae3dbbbfc7e939df61", "metadata": "2ac8b6220c4d44ae3dbbbfc7e939df61", "filename": "2ac8b6220c4d44ae3dbbbfc7e939df61"}, "153": {"name": "2d8b4d96cf9d790c1b225e681f8d57dd", "metadata": "2d8b4d96cf9d790c1b225e681f8d57dd", "filename": "2d8b4d96cf9d790c1b225e681f8d57dd"}, "154": {"name": "1bc3dc1d05e30383d4d098b7de944951", "metadata": "1bc3dc1d05e30383d4d098b7de944951", "filename": "1bc3dc1d05e30383d4d098b7de944951"}, "155": {"name": "f20e83bfa143ade475b9403b9e21641f", "metadata": "f20e83bfa143ade475b9403b9e21641f", "filename": "f20e83bfa143ade475b9403b9e21641f"}, "156": {"name": "46656e34e01f58e22f22a622f44fa658", "metadata": "46656e34e01f58e22f22a622f44fa658", "filename": "46656e34e01f58e22f22a622f44fa658"}, "157": {"name": "9f1f319740e92e9632ce9255f0c57114", "metadata": "9f1f319740e92e9632ce9255f0c57114", "filename": "9f1f319740e92e9632ce9255f0c57114"}, "158": {"name": "bd0ce2248788b643ddfb8276aea4b659", "metadata": "bd0ce2248788b643ddfb8276aea4b659", "filename": "bd0ce2248788b643ddfb8276aea4b659"}, "159": {"name": "c3569a0e65500c3fd92ae9b8c3ac617b", "metadata": "c3569a0e65500c3fd92ae9b8c3ac617b", "filename": "c3569a0e65500c3fd92ae9b8c3ac617b"}, "160": {"name": "5c07a864e3f86a705a73b89bd517979f", "metadata": "5c07a864e3f86a705a73b89bd517979f", "filename": "5c07a864e3f86a705a73b89bd517979f"}, "161": {"name": "57c3b9f54148e7ea9ebfdabbe530f131", "metadata": "57c3b9f54148e7ea9ebfdabbe530f131", "filename": "57c3b9f54148e7ea9ebfdabbe530f131"}, "162": {"name": "bf69634e838466bf0b83aa42a5fd1cd1", "metadata": "bf69634e838466bf0b83aa42a5fd1cd1", "filename": "bf69634e838466bf0b83aa42a5fd1cd1"}, "163": {"name": "f8f0617cdbe162dec828ac596feae35d", "metadata": "f8f0617cdbe162dec828ac596feae35d", "filename": "f8f0617cdbe162dec828ac596feae35d"}, "164": {"name": "5d813a4d317f9ad17154a99d296d081b", "metadata": "5d813a4d317f9ad17154a99d296d081b", "filename": "5d813a4d317f9ad17154a99d296d081b"}, "165": {"name": "66d40738bd608b83ef35b205a561eaa5", "metadata": "66d40738bd608b83ef35b205a561eaa5", "filename": "66d40738bd608b83ef35b205a561eaa5"}, "166": {"name": "004459575e3657bca8a3d0424545f082", "metadata": "004459575e3657bca8a3d0424545f082", "filename": "004459575e3657bca8a3d0424545f082"}, "167": {"name": "b1ffa01e0fa86ef9025003261eb181e3", "metadata": "b1ffa01e0fa86ef9025003261eb181e3", "filename": "b1ffa01e0fa86ef9025003261eb181e3"}, "168": {"name": "82e0290a438650db9a93ce29276f931b", "metadata": "82e0290a438650db9a93ce29276f931b", "filename": "82e0290a438650db9a93ce29276f931b"}, "169": {"name": "826776746a37223786ffec552e769c88", "metadata": "826776746a37223786ffec552e769c88", "filename": "826776746a37223786ffec552e769c88"}, "170": {"name": "1fb580f790c207278b55c408a68ff391", "metadata": "1fb580f790c207278b55c408a68ff391", "filename": "1fb580f790c207278b55c408a68ff391"}, "171": {"name": "ea077bf729e6510381278c68ee7c2b07", "metadata": "ea077bf729e6510381278c68ee7c2b07", "filename": "ea077bf729e6510381278c68ee7c2b07"}, "172": {"name": "e4521446df02458bd88840687af212c5", "metadata": "e4521446df02458bd88840687af212c5", "filename": "e4521446df02458bd88840687af212c5"}, "173": {"name": "20c1ca3c7ff0bb437afd69042bb5f852", "metadata": "20c1ca3c7ff0bb437afd69042bb5f852", "filename": "20c1ca3c7ff0bb437afd69042bb5f852"}, "174": {"name": "0d5e85e6ec8d82f2ae38ecb7f1394a04", "metadata": "0d5e85e6ec8d82f2ae38ecb7f1394a04", "filename": "0d5e85e6ec8d82f2ae38ecb7f1394a04"}, "175": {"name": "57256c01ec9b9980ecda5b97acb236c3", "metadata": "57256c01ec9b9980ecda5b97acb236c3", "filename": "57256c01ec9b9980ecda5b97acb236c3"}, "176": {"name": "fcc0a2c84b265a77211ff0d4bd4a413a", "metadata": "fcc0a2c84b265a77211ff0d4bd4a413a", "filename": "fcc0a2c84b265a77211ff0d4bd4a413a"}, "177": {"name": "a897a797fc00c21ec3ef5062b38cfb90", "metadata": "a897a797fc00c21ec3ef5062b38cfb90", "filename": "a897a797fc00c21ec3ef5062b38cfb90"}, "178": {"name": "4b57274be5a768732716b9dc600a1f14", "metadata": "4b57274be5a768732716b9dc600a1f14", "filename": "4b57274be5a768732716b9dc600a1f14"}, "179": {"name": "1f86e7c6c52baed6ce168b05f23013fc", "metadata": "1f86e7c6c52baed6ce168b05f23013fc", "filename": "1f86e7c6c52baed6ce168b05f23013fc"}, "180": {"name": "a448b639491a6f75649c2b1c960780d9", "metadata": "a448b639491a6f75649c2b1c960780d9", "filename": "a448b639491a6f75649c2b1c960780d9"}, "181": {"name": "3685ee5f820ab6e840ec6b0fe090e754", "metadata": "3685ee5f820ab6e840ec6b0fe090e754", "filename": "3685ee5f820ab6e840ec6b0fe090e754"}, "182": {"name": "972d83aa010954aaf5ad0a56f30f43b6", "metadata": "972d83aa010954aaf5ad0a56f30f43b6", "filename": "972d83aa010954aaf5ad0a56f30f43b6"}, "183": {"name": "38cf0b7d644963ee846734251e9c0175", "metadata": "38cf0b7d644963ee846734251e9c0175", "filename": "38cf0b7d644963ee846734251e9c0175"}, "184": {"name": "589cddfc349d8950af5bbd90f5db7060", "metadata": "589cddfc349d8950af5bbd90f5db7060", "filename": "589cddfc349d8950af5bbd90f5db7060"}, "185": {"name": "133743f147335b4d31b0e91480606339", "metadata": "133743f147335b4d31b0e91480606339", "filename": "133743f147335b4d31b0e91480606339"}, "186": {"name": "a532ca6bf98299efeb2915715b9a21c3", "metadata": "a532ca6bf98299efeb2915715b9a21c3", "filename": "a532ca6bf98299efeb2915715b9a21c3"}, "187": {"name": "633f4c15ae21310dac488c09d66c2d38", "metadata": "633f4c15ae21310dac488c09d66c2d38", "filename": "633f4c15ae21310dac488c09d66c2d38"}, "188": {"name": "0382590e3740e4c94455b4d52fff9143", "metadata": "0382590e3740e4c94455b4d52fff9143", "filename": "0382590e3740e4c94455b4d52fff9143"}, "189": {"name": "e5e9f136633ee06623a457e6c5bd9eac", "metadata": "e5e9f136633ee06623a457e6c5bd9eac", "filename": "e5e9f136633ee06623a457e6c5bd9eac"}, "190": {"name": "e6c06dd6393e96514f79db5d3bca100a", "metadata": "e6c06dd6393e96514f79db5d3bca100a", "filename": "e6c06dd6393e96514f79db5d3bca100a"}, "191": {"name": "97c19c413385bc830c6b2ddc855ac23a", "metadata": "97c19c413385bc830c6b2ddc855ac23a", "filename": "97c19c413385bc830c6b2ddc855ac23a"}, "192": {"name": "58459874f43a5d202d46018eb8f06e4c", "metadata": "58459874f43a5d202d46018eb8f06e4c", "filename": "58459874f43a5d202d46018eb8f06e4c"}, "193": {"name": "896f1c578904fad60f17061146946732", "metadata": "896f1c578904fad60f17061146946732", "filename": "896f1c578904fad60f17061146946732"}, "194": {"name": "8b5bef6feeb06603e70fde66df521edf", "metadata": "8b5bef6feeb06603e70fde66df521edf", "filename": "8b5bef6feeb06603e70fde66df521edf"}, "195": {"name": "c0c13a28a687b655688f8e10551e8df1", "metadata": "c0c13a28a687b655688f8e10551e8df1", "filename": "c0c13a28a687b655688f8e10551e8df1"}, "196": {"name": "91fea76b0361f4f812c07e7d43f434f3", "metadata": "91fea76b0361f4f812c07e7d43f434f3", "filename": "91fea76b0361f4f812c07e7d43f434f3"}, "197": {"name": "e32a1cc092043108ec5025c1098a1dc8", "metadata": "e32a1cc092043108ec5025c1098a1dc8", "filename": "e32a1cc092043108ec5025c1098a1dc8"}, "198": {"name": "7ddb71d74edc6d1e0c724a8f1bd3c8cc", "metadata": "7ddb71d74edc6d1e0c724a8f1bd3c8cc", "filename": "7ddb71d74edc6d1e0c724a8f1bd3c8cc"}}} \ No newline at end of file +{"storage": {"args": {"path": ".sbt.subset"}, "backend": "FSStorage"}, "version": 5, "d": 2, "nodes": {"0": {"name": "internal.0", "metadata": {"min_n_below": 314}, "filename": "internal.0"}, "1": {"name": "internal.1", "metadata": {"min_n_below": 314}, "filename": "internal.1"}, "2": {"name": "internal.2", "metadata": {"min_n_below": 870}, "filename": "internal.2"}, "3": {"name": "internal.3", "metadata": {"min_n_below": 314}, "filename": "internal.3"}, "4": {"name": "internal.4", "metadata": {"min_n_below": 652}, "filename": "internal.4"}, "5": {"name": "internal.5", "metadata": {"min_n_below": 870}, "filename": "internal.5"}, "6": {"name": "internal.6", "metadata": {"min_n_below": 970}, "filename": "internal.6"}, "7": {"name": "internal.7", "metadata": {"min_n_below": 522}, "filename": "internal.7"}, "8": {"name": "internal.8", "metadata": {"min_n_below": 314}, "filename": "internal.8"}, "9": {"name": "internal.9", "metadata": {"min_n_below": 1378}, "filename": "internal.9"}, "10": {"name": "internal.10", "metadata": {"min_n_below": 652}, "filename": "internal.10"}, "11": {"name": "internal.11", "metadata": {"min_n_below": 1385}, "filename": "internal.11"}, "12": {"name": "internal.12", "metadata": {"min_n_below": 870}, "filename": "internal.12"}, "13": {"name": "internal.13", "metadata": {"min_n_below": 1064}, "filename": "internal.13"}, "14": {"name": "internal.14", "metadata": {"min_n_below": 970}, "filename": "internal.14"}, "15": {"name": "internal.15", "metadata": {"min_n_below": 522}, "filename": "internal.15"}, "16": {"name": "internal.16", "metadata": {"min_n_below": 1032}, "filename": "internal.16"}, "17": {"name": "internal.17", "metadata": {"min_n_below": 314}, "filename": "internal.17"}, "18": {"name": "internal.18", "metadata": {"min_n_below": 424}, "filename": "internal.18"}, "19": {"name": "internal.19", "metadata": {"min_n_below": 1378}, "filename": "internal.19"}, "20": {"name": "internal.20", "metadata": {"min_n_below": 1509}, "filename": "internal.20"}, "21": {"name": "internal.21", "metadata": {"min_n_below": 993}, "filename": "internal.21"}, "22": {"name": "internal.22", "metadata": {"min_n_below": 652}, "filename": "internal.22"}, "23": {"name": "internal.23", "metadata": {"min_n_below": 1385}, "filename": "internal.23"}, "24": {"name": "internal.24", "metadata": {"min_n_below": 2159}, "filename": "internal.24"}, "25": {"name": "internal.25", "metadata": {"min_n_below": 870}, "filename": "internal.25"}, "26": {"name": "internal.26", "metadata": {"min_n_below": 1109}, "filename": "internal.26"}, "27": {"name": "internal.27", "metadata": {"min_n_below": 1166}, "filename": "internal.27"}, "28": {"name": "internal.28", "metadata": {"min_n_below": 1064}, "filename": "internal.28"}, "29": {"name": "internal.29", "metadata": {"min_n_below": 970}, "filename": "internal.29"}, "30": {"name": "internal.30", "metadata": {"min_n_below": 2215}, "filename": "internal.30"}, "31": {"name": "internal.31", "metadata": {"min_n_below": 522}, "filename": "internal.31"}, "32": {"name": "internal.32", "metadata": {"min_n_below": 1946}, "filename": "internal.32"}, "33": {"name": "internal.33", "metadata": {"min_n_below": 1158}, "filename": "internal.33"}, "34": {"name": "internal.34", "metadata": {"min_n_below": 1032}, "filename": "internal.34"}, "35": {"name": "internal.35", "metadata": {"min_n_below": 314}, "filename": "internal.35"}, "36": {"name": "internal.36", "metadata": {"min_n_below": 1396}, "filename": "internal.36"}, "37": {"name": "internal.37", "metadata": {"min_n_below": 424}, "filename": "internal.37"}, "38": {"name": "internal.38", "metadata": {"min_n_below": 1406}, "filename": "internal.38"}, "39": {"name": "internal.39", "metadata": {"min_n_below": 1378}, "filename": "internal.39"}, "40": {"name": "internal.40", "metadata": {"min_n_below": 1390}, "filename": "internal.40"}, "41": {"name": "internal.41", "metadata": {"min_n_below": 1509}, "filename": "internal.41"}, "42": {"name": "internal.42", "metadata": {"min_n_below": 1588}, "filename": "internal.42"}, "43": {"name": "internal.43", "metadata": {"min_n_below": 993}, "filename": "internal.43"}, "44": {"name": "internal.44", "metadata": {"min_n_below": 1027}, "filename": "internal.44"}, "45": {"name": "internal.45", "metadata": {"min_n_below": 652}, "filename": "internal.45"}, "46": {"name": "internal.46", "metadata": {"min_n_below": 1403}, "filename": "internal.46"}, "47": {"name": "internal.47", "metadata": {"min_n_below": 1422}, "filename": "internal.47"}, "48": {"name": "internal.48", "metadata": {"min_n_below": 1385}, "filename": "internal.48"}, "49": {"name": "internal.49", "metadata": {"min_n_below": 3270}, "filename": "internal.49"}, "50": {"name": "internal.50", "metadata": {"min_n_below": 2159}, "filename": "internal.50"}, "51": {"name": "internal.51", "metadata": {"min_n_below": 870}, "filename": "internal.51"}, "52": {"name": "internal.52", "metadata": {"min_n_below": 1766}, "filename": "internal.52"}, "53": {"name": "internal.53", "metadata": {"min_n_below": 1109}, "filename": "internal.53"}, "54": {"name": "internal.54", "metadata": {"min_n_below": 1482}, "filename": "internal.54"}, "55": {"name": "internal.55", "metadata": {"min_n_below": 1166}, "filename": "internal.55"}, "56": {"name": "internal.56", "metadata": {"min_n_below": 1422}, "filename": "internal.56"}, "57": {"name": "internal.57", "metadata": {"min_n_below": 1064}, "filename": "internal.57"}, "58": {"name": "internal.58", "metadata": {"min_n_below": 1081}, "filename": "internal.58"}, "59": {"name": "internal.59", "metadata": {"min_n_below": 970}, "filename": "internal.59"}, "60": {"name": "internal.60", "metadata": {"min_n_below": 1679}, "filename": "internal.60"}, "61": {"name": "internal.61", "metadata": {"min_n_below": 2215}, "filename": "internal.61"}, "62": {"name": "internal.62", "metadata": {"min_n_below": 2330}, "filename": "internal.62"}, "63": {"name": "internal.63", "metadata": {"min_n_below": 927}, "filename": "internal.63"}, "64": {"name": "internal.64", "metadata": {"min_n_below": 522}, "filename": "internal.64"}, "65": {"name": "internal.65", "metadata": {"min_n_below": 2159}, "filename": "internal.65"}, "66": {"name": "internal.66", "metadata": {"min_n_below": 1946}, "filename": "internal.66"}, "67": {"name": "internal.67", "metadata": {"min_n_below": 2427}, "filename": "internal.67"}, "68": {"name": "internal.68", "metadata": {"min_n_below": 1158}, "filename": "internal.68"}, "69": {"name": "internal.69", "metadata": {"min_n_below": 1032}, "filename": "internal.69"}, "70": {"name": "internal.70", "metadata": {"min_n_below": 1104}, "filename": "internal.70"}, "71": {"name": "internal.71", "metadata": {"min_n_below": 314}, "filename": "internal.71"}, "72": {"name": "internal.72", "metadata": {"min_n_below": 1410}, "filename": "internal.72"}, "73": {"name": "internal.73", "metadata": {"min_n_below": 1396}, "filename": "internal.73"}, "74": {"name": "internal.74", "metadata": {"min_n_below": 1618}, "filename": "internal.74"}, "75": {"name": "internal.75", "metadata": {"min_n_below": 1921}, "filename": "internal.75"}, "76": {"name": "internal.76", "metadata": {"min_n_below": 424}, "filename": "internal.76"}, "77": {"name": "internal.77", "metadata": {"min_n_below": 1406}, "filename": "internal.77"}, "78": {"name": "internal.78", "metadata": {"min_n_below": 2161}, "filename": "internal.78"}, "79": {"name": "internal.79", "metadata": {"min_n_below": 1378}, "filename": "internal.79"}, "80": {"name": "internal.80", "metadata": {"min_n_below": 3023}, "filename": "internal.80"}, "81": {"name": "internal.81", "metadata": {"min_n_below": 1428}, "filename": "internal.81"}, "82": {"name": "internal.82", "metadata": {"min_n_below": 1390}, "filename": "internal.82"}, "83": {"name": "internal.83", "metadata": {"min_n_below": 1509}, "filename": "internal.83"}, "84": {"name": "internal.84", "metadata": {"min_n_below": 2165}, "filename": "internal.84"}, "85": {"name": "internal.85", "metadata": {"min_n_below": 1588}, "filename": "internal.85"}, "86": {"name": "internal.86", "metadata": {"min_n_below": 1888}, "filename": "internal.86"}, "87": {"name": "internal.87", "metadata": {"min_n_below": 993}, "filename": "internal.87"}, "88": {"name": "internal.88", "metadata": {"min_n_below": 1002}, "filename": "internal.88"}, "89": {"name": "internal.89", "metadata": {"min_n_below": 1027}, "filename": "internal.89"}, "90": {"name": "internal.90", "metadata": {"min_n_below": 1037}, "filename": "internal.90"}, "91": {"name": "internal.91", "metadata": {"min_n_below": 1973}, "filename": "internal.91"}, "92": {"name": "internal.92", "metadata": {"min_n_below": 652}, "filename": "internal.92"}, "93": {"name": "internal.93", "metadata": {"min_n_below": 1403}, "filename": "internal.93"}, "94": {"name": "internal.94", "metadata": {"min_n_below": 2910}, "filename": "internal.94"}, "95": {"name": "internal.95", "metadata": {"min_n_below": 1433}, "filename": "internal.95"}, "96": {"name": "internal.96", "metadata": {"min_n_below": 1422}, "filename": "internal.96"}, "97": {"name": "internal.97", "metadata": {"min_n_below": 1395}, "filename": "internal.97"}, "98": {"name": "internal.98", "metadata": {"min_n_below": 1385}, "filename": "internal.98"}}, "factory": {"class": "GraphFactory", "args": [1, 100000, 4]}, "signatures": {"99": {"name": "a573351886e921b7204064508010fde0", "metadata": "a573351886e921b7204064508010fde0", "filename": "a573351886e921b7204064508010fde0"}, "100": {"name": "da939f28327bdd52576eec2c516a9ad3", "metadata": "da939f28327bdd52576eec2c516a9ad3", "filename": "da939f28327bdd52576eec2c516a9ad3"}, "101": {"name": "0418b8351b86bb41c8224d9d15474614", "metadata": "0418b8351b86bb41c8224d9d15474614", "filename": "0418b8351b86bb41c8224d9d15474614"}, "102": {"name": "978c4674f45437d4e84d0a0fc11c424b", "metadata": "978c4674f45437d4e84d0a0fc11c424b", "filename": "978c4674f45437d4e84d0a0fc11c424b"}, "103": {"name": "edbfe6ac404682ef420377507d52ca4d", "metadata": "edbfe6ac404682ef420377507d52ca4d", "filename": "edbfe6ac404682ef420377507d52ca4d"}, "104": {"name": "56b9e7bddb830ebe3faaf9923322b51e", "metadata": "56b9e7bddb830ebe3faaf9923322b51e", "filename": "56b9e7bddb830ebe3faaf9923322b51e"}, "105": {"name": "802dc6bf5787992180db6ca259313edf", "metadata": "802dc6bf5787992180db6ca259313edf", "filename": "802dc6bf5787992180db6ca259313edf"}, "106": {"name": "95fd78a08b85bfc006555fe7cbd7e3de", "metadata": "95fd78a08b85bfc006555fe7cbd7e3de", "filename": "95fd78a08b85bfc006555fe7cbd7e3de"}, "107": {"name": "81d3020367359d305b0e21933faece1b", "metadata": "81d3020367359d305b0e21933faece1b", "filename": "81d3020367359d305b0e21933faece1b"}, "108": {"name": "0f6508728e178731f3884f59dc7ff3c3", "metadata": "0f6508728e178731f3884f59dc7ff3c3", "filename": "0f6508728e178731f3884f59dc7ff3c3"}, "109": {"name": "857129e817ea9f95c22893d652c98a23", "metadata": "857129e817ea9f95c22893d652c98a23", "filename": "857129e817ea9f95c22893d652c98a23"}, "110": {"name": "43637330e78d1518b4d8f7603fdb6899", "metadata": "43637330e78d1518b4d8f7603fdb6899", "filename": "43637330e78d1518b4d8f7603fdb6899"}, "111": {"name": "849cf867719c1edecd4ec5cf2fb01a32", "metadata": "849cf867719c1edecd4ec5cf2fb01a32", "filename": "849cf867719c1edecd4ec5cf2fb01a32"}, "112": {"name": "f13c7b1cf4a5280dcc066d583c09bf78", "metadata": "f13c7b1cf4a5280dcc066d583c09bf78", "filename": "f13c7b1cf4a5280dcc066d583c09bf78"}, "113": {"name": "e2057ff9c72f6820163717d7ca69bf55", "metadata": "e2057ff9c72f6820163717d7ca69bf55", "filename": "e2057ff9c72f6820163717d7ca69bf55"}, "114": {"name": "d719ca7fca663697fd096fb757b89920", "metadata": "d719ca7fca663697fd096fb757b89920", "filename": "d719ca7fca663697fd096fb757b89920"}, "115": {"name": "5d19af7826508594c035efe76443138c", "metadata": "5d19af7826508594c035efe76443138c", "filename": "5d19af7826508594c035efe76443138c"}, "116": {"name": "e1595bc3bec4961c61a5304c1df226f0", "metadata": "e1595bc3bec4961c61a5304c1df226f0", "filename": "e1595bc3bec4961c61a5304c1df226f0"}, "117": {"name": "43fcce1eeeebc8d40162d5247202aef8", "metadata": "43fcce1eeeebc8d40162d5247202aef8", "filename": "43fcce1eeeebc8d40162d5247202aef8"}, "118": {"name": "09cc8e435e5570a5ba3b086bed8c831f", "metadata": "09cc8e435e5570a5ba3b086bed8c831f", "filename": "09cc8e435e5570a5ba3b086bed8c831f"}, "119": {"name": "1131a68ec746703c8c4a2bd13557bf6a", "metadata": "1131a68ec746703c8c4a2bd13557bf6a", "filename": "1131a68ec746703c8c4a2bd13557bf6a"}, "120": {"name": "9d3cbd4300d2ca17479b351d152ea677", "metadata": "9d3cbd4300d2ca17479b351d152ea677", "filename": "9d3cbd4300d2ca17479b351d152ea677"}, "121": {"name": "85ac23fb1585cbe148318b4947a702b7", "metadata": "85ac23fb1585cbe148318b4947a702b7", "filename": "85ac23fb1585cbe148318b4947a702b7"}, "122": {"name": "e0de0a27b7e73c2def0370febf06b399", "metadata": "e0de0a27b7e73c2def0370febf06b399", "filename": "e0de0a27b7e73c2def0370febf06b399"}, "123": {"name": "c787b5e7ee5160e73735755b872c0a41", "metadata": "c787b5e7ee5160e73735755b872c0a41", "filename": "c787b5e7ee5160e73735755b872c0a41"}, "124": {"name": "ecc266c69b70073c8b0f6682498d0675", "metadata": "ecc266c69b70073c8b0f6682498d0675", "filename": "ecc266c69b70073c8b0f6682498d0675"}, "125": {"name": "6381609c5cdcd210887f5add4bab44d9", "metadata": "6381609c5cdcd210887f5add4bab44d9", "filename": "6381609c5cdcd210887f5add4bab44d9"}, "126": {"name": "175e67b69e1833bdee0859a3a99495b1", "metadata": "175e67b69e1833bdee0859a3a99495b1", "filename": "175e67b69e1833bdee0859a3a99495b1"}, "127": {"name": "8f9573dec32c97fe824c06edaa16c979", "metadata": "8f9573dec32c97fe824c06edaa16c979", "filename": "8f9573dec32c97fe824c06edaa16c979"}, "128": {"name": "c4bcae8b1308ed48980134bb4d83f804", "metadata": "c4bcae8b1308ed48980134bb4d83f804", "filename": "c4bcae8b1308ed48980134bb4d83f804"}, "129": {"name": "9893ff8c4def73d5566baa3541b0a806", "metadata": "9893ff8c4def73d5566baa3541b0a806", "filename": "9893ff8c4def73d5566baa3541b0a806"}, "130": {"name": "de7c51531b29594960229029a8eb6bd3", "metadata": "de7c51531b29594960229029a8eb6bd3", "filename": "de7c51531b29594960229029a8eb6bd3"}, "131": {"name": "01bb2bac3849b82dee57a6ecf8725432", "metadata": "01bb2bac3849b82dee57a6ecf8725432", "filename": "01bb2bac3849b82dee57a6ecf8725432"}, "132": {"name": "736138545b801b99385439231a69ab77", "metadata": "736138545b801b99385439231a69ab77", "filename": "736138545b801b99385439231a69ab77"}, "133": {"name": "c2e8e26fb3377705376b1d6d434e1233", "metadata": "c2e8e26fb3377705376b1d6d434e1233", "filename": "c2e8e26fb3377705376b1d6d434e1233"}, "134": {"name": "a893a18ca62dd4e2bcb3c9aaf9c1957b", "metadata": "a893a18ca62dd4e2bcb3c9aaf9c1957b", "filename": "a893a18ca62dd4e2bcb3c9aaf9c1957b"}, "135": {"name": "b3068619013f7ce1bddaa4a67305a571", "metadata": "b3068619013f7ce1bddaa4a67305a571", "filename": "b3068619013f7ce1bddaa4a67305a571"}, "136": {"name": "7d859a1c0be4c97c9e1c6c8272950772", "metadata": "7d859a1c0be4c97c9e1c6c8272950772", "filename": "7d859a1c0be4c97c9e1c6c8272950772"}, "137": {"name": "3e18cd65bcc35fa150185fea0f162549", "metadata": "3e18cd65bcc35fa150185fea0f162549", "filename": "3e18cd65bcc35fa150185fea0f162549"}, "138": {"name": "7e22d796abc7a6c49dadb94c4dae16b3", "metadata": "7e22d796abc7a6c49dadb94c4dae16b3", "filename": "7e22d796abc7a6c49dadb94c4dae16b3"}, "139": {"name": "a20adff6c30627dc22e74d57d7f9b7db", "metadata": "a20adff6c30627dc22e74d57d7f9b7db", "filename": "a20adff6c30627dc22e74d57d7f9b7db"}, "140": {"name": "7574dd67e829221b9ae553692c2d5258", "metadata": "7574dd67e829221b9ae553692c2d5258", "filename": "7574dd67e829221b9ae553692c2d5258"}, "141": {"name": "b476935e55355147395b4d4ac5c7fc8d", "metadata": "b476935e55355147395b4d4ac5c7fc8d", "filename": "b476935e55355147395b4d4ac5c7fc8d"}, "142": {"name": "3988811e454e96213af488031d84eed9", "metadata": "3988811e454e96213af488031d84eed9", "filename": "3988811e454e96213af488031d84eed9"}, "143": {"name": "8963a3ad29f88d2b9869c51ac102ddf6", "metadata": "8963a3ad29f88d2b9869c51ac102ddf6", "filename": "8963a3ad29f88d2b9869c51ac102ddf6"}, "144": {"name": "8bdbd61dd2e99b7716968338bcfe9660", "metadata": "8bdbd61dd2e99b7716968338bcfe9660", "filename": "8bdbd61dd2e99b7716968338bcfe9660"}, "145": {"name": "4f1bf83d739fa4d420ceb0aae9403400", "metadata": "4f1bf83d739fa4d420ceb0aae9403400", "filename": "4f1bf83d739fa4d420ceb0aae9403400"}, "146": {"name": "8276440e3138e74a1c91b4d91f21072b", "metadata": "8276440e3138e74a1c91b4d91f21072b", "filename": "8276440e3138e74a1c91b4d91f21072b"}, "147": {"name": "205c4b9623e7331a907f293865925dfa", "metadata": "205c4b9623e7331a907f293865925dfa", "filename": "205c4b9623e7331a907f293865925dfa"}, "148": {"name": "b23715f6943b9a84bd64315d27cc8bd5", "metadata": "b23715f6943b9a84bd64315d27cc8bd5", "filename": "b23715f6943b9a84bd64315d27cc8bd5"}, "149": {"name": "a04ede5f60924719bfaabef59c1de821", "metadata": "a04ede5f60924719bfaabef59c1de821", "filename": "a04ede5f60924719bfaabef59c1de821"}, "150": {"name": "c6487fc895704808bfbd28461eb5406d", "metadata": "c6487fc895704808bfbd28461eb5406d", "filename": "c6487fc895704808bfbd28461eb5406d"}, "151": {"name": "2b28f083395d8f5e7b88c1899ff4a212", "metadata": "2b28f083395d8f5e7b88c1899ff4a212", "filename": "2b28f083395d8f5e7b88c1899ff4a212"}, "152": {"name": "2ac8b6220c4d44ae3dbbbfc7e939df61", "metadata": "2ac8b6220c4d44ae3dbbbfc7e939df61", "filename": "2ac8b6220c4d44ae3dbbbfc7e939df61"}, "153": {"name": "2d8b4d96cf9d790c1b225e681f8d57dd", "metadata": "2d8b4d96cf9d790c1b225e681f8d57dd", "filename": "2d8b4d96cf9d790c1b225e681f8d57dd"}, "154": {"name": "1bc3dc1d05e30383d4d098b7de944951", "metadata": "1bc3dc1d05e30383d4d098b7de944951", "filename": "1bc3dc1d05e30383d4d098b7de944951"}, "155": {"name": "f20e83bfa143ade475b9403b9e21641f", "metadata": "f20e83bfa143ade475b9403b9e21641f", "filename": "f20e83bfa143ade475b9403b9e21641f"}, "156": {"name": "46656e34e01f58e22f22a622f44fa658", "metadata": "46656e34e01f58e22f22a622f44fa658", "filename": "46656e34e01f58e22f22a622f44fa658"}, "157": {"name": "9f1f319740e92e9632ce9255f0c57114", "metadata": "9f1f319740e92e9632ce9255f0c57114", "filename": "9f1f319740e92e9632ce9255f0c57114"}, "158": {"name": "bd0ce2248788b643ddfb8276aea4b659", "metadata": "bd0ce2248788b643ddfb8276aea4b659", "filename": "bd0ce2248788b643ddfb8276aea4b659"}, "159": {"name": "c3569a0e65500c3fd92ae9b8c3ac617b", "metadata": "c3569a0e65500c3fd92ae9b8c3ac617b", "filename": "c3569a0e65500c3fd92ae9b8c3ac617b"}, "160": {"name": "5c07a864e3f86a705a73b89bd517979f", "metadata": "5c07a864e3f86a705a73b89bd517979f", "filename": "5c07a864e3f86a705a73b89bd517979f"}, "161": {"name": "57c3b9f54148e7ea9ebfdabbe530f131", "metadata": "57c3b9f54148e7ea9ebfdabbe530f131", "filename": "57c3b9f54148e7ea9ebfdabbe530f131"}, "162": {"name": "bf69634e838466bf0b83aa42a5fd1cd1", "metadata": "bf69634e838466bf0b83aa42a5fd1cd1", "filename": "bf69634e838466bf0b83aa42a5fd1cd1"}, "163": {"name": "f8f0617cdbe162dec828ac596feae35d", "metadata": "f8f0617cdbe162dec828ac596feae35d", "filename": "f8f0617cdbe162dec828ac596feae35d"}, "164": {"name": "5d813a4d317f9ad17154a99d296d081b", "metadata": "5d813a4d317f9ad17154a99d296d081b", "filename": "5d813a4d317f9ad17154a99d296d081b"}, "165": {"name": "66d40738bd608b83ef35b205a561eaa5", "metadata": "66d40738bd608b83ef35b205a561eaa5", "filename": "66d40738bd608b83ef35b205a561eaa5"}, "166": {"name": "004459575e3657bca8a3d0424545f082", "metadata": "004459575e3657bca8a3d0424545f082", "filename": "004459575e3657bca8a3d0424545f082"}, "167": {"name": "b1ffa01e0fa86ef9025003261eb181e3", "metadata": "b1ffa01e0fa86ef9025003261eb181e3", "filename": "b1ffa01e0fa86ef9025003261eb181e3"}, "168": {"name": "82e0290a438650db9a93ce29276f931b", "metadata": "82e0290a438650db9a93ce29276f931b", "filename": "82e0290a438650db9a93ce29276f931b"}, "169": {"name": "826776746a37223786ffec552e769c88", "metadata": "826776746a37223786ffec552e769c88", "filename": "826776746a37223786ffec552e769c88"}, "170": {"name": "1fb580f790c207278b55c408a68ff391", "metadata": "1fb580f790c207278b55c408a68ff391", "filename": "1fb580f790c207278b55c408a68ff391"}, "171": {"name": "ea077bf729e6510381278c68ee7c2b07", "metadata": "ea077bf729e6510381278c68ee7c2b07", "filename": "ea077bf729e6510381278c68ee7c2b07"}, "172": {"name": "e4521446df02458bd88840687af212c5", "metadata": "e4521446df02458bd88840687af212c5", "filename": "e4521446df02458bd88840687af212c5"}, "173": {"name": "20c1ca3c7ff0bb437afd69042bb5f852", "metadata": "20c1ca3c7ff0bb437afd69042bb5f852", "filename": "20c1ca3c7ff0bb437afd69042bb5f852"}, "174": {"name": "0d5e85e6ec8d82f2ae38ecb7f1394a04", "metadata": "0d5e85e6ec8d82f2ae38ecb7f1394a04", "filename": "0d5e85e6ec8d82f2ae38ecb7f1394a04"}, "175": {"name": "57256c01ec9b9980ecda5b97acb236c3", "metadata": "57256c01ec9b9980ecda5b97acb236c3", "filename": "57256c01ec9b9980ecda5b97acb236c3"}, "176": {"name": "fcc0a2c84b265a77211ff0d4bd4a413a", "metadata": "fcc0a2c84b265a77211ff0d4bd4a413a", "filename": "fcc0a2c84b265a77211ff0d4bd4a413a"}, "177": {"name": "a897a797fc00c21ec3ef5062b38cfb90", "metadata": "a897a797fc00c21ec3ef5062b38cfb90", "filename": "a897a797fc00c21ec3ef5062b38cfb90"}, "178": {"name": "4b57274be5a768732716b9dc600a1f14", "metadata": "4b57274be5a768732716b9dc600a1f14", "filename": "4b57274be5a768732716b9dc600a1f14"}, "179": {"name": "1f86e7c6c52baed6ce168b05f23013fc", "metadata": "1f86e7c6c52baed6ce168b05f23013fc", "filename": "1f86e7c6c52baed6ce168b05f23013fc"}, "180": {"name": "a448b639491a6f75649c2b1c960780d9", "metadata": "a448b639491a6f75649c2b1c960780d9", "filename": "a448b639491a6f75649c2b1c960780d9"}, "181": {"name": "3685ee5f820ab6e840ec6b0fe090e754", "metadata": "3685ee5f820ab6e840ec6b0fe090e754", "filename": "3685ee5f820ab6e840ec6b0fe090e754"}, "182": {"name": "972d83aa010954aaf5ad0a56f30f43b6", "metadata": "972d83aa010954aaf5ad0a56f30f43b6", "filename": "972d83aa010954aaf5ad0a56f30f43b6"}, "183": {"name": "38cf0b7d644963ee846734251e9c0175", "metadata": "38cf0b7d644963ee846734251e9c0175", "filename": "38cf0b7d644963ee846734251e9c0175"}, "184": {"name": "589cddfc349d8950af5bbd90f5db7060", "metadata": "589cddfc349d8950af5bbd90f5db7060", "filename": "589cddfc349d8950af5bbd90f5db7060"}, "185": {"name": "133743f147335b4d31b0e91480606339", "metadata": "133743f147335b4d31b0e91480606339", "filename": "133743f147335b4d31b0e91480606339"}, "186": {"name": "a532ca6bf98299efeb2915715b9a21c3", "metadata": "a532ca6bf98299efeb2915715b9a21c3", "filename": "a532ca6bf98299efeb2915715b9a21c3"}, "187": {"name": "633f4c15ae21310dac488c09d66c2d38", "metadata": "633f4c15ae21310dac488c09d66c2d38", "filename": "633f4c15ae21310dac488c09d66c2d38"}, "188": {"name": "0382590e3740e4c94455b4d52fff9143", "metadata": "0382590e3740e4c94455b4d52fff9143", "filename": "0382590e3740e4c94455b4d52fff9143"}, "189": {"name": "e5e9f136633ee06623a457e6c5bd9eac", "metadata": "e5e9f136633ee06623a457e6c5bd9eac", "filename": "e5e9f136633ee06623a457e6c5bd9eac"}, "190": {"name": "e6c06dd6393e96514f79db5d3bca100a", "metadata": "e6c06dd6393e96514f79db5d3bca100a", "filename": "e6c06dd6393e96514f79db5d3bca100a"}, "191": {"name": "97c19c413385bc830c6b2ddc855ac23a", "metadata": "97c19c413385bc830c6b2ddc855ac23a", "filename": "97c19c413385bc830c6b2ddc855ac23a"}, "192": {"name": "58459874f43a5d202d46018eb8f06e4c", "metadata": "58459874f43a5d202d46018eb8f06e4c", "filename": "58459874f43a5d202d46018eb8f06e4c"}, "193": {"name": "896f1c578904fad60f17061146946732", "metadata": "896f1c578904fad60f17061146946732", "filename": "896f1c578904fad60f17061146946732"}, "194": {"name": "8b5bef6feeb06603e70fde66df521edf", "metadata": "8b5bef6feeb06603e70fde66df521edf", "filename": "8b5bef6feeb06603e70fde66df521edf"}, "195": {"name": "c0c13a28a687b655688f8e10551e8df1", "metadata": "c0c13a28a687b655688f8e10551e8df1", "filename": "c0c13a28a687b655688f8e10551e8df1"}, "196": {"name": "91fea76b0361f4f812c07e7d43f434f3", "metadata": "91fea76b0361f4f812c07e7d43f434f3", "filename": "91fea76b0361f4f812c07e7d43f434f3"}, "197": {"name": "e32a1cc092043108ec5025c1098a1dc8", "metadata": "e32a1cc092043108ec5025c1098a1dc8", "filename": "e32a1cc092043108ec5025c1098a1dc8"}, "198": {"name": "7ddb71d74edc6d1e0c724a8f1bd3c8cc", "metadata": "7ddb71d74edc6d1e0c724a8f1bd3c8cc", "filename": "7ddb71d74edc6d1e0c724a8f1bd3c8cc"}}} diff --git a/tests/test-data/v6.sbt.json b/tests/test-data/v6.sbt.json new file mode 100644 index 0000000000..ca19d7248a --- /dev/null +++ b/tests/test-data/v6.sbt.json @@ -0,0 +1 @@ +{"d": 2, "version": 6, "index_type": "SBT", "storage": {"backend": "FSStorage", "args": {"path": ".sbt.v3"}}, "factory": {"class": "GraphFactory", "args": [1, 100000, 4]}, "nodes": {"0": {"filename": "internal.0", "name": "internal.0", "metadata": {"min_n_below": 500}}, "1": {"filename": "internal.1", "name": "internal.1", "metadata": {"min_n_below": 500}}, "2": {"filename": "internal.2", "name": "internal.2", "metadata": {"min_n_below": 500}}, "3": {"filename": "internal.3", "name": "internal.3", "metadata": {"min_n_below": 500}}, "4": {"filename": "internal.4", "name": "internal.4", "metadata": {"min_n_below": 500}}, "5": {"filename": "internal.5", "name": "internal.5", "metadata": {"min_n_below": 500}}}, "signatures": {"6": {"filename": "6d6e87e1154e95b279e5e7db414bc37b", "name": "6d6e87e1154e95b279e5e7db414bc37b", "metadata": "6d6e87e1154e95b279e5e7db414bc37b"}, "7": {"filename": "60f7e23c24a8d94791cc7a8680c493f9", "name": "60f7e23c24a8d94791cc7a8680c493f9", "metadata": "60f7e23c24a8d94791cc7a8680c493f9"}, "8": {"filename": "0107d767a345eff67ecdaed2ee5cd7ba", "name": "0107d767a345eff67ecdaed2ee5cd7ba", "metadata": "0107d767a345eff67ecdaed2ee5cd7ba"}, "9": {"filename": "f71e78178af9e45e6f1d87a0c53c465c", "name": "f71e78178af9e45e6f1d87a0c53c465c", "metadata": "f71e78178af9e45e6f1d87a0c53c465c"}, "10": {"filename": "f0c834bc306651d2b9321fb21d3e8d8f", "name": "f0c834bc306651d2b9321fb21d3e8d8f", "metadata": "f0c834bc306651d2b9321fb21d3e8d8f"}, "11": {"filename": "4e94e60265e04f0763142e20b52c0da1", "name": "4e94e60265e04f0763142e20b52c0da1", "metadata": "4e94e60265e04f0763142e20b52c0da1"}, "12": {"filename": "b59473c94ff2889eca5d7165936e64b3", "name": "b59473c94ff2889eca5d7165936e64b3", "metadata": "b59473c94ff2889eca5d7165936e64b3"}}} diff --git a/tests/test-data/v6.sbt.zip b/tests/test-data/v6.sbt.zip new file mode 100644 index 0000000000..5a616e190d Binary files /dev/null and b/tests/test-data/v6.sbt.zip differ diff --git a/tests/test_sbt.py b/tests/test_sbt.py index 10b317b7bc..d3b4c8a142 100644 --- a/tests/test_sbt.py +++ b/tests/test_sbt.py @@ -1,15 +1,18 @@ from __future__ import print_function, unicode_literals +import json +import shutil import os import pytest -from sourmash import load_one_signature, SourmashSignature +from sourmash import load_one_signature, SourmashSignature, load_signatures +from sourmash.exceptions import IndexNotSupported from sourmash.sbt import SBT, GraphFactory, Leaf, Node from sourmash.sbtmh import (SigLeaf, search_minhashes, search_minhashes_containment) -from sourmash.sbt_storage import (FSStorage, TarStorage, - RedisStorage, IPFSStorage) +from sourmash.sbt_storage import (FSStorage, TarStorage, RedisStorage, + IPFSStorage, ZipStorage) from . import sourmash_tst_utils as utils @@ -140,11 +143,12 @@ def search_transcript(node, seq, threshold): assert set(try3) == set([ 'd', 'e' ]), try3 -def test_tree_v1_load(): - tree_v1 = SBT.load(utils.get_test_data('v1.sbt.json'), +@pytest.mark.parametrize("old_version", ["v1", "v2", "v3", "v4", "v5"]) +def test_tree_old_load(old_version): + tree_v1 = SBT.load(utils.get_test_data('{}.sbt.json'.format(old_version)), leaf_loader=SigLeaf.load) - tree_cur = SBT.load(utils.get_test_data('v4.sbt.json'), + tree_cur = SBT.load(utils.get_test_data('v6.sbt.json'), leaf_loader=SigLeaf.load) testdata1 = utils.get_test_data(utils.SIG_FILES[0]) @@ -159,61 +163,14 @@ def test_tree_v1_load(): assert len(results_v1) == 4 -def test_tree_v2_load(): - tree_v2 = SBT.load(utils.get_test_data('v2.sbt.json'), - leaf_loader=SigLeaf.load) - - tree_cur = SBT.load(utils.get_test_data('v4.sbt.json'), - leaf_loader=SigLeaf.load) +def test_load_future(tmpdir): + with open(str(tmpdir.join("v9999.sbt.json")), 'w') as f: + json.dump({'version': 9999}, f) - testdata1 = utils.get_test_data(utils.SIG_FILES[0]) - to_search = load_one_signature(testdata1) + with pytest.raises(IndexNotSupported) as excinfo: + SBT.load(str(tmpdir.join("v9999.sbt.json"))) - results_v2 = {str(s) for s in tree_v2.find(search_minhashes_containment, - to_search, 0.1)} - results_cur = {str(s) for s in tree_cur.find(search_minhashes_containment, - to_search, 0.1)} - - assert results_v2 == results_cur - assert len(results_v2) == 4 - - -def test_tree_v3_load(): - tree_v2 = SBT.load(utils.get_test_data('v3.sbt.json'), - leaf_loader=SigLeaf.load) - - tree_cur = SBT.load(utils.get_test_data('v4.sbt.json'), - leaf_loader=SigLeaf.load) - - testdata1 = utils.get_test_data(utils.SIG_FILES[0]) - to_search = load_one_signature(testdata1) - - results_v2 = {str(s) for s in tree_v2.find(search_minhashes_containment, - to_search, 0.1)} - results_cur = {str(s) for s in tree_cur.find(search_minhashes_containment, - to_search, 0.1)} - - assert results_v2 == results_cur - assert len(results_v2) == 4 - - -def test_tree_v5_load(): - tree_v2 = SBT.load(utils.get_test_data('v5.sbt.json'), - leaf_loader=SigLeaf.load) - - tree_cur = SBT.load(utils.get_test_data('v4.sbt.json'), - leaf_loader=SigLeaf.load) - - testdata1 = utils.get_test_data(utils.SIG_FILES[0]) - to_search = load_one_signature(testdata1) - - results_v2 = {str(s) for s in tree_v2.find(search_minhashes_containment, - to_search, 0.1)} - results_cur = {str(s) for s in tree_cur.find(search_minhashes_containment, - to_search, 0.1)} - - assert results_v2 == results_cur - assert len(results_v2) == 4 + assert "index format is not supported" in str(excinfo.value) def test_tree_save_load(n_children): @@ -246,36 +203,6 @@ def test_tree_save_load(n_children): assert old_result == new_result -def test_tree_save_load_v5(n_children): - factory = GraphFactory(31, 1e5, 4) - tree = SBT(factory, d=n_children) - - for f in utils.SIG_FILES: - sig = load_one_signature(utils.get_test_data(f)) - leaf = SigLeaf(os.path.basename(f), sig) - tree.add_node(leaf) - to_search = leaf - - print('*' * 60) - print("{}:".format(to_search.metadata)) - old_result = {str(s) for s in tree.find(search_minhashes, - to_search.data, 0.1)} - print(*old_result, sep='\n') - - with utils.TempDirectory() as location: - tree._save_v5(os.path.join(location, 'demo')) - tree = SBT.load(os.path.join(location, 'demo'), - leaf_loader=SigLeaf.load) - - print('*' * 60) - print("{}:".format(to_search.metadata)) - new_result = {str(s) for s in tree.find(search_minhashes, - to_search.data, 0.1)} - print(*new_result, sep='\n') - - assert old_result == new_result - - def test_search_minhashes(): factory = GraphFactory(31, 1e5, 4) tree = SBT(factory) @@ -442,6 +369,41 @@ def test_sbt_tarstorage(): assert old_result == new_result +def test_sbt_zipstorage(tmpdir): + # create tree, save to a zip, then load and search. + factory = GraphFactory(31, 1e5, 4) + + tree = SBT(factory) + + for f in utils.SIG_FILES: + sig = next(load_signatures(utils.get_test_data(f))) + leaf = SigLeaf(os.path.basename(f), sig) + tree.add_node(leaf) + to_search = leaf + + print('*' * 60) + print("{}:".format(to_search.metadata)) + old_result = {str(s) for s in tree.find(search_minhashes, + to_search.data, 0.1)} + print(*old_result, sep='\n') + + with ZipStorage(str(tmpdir.join("tree.sbt.zip"))) as storage: + tree.save(str(tmpdir.join("tree")), storage=storage) + + with ZipStorage(str(tmpdir.join("tree.sbt.zip"))) as storage: + tree = SBT.load(str(tmpdir.join("tree")), + leaf_loader=SigLeaf.load, + storage=storage) + + print('*' * 60) + print("{}:".format(to_search.metadata)) + new_result = {str(s) for s in tree.find(search_minhashes, + to_search.data, 0.1)} + print(*new_result, sep='\n') + + assert old_result == new_result + + def test_sbt_ipfsstorage(): ipfshttpclient = pytest.importorskip('ipfshttpclient') @@ -521,6 +483,73 @@ def test_sbt_redisstorage(): assert old_result == new_result +def test_save_zip(tmpdir): + # load from zipped SBT, save to zipped SBT, and then search. + testdata = utils.get_test_data("v6.sbt.zip") + testsbt = tmpdir.join("v6.sbt.zip") + newsbt = tmpdir.join("new.sbt.zip") + + shutil.copyfile(testdata, str(testsbt)) + + tree = SBT.load(str(testsbt), leaf_loader=SigLeaf.load) + tree.save(str(newsbt)) + assert newsbt.exists() + + new_tree = SBT.load(str(newsbt), leaf_loader=SigLeaf.load) + assert isinstance(new_tree.storage, ZipStorage) + assert new_tree.storage.list_sbts() == ['new.sbt.json'] + + to_search = load_one_signature(utils.get_test_data(utils.SIG_FILES[0])) + + print("*" * 60) + print("{}:".format(to_search)) + old_result = {str(s) for s in tree.find(search_minhashes, to_search, 0.1)} + new_result = {str(s) for s in new_tree.find(search_minhashes, to_search, 0.1)} + print(*new_result, sep="\n") + + assert old_result == new_result + assert len(new_result) == 2 + + +def test_load_zip(tmpdir): + # search zipped SBT + testdata = utils.get_test_data("v6.sbt.zip") + testsbt = tmpdir.join("v6.sbt.zip") + + shutil.copyfile(testdata, str(testsbt)) + + tree = SBT.load(str(testsbt), leaf_loader=SigLeaf.load) + + to_search = load_one_signature(utils.get_test_data(utils.SIG_FILES[0])) + + print("*" * 60) + print("{}:".format(to_search)) + new_result = {str(s) for s in tree.find(search_minhashes, to_search, 0.1)} + print(*new_result, sep="\n") + assert len(new_result) == 2 + + +def test_load_zip_uncompressed(tmpdir): + # uncompress zipped SBT into a tmpdir and search unpacked SBT + import zipfile + + testdata = utils.get_test_data("v6.sbt.zip") + testsbt = tmpdir.join("v6.sbt.json") + + with zipfile.ZipFile(testdata, 'r') as z: + z.extractall(str(tmpdir)) + + tree = SBT.load(str(testsbt), leaf_loader=SigLeaf.load) + + to_search = load_one_signature(utils.get_test_data(utils.SIG_FILES[0])) + + print("*" * 60) + print("{}:".format(to_search)) + new_result = {str(s) for s in tree.find(search_minhashes, to_search, 0.1)} + print(*new_result, sep="\n") + assert len(new_result) == 2 + + def test_tree_repair(): tree_repair = SBT.load(utils.get_test_data('leaves.sbt.json'), leaf_loader=SigLeaf.load) diff --git a/tests/test_signature.py b/tests/test_signature.py index e1faffaa4b..384f604b9f 100644 --- a/tests/test_signature.py +++ b/tests/test_signature.py @@ -277,8 +277,8 @@ def test_save_minified(track_abundance): sig2 = SourmashSignature(e2, name="bar baz") x = save_signatures([sig1, sig2]) - assert '\n' not in x - assert len(x.split('\n')) == 1 + assert b'\n' not in x + assert len(x.split(b'\n')) == 1 y = list(load_signatures(x)) assert len(y) == 2 @@ -294,7 +294,20 @@ def test_load_minified(track_abundance): with open(sigfile, 'r') as f: orig_file = f.read() assert len(minified) < len(orig_file) - assert '\n' not in minified + assert b'\n' not in minified + + +def test_load_compressed(track_abundance): + e1 = sourmash.MinHash(n=1, ksize=20, track_abundance=track_abundance) + sig1 = SourmashSignature(e1) + + x = save_signatures([sig1], compression=5) + + y = load_one_signature(x) + assert sig1 == y + + sigfile = utils.get_test_data('genome-s10+s11.sig.gz') + sigs = load_signatures(sigfile) def test_binary_fp(tmpdir, track_abundance): diff --git a/tests/test_sourmash.py b/tests/test_sourmash.py index eafc60c269..8a957d6e7b 100644 --- a/tests/test_sourmash.py +++ b/tests/test_sourmash.py @@ -11,6 +11,7 @@ import csv import pytest import sys +import zipfile from . import sourmash_tst_utils as utils import sourmash @@ -3531,3 +3532,69 @@ def test_license_load_non_cc0(): sig = next(signature.load_signatures(sigfile, do_raise=True)) except Exception as e: assert "sourmash only supports CC0-licensed signatures" in str(e) + + +@utils.in_tempdir +def test_do_sourmash_index_zipfile(c): + testdata_glob = utils.get_test_data('gather/GCF*.sig') + testdata_sigs = glob.glob(testdata_glob) + + c.run_sourmash('index', '-k', '31', 'zzz.sbt.zip', + *testdata_sigs) + + outfile = c.output('zzz.sbt.zip') + assert os.path.exists(outfile) + + print(c) + assert c.last_result.status == 0 + assert 'Finished saving SBT, available at' in c.last_result.err + + with zipfile.ZipFile(outfile) as zf: + content = zf.namelist() + assert len(content) == 25 + assert len([c for c in content if 'internal' in c]) == 11 + assert ".sbt.zzz/" in content + sbts = [c for c in content if c.endswith(".sbt.json")] + assert len(sbts) == 1 + assert sbts[0] == "zzz.sbt.json" + + +@utils.in_tempdir +def test_do_sourmash_index_zipfile_append(c): + testdata_glob = utils.get_test_data('gather/GCF*.sig') + testdata_sigs = glob.glob(testdata_glob) + half_point = int(len(testdata_sigs) / 2) + first_half = testdata_sigs[:half_point] + second_half = testdata_sigs[half_point:] + + with pytest.warns(None) as record: + c.run_sourmash('index', '-k', '31', 'zzz.sbt.zip', + *first_half) + # UserWarning is raised when there are duplicated entries in the zipfile + assert not record + + outfile = c.output('zzz.sbt.zip') + assert os.path.exists(outfile) + + print(c) + assert c.last_result.status == 0 + assert 'Finished saving SBT, available at' in c.last_result.err + + with pytest.warns(None) as record: + c.run_sourmash('index', "--append", '-k', '31', 'zzz.sbt.zip', + *second_half) + # UserWarning is raised when there are duplicated entries in the zipfile + assert not record + + print(c) + assert c.last_result.status == 0 + assert 'Finished saving SBT, available at' in c.last_result.err + + with zipfile.ZipFile(outfile) as zf: + content = zf.namelist() + assert len(content) == 25 + assert len([c for c in content if 'internal' in c]) == 11 + assert ".sbt.zzz/" in content + sbts = [c for c in content if c.endswith(".sbt.json")] + assert len(sbts) == 1 + assert sbts[0] == "zzz.sbt.json"