diff --git a/.travis-data/test_daemon.py b/.travis-data/test_daemon.py index c96f8f665b..302939f0ec 100644 --- a/.travis-data/test_daemon.py +++ b/.travis-data/test_daemon.py @@ -98,6 +98,65 @@ def validate_workchains(expected_results): return valid +def validate_cached(cached_calcs): + """ + Check that the calculations with created with caching are indeed cached. + """ + return all( + '_aiida_cached_from' in calc.extras() and + calc.get_hash() == calc.get_extra('_aiida_hash') + for calc in cached_calcs + ) + +def create_calculation(code, counter, inputval, use_cache=False): + parameters = ParameterData(dict={'value': inputval}) + template = ParameterData(dict={ + ## The following line adds a significant sleep time. + ## I set it to 1 second to speed up tests + ## I keep it to a non-zero value because I want + ## To test the case when AiiDA finds some calcs + ## in a queued state + #'cmdline_params': ["{}".format(counter % 3)], # Sleep time + 'cmdline_params': ["1"], + 'input_file_template': "{value}", # File just contains the value to double + 'input_file_name': 'value_to_double.txt', + 'output_file_name': 'output.txt', + 'retrieve_temporary_files': ['triple_value.tmp'] + }) + calc = code.new_calc() + calc.set_max_wallclock_seconds(5 * 60) # 5 min + calc.set_resources({"num_machines": 1}) + calc.set_withmpi(False) + calc.set_parser_name('simpleplugins.templatereplacer.test.doubler') + + calc.use_parameters(parameters) + calc.use_template(template) + calc.store_all(use_cache=use_cache) + expected_result = { + 'value': 2 * inputval, + 'retrieved_temporary_files': { + 'triple_value.tmp': str(inputval * 3) + } + } + print "[{}] created calculation {}, pk={}".format( + counter, calc.uuid, calc.dbnode.pk) + return calc, expected_result + +def submit_calculation(code, counter, inputval): + calc, expected_result = create_calculation( + code=code, counter=counter, inputval=inputval + ) + calc.submit() + print "[{}] calculation submitted.".format(counter) + return calc, expected_result + +def create_cache_calc(code, counter, inputval): + calc, expected_result = create_calculation( + code=code, counter=counter, inputval=inputval, use_cache=True + ) + print "[{}] created cached calculation.".format(counter) + return calc, expected_result + def main(): # Submitting the Calculations @@ -106,39 +165,10 @@ def main(): expected_results_calculations = {} for counter in range(1, number_calculations + 1): inputval = counter - parameters = ParameterData(dict={'value': inputval}) - template = ParameterData(dict={ - ## The following line adds a significant sleep time. - ## I set it to 1 second to speed up tests - ## I keep it to a non-zero value because I want - ## To test the case when AiiDA finds some calcs - ## in a queued state - #'cmdline_params': ["{}".format(counter % 3)], # Sleep time - 'cmdline_params': ["1"], - 'input_file_template': "{value}", # File just contains the value to double - 'input_file_name': 'value_to_double.txt', - 'output_file_name': 'output.txt', - 'retrieve_temporary_files': ['triple_value.tmp'] - }) - calc = code.new_calc() - calc.set_max_wallclock_seconds(5 * 60) # 5 min - calc.set_resources({"num_machines": 1}) - calc.set_withmpi(False) - calc.set_parser_name('simpleplugins.templatereplacer.test.doubler') - - calc.use_parameters(parameters) - calc.use_template(template) - calc.store_all() - print "[{}] created calculation {}, pk={}".format( - counter, calc.uuid, calc.dbnode.pk) - expected_results_calculations[calc.pk] = { - 'value': inputval * 2, - 'retrieved_temporary_files': { - 'triple_value.tmp': str(inputval * 3) - } - } - calc.submit() - print "[{}] calculation submitted.".format(counter) + calc, expected_result = submit_calculation( + code=code, counter=counter, inputval=inputval + ) + expected_results_calculations[calc.pk] = expected_result # Submitting the Workchains print "Submitting {} workchains to the daemon".format(number_workchains) @@ -158,7 +188,7 @@ def main(): exited_with_timeout = True while time.time() - start_time < timeout_secs: time.sleep(15) # Wait a few seconds - + # Print some debug info, both for debugging reasons and to avoid # that the test machine is shut down because there is no output @@ -168,7 +198,7 @@ def main(): print "Output of 'verdi calculation list -a':" try: print subprocess.check_output( - ["verdi", "calculation", "list", "-a"], + ["verdi", "calculation", "list", "-a"], stderr=subprocess.STDOUT, ) except subprocess.CalledProcessError as e: @@ -177,7 +207,7 @@ def main(): print "Output of 'verdi work list':" try: print subprocess.check_output( - ['verdi', 'work', 'list'], + ['verdi', 'work', 'list'], stderr=subprocess.STDOUT, ) except subprocess.CalledProcessError as e: @@ -186,7 +216,7 @@ def main(): print "Output of 'verdi daemon status':" try: print subprocess.check_output( - ["verdi", "daemon", "status"], + ["verdi", "daemon", "status"], stderr=subprocess.STDOUT, ) except subprocess.CalledProcessError as e: @@ -204,8 +234,18 @@ def main(): timeout_secs) sys.exit(2) else: + # create cached calculations -- these should be FINISHED immediately + cached_calcs = [] + for counter in range(1, number_calculations + 1): + inputval = counter + calc, expected_result = create_cache_calc( + code=code, counter=counter, inputval=inputval + ) + cached_calcs.append(calc) + expected_results_calculations[calc.pk] = expected_result if (validate_calculations(expected_results_calculations) - and validate_workchains(expected_results_workchains)): + and validate_workchains(expected_results_workchains) + and validate_cached(cached_calcs)): print_daemon_log() print "" print "OK, all calculations have the expected parsed result" diff --git a/.travis.yml b/.travis.yml index 0c9e85d98b..23f643d999 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,9 +44,10 @@ install: # Install AiiDA with some optional dependencies - pip install .[REST,docs,atomic_tools,testing,dev_precommit] + env: ## Build matrix to test both backends, and the docs -## I still let it create the test backend for django +## I still let it create the test backend for django ## also when building the docs ## because otherwise the code would complain. Also, I need latex. - TEST_TYPE="pre-commit" diff --git a/aiida/backends/djsite/db/subtests/generic.py b/aiida/backends/djsite/db/subtests/generic.py index e9cab03d09..bdc5c60614 100644 --- a/aiida/backends/djsite/db/subtests/generic.py +++ b/aiida/backends/djsite/db/subtests/generic.py @@ -133,18 +133,22 @@ def test_replacement_1(self): DbExtra.set_value_for_node(n1.dbnode, "pippobis", [5, 6, 'c']) DbExtra.set_value_for_node(n2.dbnode, "pippo2", [3, 4, 'b']) - self.assertEquals(n1.dbnode.extras, {'pippo': [1, 2, 'a'], - 'pippobis': [5, 6, 'c']}) - self.assertEquals(n2.dbnode.extras, {'pippo2': [3, 4, 'b']}) + self.assertEquals(n1.get_extras(), {'pippo': [1, 2, 'a'], + 'pippobis': [5, 6, 'c'], + '_aiida_hash': n1.get_hash() + }) + self.assertEquals(n2.get_extras(), {'pippo2': [3, 4, 'b'], + '_aiida_hash': n2.get_hash() + }) new_attrs = {"newval1": "v", "newval2": [1, {"c": "d", "e": 2}]} DbExtra.reset_values_for_node(n1.dbnode, attributes=new_attrs) - self.assertEquals(n1.dbnode.extras, new_attrs) - self.assertEquals(n2.dbnode.extras, {'pippo2': [3, 4, 'b']}) + self.assertEquals(n1.get_extras(), new_attrs) + self.assertEquals(n2.get_extras(), {'pippo2': [3, 4, 'b'], '_aiida_hash': n2.get_hash()}) DbExtra.del_value_for_node(n1.dbnode, key='newval2') del new_attrs['newval2'] - self.assertEquals(n1.dbnode.extras, new_attrs) + self.assertEquals(n1.get_extras(), new_attrs) # Also check that other nodes were not damaged - self.assertEquals(n2.dbnode.extras, {'pippo2': [3, 4, 'b']}) + self.assertEquals(n2.get_extras(), {'pippo2': [3, 4, 'b'], '_aiida_hash': n2.get_hash()}) diff --git a/aiida/backends/sqlalchemy/tests/generic.py b/aiida/backends/sqlalchemy/tests/generic.py index c63e8c1d6f..5ab37243b0 100644 --- a/aiida/backends/sqlalchemy/tests/generic.py +++ b/aiida/backends/sqlalchemy/tests/generic.py @@ -141,18 +141,18 @@ def test_replacement_1(self): n2.set_extra("pippo2", [3, 4, u'b']) - self.assertEqual(n1.get_extras(),{'pippo': [1, 2, u'a'], 'pippobis': [5, 6, u'c']}) + self.assertEqual(n1.get_extras(),{'pippo': [1, 2, u'a'], 'pippobis': [5, 6, u'c'], '_aiida_hash': n1.get_hash()}) - self.assertEquals(n2.get_extras(), {'pippo2': [3, 4, 'b']}) + self.assertEquals(n2.get_extras(), {'pippo2': [3, 4, 'b'], '_aiida_hash': n2.get_hash()}) new_attrs = {"newval1": "v", "newval2": [1, {"c": "d", "e": 2}]} n1.reset_extras(new_attrs) self.assertEquals(n1.get_extras(), new_attrs) - self.assertEquals(n2.get_extras(), {'pippo2': [3, 4, 'b']}) + self.assertEquals(n2.get_extras(), {'pippo2': [3, 4, 'b'], '_aiida_hash': n2.get_hash()}) n1.del_extra('newval2') del new_attrs['newval2'] self.assertEquals(n1.get_extras(), new_attrs) # Also check that other nodes were not damaged - self.assertEquals(n2.get_extras(), {'pippo2': [3, 4, 'b']}) + self.assertEquals(n2.get_extras(), {'pippo2': [3, 4, 'b'], '_aiida_hash': n2.get_hash()}) diff --git a/aiida/backends/testbase.py b/aiida/backends/testbase.py index f5c1683141..26fe9700c6 100644 --- a/aiida/backends/testbase.py +++ b/aiida/backends/testbase.py @@ -144,7 +144,7 @@ def run_aiida_db_tests(tests_to_run, verbose=False): actually_run_tests = [] num_tests_expected = 0 - + # To avoid adding more than once the same test # (e.g. if you type both db and db.xxx) found_modulenames = set() diff --git a/aiida/backends/tests/__init__.py b/aiida/backends/tests/__init__.py index 7ec5e7b3b0..431791500e 100644 --- a/aiida/backends/tests/__init__.py +++ b/aiida/backends/tests/__init__.py @@ -66,6 +66,8 @@ 'pluginloader': ['aiida.backends.tests.test_plugin_loader'], 'daemon': ['aiida.backends.tests.daemon'], 'verdi_commands': ['aiida.backends.tests.verdi_commands'], + 'caching_config': ['aiida.backends.tests.test_caching_config'], + 'inline_calculation': ['aiida.backends.tests.inline_calculation'], } } diff --git a/aiida/backends/tests/inline_calculation.py b/aiida/backends/tests/inline_calculation.py new file mode 100644 index 0000000000..e0232e32e6 --- /dev/null +++ b/aiida/backends/tests/inline_calculation.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +########################################################################### +# Copyright (c), The AiiDA team. All rights reserved. # +# This file is part of the AiiDA code. # +# # +# The code is hosted on GitHub at https://github.com/aiidateam/aiida_core # +# For further information on the license, see the LICENSE.txt file # +# For further information please visit http://www.aiida.net # +########################################################################### +""" +Tests for inline calculations. +""" + +from aiida.orm.data.base import Int +from aiida.common.caching import enable_caching +from aiida.orm.calculation.inline import make_inline, InlineCalculation +from aiida.backends.testbase import AiidaTestCase + +class TestInlineCalculation(AiidaTestCase): + """ + Tests for the InlineCalculation calculations. + """ + def setUp(self): + @make_inline + def incr_inline(inp): + return {'res': Int(inp.value + 1)} + + self.incr_inline = incr_inline + + def test_incr(self): + """ + Simple test for the inline increment function. + """ + for i in [-4, 0, 3, 10]: + calc, res = self.incr_inline(inp=Int(i)) + self.assertEqual(res['res'].value, i + 1) + + def test_caching(self): + with enable_caching(InlineCalculation): + calc1, res1 = self.incr_inline(inp=Int(11)) + calc2, res2 = self.incr_inline(inp=Int(11)) + self.assertEquals(res1['res'].value, res2['res'].value, 12) + self.assertEquals(calc1.get_extra('_aiida_cached_from', calc1.uuid), calc2.get_extra('_aiida_cached_from')) + + def test_caching_change_code(self): + with enable_caching(InlineCalculation): + calc1, res1 = self.incr_inline(inp=Int(11)) + + @make_inline + def incr_inline(inp): + return {'res': Int(inp.value + 2)} + + calc2, res2 = incr_inline(inp=Int(11)) + self.assertNotEquals(res1['res'].value, res2['res'].value) + self.assertFalse('_aiida_cached_from' in calc2.extras()) diff --git a/aiida/backends/tests/nodes.py b/aiida/backends/tests/nodes.py index 47dd26deb3..ab0935d970 100644 --- a/aiida/backends/tests/nodes.py +++ b/aiida/backends/tests/nodes.py @@ -19,10 +19,119 @@ from aiida.backends.testbase import AiidaTestCase from aiida.common.exceptions import ModificationNotAllowed, UniquenessError from aiida.common.links import LinkType +from aiida.common import caching +from aiida.orm.code import Code from aiida.orm.data import Data from aiida.orm.node import Node from aiida.orm.utils import load_node +class TestNodeHashing(AiidaTestCase): + """ + Tests the functionality of hashing a node + """ + @staticmethod + def create_simple_node(a, b=0, c=0): + n = Node() + n._set_attr('a', a) + n._set_attr('b', b) + n._set_attr('c', c) + return n + + def test_simple_equal_nodes(self): + attributes = [ + (1.0, 1.1, 1.2), + ({'a': 'b', 'c': 'd'}, [1, 2, 3], {4, 1, 2}) + ] + for attr in attributes: + n1 = self.create_simple_node(*attr) + n2 = self.create_simple_node(*attr) + n1.store(use_cache=True) + n2.store(use_cache=True) + self.assertEqual(n1.uuid, n2.get_extra('_aiida_cached_from')) + + @staticmethod + def create_folderdata_with_empty_file(): + from aiida.orm.data.folder import FolderData + res = FolderData() + with res.folder.get_subfolder('path').open('name', 'w') as f: + pass + return res + + @staticmethod + def create_folderdata_with_empty_folder(): + from aiida.orm.data.folder import FolderData + res = FolderData() + res.folder.get_subfolder('path/name').create() + return res + + def test_folder_file_different(self): + f1 = self.create_folderdata_with_empty_file() + f2 = self.create_folderdata_with_empty_folder() + + assert ( + f1.folder.get_subfolder('path').get_content_list() == + f2.folder.get_subfolder('path').get_content_list() + ) + assert f1.get_hash() != f2.get_hash() + + def test_folder_same(self): + f1 = self.create_folderdata_with_empty_folder() + f2 = self.create_folderdata_with_empty_folder() + f1.store() + f2.store(use_cache=True) + assert f1.uuid == f2.get_extra('_aiida_cached_from') + + def test_file_same(self): + f1 = self.create_folderdata_with_empty_file() + f2 = self.create_folderdata_with_empty_file() + f1.store() + f2.store(use_cache=True) + assert f1.uuid == f2.get_extra('_aiida_cached_from') + + def test_simple_unequal_nodes(self): + attributes = [ + [(1.0, 1.1, 1.2), (2.0, 1.1, 1.2)], + [(1e-14,), (2e-14,)], + ] + for attr1, attr2 in attributes: + n1 = self.create_simple_node(*attr1) + n2 = self.create_simple_node(*attr2) + n1.store() + n2.store(use_cache=True) + self.assertNotEquals(n1.uuid, n2.uuid) + self.assertFalse('_aiida_cached_from' in n2.extras()) + + def test_unequal_arrays(self): + import numpy as np + from aiida.orm.data.array import ArrayData + arrays = [ + (np.zeros(1001), np.zeros(1005)), + (np.array([1, 2, 3]), np.array([2, 3, 4])) + ] + def create_arraydata(arr): + a = ArrayData() + a.set_array('a', arr) + return a + + for arr1, arr2 in arrays: + a1 = create_arraydata(arr1) + a1.store() + a2 = create_arraydata(arr2) + a2.store(use_cache=True) + self.assertNotEquals(a1.uuid, a2.uuid) + self.assertFalse('_aiida_cached_from' in a2.extras()) + + def test_updatable_attributes(self): + """ + Tests that updatable attributes are ignored. + """ + c = Code() + hash1 = c.get_hash() + c._hide() + hash2 = c.get_hash() + self.assertNotEquals(hash1, None) + self.assertEquals(hash1, hash2) + class TestDataNode(AiidaTestCase): """ These tests check the features of Data nodes that differ from the base Node @@ -480,6 +589,7 @@ def test_attributes_on_copy(self): 'bool': 'some non-boolean value', 'some_other_name': 987 } + all_extras = dict(_aiida_hash=AnyValue(), **extras_to_set) for k, v in extras_to_set.iteritems(): a.set_extra(k, v) @@ -504,12 +614,12 @@ def test_attributes_on_copy(self): b.store() # and I finally add a extras b.set_extra('meta', 'textofext') - b_expected_extras = {'meta': 'textofext'} + b_expected_extras = {'meta': 'textofext', '_aiida_hash': AnyValue()} # Now I check for the attributes # First I check that nothing has changed self.assertEquals({k: v for k, v in a.iterattrs()}, attrs_to_set) - self.assertEquals({k: v for k, v in a.iterextras()}, extras_to_set) + self.assertEquals({k: v for k, v in a.iterextras()}, all_extras) # I check then on the 'b' copy self.assertEquals({k: v @@ -834,7 +944,7 @@ def test_attr_and_extras(self): self.assertEquals(self.boolval, a.get_attr('bool')) self.assertEquals(a_string, a.get_extra('bool')) - self.assertEquals(a.get_extras(), {'bool': a_string}) + self.assertEquals(a.get_extras(), {'bool': a_string, '_aiida_hash': AnyValue()}) def test_get_extras_with_default(self): a = Node() @@ -912,14 +1022,16 @@ def test_attr_listing(self): for k, v in extras_to_set.iteritems(): a.set_extra(k, v) + all_extras = dict(_aiida_hash=AnyValue(), **extras_to_set) + self.assertEquals(set(a.attrs()), set(attrs_to_set.keys())) - self.assertEquals(set(a.extras()), set(extras_to_set.keys())) + self.assertEquals(set(a.extras()), set(all_extras.keys())) returned_internal_attrs = {k: v for k, v in a.iterattrs()} self.assertEquals(returned_internal_attrs, attrs_to_set) returned_attrs = {k: v for k, v in a.iterextras()} - self.assertEquals(returned_attrs, extras_to_set) + self.assertEquals(returned_attrs, all_extras) def test_versioning_and_postsave_attributes(self): """ @@ -987,10 +1099,12 @@ def test_delete_extras(self): 'further': 267, } + all_extras = dict(_aiida_hash=AnyValue(), **extras_to_set) + for k, v in extras_to_set.iteritems(): a.set_extra(k, v) - self.assertEquals({k: v for k, v in a.iterextras()}, extras_to_set) + self.assertEquals({k: v for k, v in a.iterextras()}, all_extras) # I pregenerate it, it cannot change during iteration list_keys = list(extras_to_set.keys()) @@ -998,8 +1112,8 @@ def test_delete_extras(self): # I delete one by one the keys and check if the operation is # performed correctly a.del_extra(k) - del extras_to_set[k] - self.assertEquals({k: v for k, v in a.iterextras()}, extras_to_set) + del all_extras[k] + self.assertEquals({k: v for k, v in a.iterextras()}, all_extras) def test_replace_extras_1(self): """ @@ -1023,6 +1137,7 @@ def test_replace_extras_1(self): 'h': 'j' }, [9, 8, 7]], } + all_extras = dict(_aiida_hash=AnyValue(), **extras_to_set) # I redefine the keys with more complicated data, and # changing the data type too @@ -1041,7 +1156,7 @@ def test_replace_extras_1(self): for k, v in extras_to_set.iteritems(): a.set_extra(k, v) - self.assertEquals({k: v for k, v in a.iterextras()}, extras_to_set) + self.assertEquals({k: v for k, v in a.iterextras()}, all_extras) for k, v in new_extras.iteritems(): # I delete one by one the keys and check if the operation is @@ -1050,8 +1165,8 @@ def test_replace_extras_1(self): # I update extras_to_set with the new entries, and do the comparison # again - extras_to_set.update(new_extras) - self.assertEquals({k: v for k, v in a.iterextras()}, extras_to_set) + all_extras.update(new_extras) + self.assertEquals({k: v for k, v in a.iterextras()}, all_extras) def test_basetype_as_attr(self): """ @@ -1960,6 +2075,12 @@ def test_check_single_calc_source(self): with self.assertRaises(ValueError): d1.add_link_from(calc2, link_type=LinkType.CREATE) +class AnyValue(object): + """ + Helper class that compares equal to everything. + """ + def __eq__(self, other): + return True class TestNodeDeletion(AiidaTestCase): def _check_existence(self, uuids_check_existence, uuids_check_deleted): diff --git a/aiida/backends/tests/test_caching_config.py b/aiida/backends/tests/test_caching_config.py new file mode 100644 index 0000000000..1fe3f35e3f --- /dev/null +++ b/aiida/backends/tests/test_caching_config.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +########################################################################### +# Copyright (c), The AiiDA team. All rights reserved. # +# This file is part of the AiiDA code. # +# # +# The code is hosted on GitHub at https://github.com/aiidateam/aiida_core # +# For further information on the license, see the LICENSE.txt file # +# For further information please visit http://www.aiida.net # +########################################################################### +import unittest +import tempfile + +import yaml + +from aiida.backends.utils import get_current_profile +from aiida.common.caching import configure, get_use_cache, enable_caching, disable_caching +from aiida.orm.calculation.job.simpleplugins.templatereplacer import TemplatereplacerCalculation + +class CacheConfigTest(unittest.TestCase): + """ + Tests the caching configuration. + """ + def setUp(self): + """ + Write a temporary config file, and load the configuration. + """ + self.config_reference = { + get_current_profile(): { + 'default': True, + 'enabled': [], + 'disabled': ['aiida.orm.calculation.job.simpleplugins.templatereplacer.TemplatereplacerCalculation'] + } + } + with tempfile.NamedTemporaryFile() as tf, open(tf.name, 'w') as of: + yaml.dump(self.config_reference, of) + configure(config_file=tf.name) + + def tearDown(self): + configure() + + def test_default(self): + self.assertTrue(get_use_cache()) + + def test_caching_enabled(self): + self.assertFalse(get_use_cache(TemplatereplacerCalculation)) + + def test_invalid_config(self): + with enable_caching(TemplatereplacerCalculation): + self.assertRaises(ValueError, get_use_cache, TemplatereplacerCalculation) + + def test_disable_caching(self): + from aiida.orm.data.base import Float + with disable_caching(Float): + self.assertFalse(get_use_cache(Float)) + self.assertTrue(get_use_cache(Float)) diff --git a/aiida/backends/tests/work/workChain.py b/aiida/backends/tests/work/workChain.py index 9e777f095e..a54fed4af2 100644 --- a/aiida/backends/tests/work/workChain.py +++ b/aiida/backends/tests/work/workChain.py @@ -12,6 +12,7 @@ import unittest import aiida.backends.settings as settings +from aiida.orm import load_node from aiida.backends.testbase import AiidaTestCase from plum.engine.ticking import TickingEngine from plum.persistence.bundle import Bundle diff --git a/aiida/backends/tests/work/workfunction.py b/aiida/backends/tests/work/workfunction.py index 3a271b04fe..c2597b72ab 100644 --- a/aiida/backends/tests/work/workfunction.py +++ b/aiida/backends/tests/work/workfunction.py @@ -10,9 +10,11 @@ import plum.process_monitor from aiida.backends.testbase import AiidaTestCase from aiida.work.workfunction import workfunction -from aiida.orm.data.base import get_true_node +from aiida.orm.data.base import get_true_node, Int +from aiida.orm import load_node from aiida.work.run import run import aiida.work.util as util +from aiida.common import caching @workfunction @@ -42,4 +44,22 @@ def test_blocking(self): def test_run(self): self.assertTrue(run(simple_wf)['result']) - self.assertTrue(run(return_input, get_true_node())['result']) \ No newline at end of file + self.assertTrue(run(return_input, get_true_node())['result']) + + def test_hashes(self): + _, pid1 = run(return_input, inp=Int(2), _return_pid=True) + _, pid2 = run(return_input, inp=Int(2), _return_pid=True) + w1 = load_node(pid1) + w2 = load_node(pid2) + self.assertEqual(w1.get_hash(), w2.get_hash()) + + def test_hashes_different(self): + _, pid1 = run(return_input, inp=Int(2), _return_pid=True) + _, pid2 = run(return_input, inp=Int(3), _return_pid=True) + w1 = load_node(pid1) + w2 = load_node(pid2) + self.assertNotEqual(w1.get_hash(), w2.get_hash()) + + def _check_hash_consistent(self, pid): + wc = load_node(pid) + self.assertEqual(wc.get_hash(), wc.get_extra('_aiida_hash')) diff --git a/aiida/cmdline/commands/rehash.py b/aiida/cmdline/commands/rehash.py new file mode 100644 index 0000000000..3862d6d04a --- /dev/null +++ b/aiida/cmdline/commands/rehash.py @@ -0,0 +1,59 @@ +import sys + +import click + +from plum.util import load_class +from plum.exceptions import ClassNotFoundException + +from aiida import try_load_dbenv +from aiida.cmdline.baseclass import VerdiCommand + + +class Rehash(VerdiCommand): + """ + Re-hash all nodes. + """ + def run(self, *args): + ctx = _rehash_cmd.make_context('rehash', list(args)) + with ctx: + _rehash_cmd.invoke(ctx) + + def complete(self, subargs_idx, subargs): + """ + No completion after 'verdi rehash'. + """ + print "" + +@click.command('rehash') +@click.option('--all', '-a', is_flag=True, help='Rehash all nodes of the given Node class.') +@click.option('--class-name', type=str, default='aiida.orm.node.Node', help='Restrict nodes which are re-hashed to instances of this class.') +@click.argument('pks', type=int, nargs=-1) +def _rehash_cmd(all, class_name, pks): + try_load_dbenv() + from aiida.orm.querybuilder import QueryBuilder + + # Get the Node class to match + try: + node_class = load_class(class_name) + except ClassNotFoundException: + click.echo("Could not load class '{}'.\nAborted!".format(class_name)) + sys.exit(1) + + # Add the filters for the class and PKs. + qb = QueryBuilder() + qb.append(node_class, tag='node') + if pks: + qb.add_filter('node', {'id': {'in': pks}}) + else: + if not all: + click.echo("Nothing specified, nothing re-hashed.\nExplicitly specify the PK of the nodes, or use '--all'.") + return + + if not qb.count(): + click.echo('No matching nodes found.') + return + for i, (node,) in enumerate(qb.iterall()): + if i % 100 == 0: + click.echo('.', nl=False) + node.rehash() + click.echo('\nAll done! {} node(s) re-hashed.'.format(i + 1)) diff --git a/aiida/cmdline/verdilib.py b/aiida/cmdline/verdilib.py index e6cde33c17..cf2d3d490f 100644 --- a/aiida/cmdline/verdilib.py +++ b/aiida/cmdline/verdilib.py @@ -58,6 +58,7 @@ class inheriting from VerdiCommand, and define a run(self,*args) method from aiida.cmdline.commands.comment import Comment from aiida.cmdline.commands.shell import Shell from aiida.cmdline.commands.restapi import Restapi +from aiida.cmdline.commands.rehash import Rehash from aiida.cmdline import execname diff --git a/aiida/common/caching.py b/aiida/common/caching.py new file mode 100644 index 0000000000..383c766b6f --- /dev/null +++ b/aiida/common/caching.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import os +import copy +from functools import wraps +from contextlib import contextmanager +try: + from collections import ChainMap +except ImportError: + from chainmap import ChainMap + +import yaml +from plum.util import load_class +from future.utils import raise_from +from plum.exceptions import ClassNotFoundException + +import aiida +from aiida.common.exceptions import ConfigurationError +from aiida.common.extendeddicts import Enumerate +from aiida.common.setup import AIIDA_CONFIG_FOLDER +from aiida.backends.utils import get_current_profile + +__all__ = ['get_use_cache', 'enable_caching', 'disable_caching'] + +config_keys = Enumerate(( + 'default', 'enabled', 'disabled' +)) + +DEFAULT_CONFIG = { + config_keys.default: False, + config_keys.enabled: [], + config_keys.disabled: [], +} + +def _get_config(config_file): + try: + with open(config_file, 'r') as f: + config = yaml.load(f)[get_current_profile()] + # no config file, or no config for this profile + except (OSError, IOError, KeyError): + return DEFAULT_CONFIG + + # validate configuration + for key in config: + if key not in DEFAULT_CONFIG: + raise ValueError( + "Configuration error: Invalid key '{}' in cache_config.yml".format(key) + ) + + # add defaults where config is missing + for key, default_config in DEFAULT_CONFIG.items(): + config[key] = config.get(key, default_config) + + # load classes + try: + for key in [config_keys.enabled, config_keys.disabled]: + config[key] = [load_class(c) for c in config[key]] + except (ImportError, ClassNotFoundException) as err: + raise_from( + ConfigurationError("Unknown class given in 'cache_config.yml': '{}'".format(err)), + err + ) + return config + +_CONFIG = {} + +def configure(config_file=os.path.join(os.path.expanduser(AIIDA_CONFIG_FOLDER), 'cache_config.yml')): + """ + Reads the caching configuration file and sets the _CONFIG variable. + """ + global _CONFIG + _CONFIG.clear() + _CONFIG.update(_get_config(config_file=config_file)) + +def _with_config(func): + @wraps(func) + def inner(*args, **kwargs): + if not _CONFIG: + configure() + return func(*args, **kwargs) + return inner + +@_with_config +def get_use_cache(node_class=None): + if node_class is not None: + enabled = node_class in _CONFIG[config_keys.enabled] + disabled = node_class in _CONFIG[config_keys.disabled] + if enabled and disabled: + raise ValueError('Invalid configuration: Caching for {} is both enabled and disabled.'.format(node_class)) + elif enabled: + return True + elif disabled: + return False + return _CONFIG[config_keys.default] + +@contextmanager +@_with_config +def _reset_config(): + global _CONFIG + config_copy = copy.deepcopy(_CONFIG) + yield + _CONFIG.clear() + _CONFIG.update(config_copy) + +@contextmanager +def enable_caching(node_class=None): + with _reset_config(): + if node_class is None: + _CONFIG[config_keys.default] = True + else: + _CONFIG[config_keys.enabled].append(node_class) + yield + +@contextmanager +def disable_caching(node_class=None): + with _reset_config(): + if node_class is None: + _CONFIG[config_keys.default] = True + else: + _CONFIG[config_keys.disabled].append(node_class) + yield diff --git a/aiida/common/hashing.py b/aiida/common/hashing.py index dfc0ccd489..c1bee463c6 100644 --- a/aiida/common/hashing.py +++ b/aiida/common/hashing.py @@ -12,6 +12,17 @@ import hashlib import time from datetime import datetime +import numbers +try: # Python3 + from functools import singledispatch + from collections import abc +except ImportError: # Python2 + from singledispatch import singledispatch + import collections as abc + +import numpy as np + +from .folders import Folder """ Here we define a single password hashing instance for the full AiiDA. @@ -112,14 +123,14 @@ def make_hash_with_type(type_chr, string_to_hash): """ return hashlib.sha224("{}{}".format(type_chr, string_to_hash)).hexdigest() -def make_hash(object_to_hash, float_precision=12): +@singledispatch +def make_hash(object_to_hash, **kwargs): """ Makes a hash from a dictionary, list, tuple or set to any level, that contains only other hashable or nonhashable types (including lists, tuples, sets, and dictionaries). :param object_to_hash: the object to hash - :param int float_precision: the precision when converting floats to strings :returns: a unique hash @@ -181,66 +192,127 @@ def make_hash(object_to_hash, float_precision=12): the string of dictionary do not suffice if we want to check for equality of dictionaries using hashes. """ - import numpy as np - - if isinstance(object_to_hash, (tuple, list)): - hashes = tuple([ - make_hash(_, float_precision=float_precision) - for _ - in object_to_hash - ]) - # We treat lists and tuples as if they are the same thing, - # but I think this is OK - return make_hash_with_type('L', "".join(hashes)) - - elif isinstance(object_to_hash, set): - hashes = tuple([ - make_hash(_, float_precision=float_precision) - for _ - in sorted(object_to_hash) - ]) - return make_hash_with_type('S', "".join(hashes)) - - elif isinstance(object_to_hash, dict): - hashed_dictionary = { - k: make_hash(v, float_precision=float_precision) - for k,v - in object_to_hash.items() - } - return make_hash_with_type( - 'D', make_hash(sorted( - hashed_dictionary.items()), float_precision=float_precision - ) - ) - - elif isinstance(object_to_hash, float): - return make_hash_with_type( - 'f','{:.{precision}f}'.format( - object_to_hash, precision=float_precision - ) - ) - # If is numpy: - elif type(object_to_hash).__module__ == np.__name__: - return make_hash_with_type('N', str(object_to_hash)) - - elif isinstance(object_to_hash, basestring): - return make_hash_with_type('s', object_to_hash) + raise ValueError("Value of type {} cannot be hashed".format( + type(object_to_hash)) + ) - elif isinstance(object_to_hash, bool): # bool must come before int - # I prefer to be sure of what I hash instead of using 'str' - return make_hash_with_type('b', "True" if object_to_hash else "False") +@make_hash.register(abc.Sequence) +def _(sequence, **kwargs): + hashes = tuple([ + make_hash(x, **kwargs) for x in sequence + ]) + return make_hash_with_type('L', ",".join(hashes)) + +@make_hash.register(abc.Set) +def _(object_to_hash, **kwargs): + hashes = tuple([ + make_hash(x, **kwargs) + for x + in sorted(object_to_hash) + ]) + return make_hash_with_type('S', ",".join(hashes)) + +@make_hash.register(abc.Mapping) +def _(mapping, **kwargs): + hashed_dictionary = { + k: make_hash(v, **kwargs) + for k,v + in mapping.items() + } + return make_hash_with_type( + 'D', + make_hash(sorted(hashed_dictionary.items()), **kwargs) + ) - elif object_to_hash is None: - return make_hash_with_type('n', "None") +@make_hash.register(numbers.Real) +def _(object_to_hash, **kwargs): + return make_hash_with_type( + 'f', + truncate_float64(object_to_hash).tobytes() + ) + +@make_hash.register(numbers.Complex) +def _(object_to_hash, **kwargs): + return make_hash_with_type( + 'c', + ','.join([ + make_hash(object_to_hash.real, **kwargs), + make_hash(object_to_hash.imag, **kwargs) + ]) + ) - elif isinstance(object_to_hash, int): - return make_hash_with_type('i', str(object_to_hash)) - elif isinstance(object_to_hash, long): - return make_hash_with_type('l', str(object_to_hash)) +@make_hash.register(numbers.Integral) +def _(object_to_hash, **kwargs): + return make_hash_with_type('i', str(object_to_hash)) + +@make_hash.register(basestring) +def _(object_to_hash, **kwargs): + return make_hash_with_type('s', object_to_hash) + +@make_hash.register(bool) +def _(object_to_hash, **kwargs): + return make_hash_with_type('b', str(object_to_hash)) + +@make_hash.register(type(None)) +def _(object_to_hash, **kwargs): + return make_hash_with_type('n', str(object_to_hash)) + +@make_hash.register(datetime) +def _(object_to_hash, **kwargs): + return make_hash_with_type('d', str(object_to_hash)) + +@make_hash.register(Folder) +def _(folder, **kwargs): + # make sure file is closed after being read + def _read_file(folder, name): + with folder.open(name) as f: + return f.read() + + ignored_folder_content = kwargs.get('ignored_folder_content', []) + + return make_hash_with_type( + 'pd', + make_hash([ + ( + name, + folder.get_subfolder(name) if folder.isdir(name) else + make_hash_with_type('pf', _read_file(folder, name)) + ) + for name in sorted(folder.get_content_list()) + if name not in ignored_folder_content + ], **kwargs) + ) - elif isinstance(object_to_hash, datetime): - return make_hash_with_type('d', str(object_to_hash)) - # Possibly add more types here, as needed +@make_hash.register(np.ndarray) +def _(object_to_hash, **kwargs): + if object_to_hash.dtype == np.float64: + return make_hash_with_type( + 'af', + make_hash(truncate_array64(object_to_hash).tobytes(), **kwargs) + ) + elif object_to_hash.dtype == np.complex128: + return make_hash_with_type( + 'ac', + make_hash([ + object_to_hash.real, + object_to_hash.imag + ], **kwargs) + ) else: - raise ValueError("Value of type {} cannot be hashed".format( - type(object_to_hash))) + return make_hash_with_type( + 'ao', + make_hash(object_to_hash.tobytes(), **kwargs) + ) + +def truncate_float64(x, num_bits=4): + mask = ~(2**num_bits - 1) + int_repr = np.float64(x).view(np.int64) + masked_int = int_repr & mask + truncated_x = masked_int.view(np.float64) + return truncated_x + +def truncate_array64(x, num_bits=4): + mask = ~(2**num_bits - 1) + int_array = np.array(x, dtype=np.float64).view(np.int64) + masked_array = int_array & mask + return masked_array.view(np.float64) diff --git a/aiida/orm/calculation/inline.py b/aiida/orm/calculation/inline.py index 5749c0bc9b..1869e3d602 100644 --- a/aiida/orm/calculation/inline.py +++ b/aiida/orm/calculation/inline.py @@ -9,21 +9,7 @@ ########################################################################### -from aiida.orm.implementation.calculation import InlineCalculation as _IC, \ - make_inline - - -class InlineCalculation(_IC): - """ - Here I put all the attributes/method that are common to all backends - """ - def get_desc(self): - """ - Returns a string with infos retrieved from a InlineCalculation node's - properties. - :return: description string - """ - return "{}()".format(self.get_function_name()) +from aiida.orm.implementation.calculation import InlineCalculation, make_inline def optional_inline(func): """ @@ -63,5 +49,3 @@ def wrapped_function(*args, **kwargs): return func(*args, **kwargs) return wrapped_function - - diff --git a/aiida/orm/implementation/django/calculation/inline.py b/aiida/orm/implementation/django/calculation/inline.py index 41feecc4bf..112ac0e63e 100644 --- a/aiida/orm/implementation/django/calculation/inline.py +++ b/aiida/orm/implementation/django/calculation/inline.py @@ -92,6 +92,7 @@ def wrapped_function(*args, **kwargs): for v in retval.itervalues(): v.store(with_transaction=False) + c.seal() # Return the calculation and the return values return c, retval diff --git a/aiida/orm/implementation/django/node.py b/aiida/orm/implementation/django/node.py index 9f116768c2..1113a1afe7 100644 --- a/aiida/orm/implementation/django/node.py +++ b/aiida/orm/implementation/django/node.py @@ -21,7 +21,7 @@ from aiida.common.folders import RepositoryFolder from aiida.common.links import LinkType from aiida.common.utils import get_new_uuid -from aiida.orm.implementation.general.node import AbstractNode, _NO_DEFAULT +from aiida.orm.implementation.general.node import AbstractNode, _NO_DEFAULT, _HASH_EXTRA_KEY from aiida.orm.mixins import Sealable # from aiida.orm.implementation.django.utils import get_db_columns from aiida.orm.implementation.general.utils import get_db_columns @@ -435,7 +435,7 @@ def _increment_version_number_db(self): # otherwise I only get the Django Field F object as a result! self._dbnode = DbNode.objects.get(pk=self._dbnode.pk) - def copy(self): + def copy(self, **kwargs): newobject = self.__class__() newobject.dbnode.type = self.dbnode.type # Inherit type newobject.dbnode.label = self.dbnode.label # Inherit label @@ -468,7 +468,7 @@ def dbnode(self): # self._dbnode = DbNode.objects.get(pk=self._dbnode.pk) return self._dbnode - def _db_store_all(self, with_transaction=True): + def _db_store_all(self, with_transaction=True, use_cache=None): """ Store the node, together with all input links, if cached, and also the linked nodes, if they were not stored yet. @@ -489,7 +489,7 @@ def _db_store_all(self, with_transaction=True): # Always without transaction: either it is the context_man here, # or it is managed outside self._store_input_nodes() - self.store(with_transaction=False) + self.store(with_transaction=False, use_cache=use_cache) self._store_cached_input_links(with_transaction=False) return self @@ -560,6 +560,8 @@ def _db_store(self, with_transaction=True): :parameter with_transaction: if False, no transaction is used. This is meant to be used ONLY if the outer calling function has already a transaction open! + + :param bool use_cache: Whether I attempt to find an equal node in the DB. """ # TODO: This needs to be generalized, allowing for flexible methods # for storing data and its attributes. @@ -615,5 +617,8 @@ def _db_store(self, with_transaction=True): self._repository_folder.abspath, move=True, overwrite=True) raise - return self + from aiida.backends.djsite.db.models import DbExtra + # I store the hash without cleaning and without incrementing the nodeversion number + DbExtra.set_value_for_node(self.dbnode, _HASH_EXTRA_KEY, self.get_hash()) + return self diff --git a/aiida/orm/implementation/general/calculation/__init__.py b/aiida/orm/implementation/general/calculation/__init__.py index 57c1098c06..c31659cb90 100644 --- a/aiida/orm/implementation/general/calculation/__init__.py +++ b/aiida/orm/implementation/general/calculation/__init__.py @@ -8,6 +8,7 @@ # For further information please visit http://www.aiida.net # ########################################################################### +import abc import collections from aiida.common.utils import classproperty @@ -82,8 +83,17 @@ class AbstractCalculation(SealableWithUpdatableAttributes): calculations run via a scheduler. """ - # A tuple with attributes that can be updated even after - # the call of the store() method + _cacheable = False + + @classproperty + def _hash_ignored_attributes(cls): + return super(AbstractCalculation, cls)._hash_ignored_attributes + [ + '_sealed', + '_finished', + ] + + # The link_type might not be correct while the object is being created. + _hash_ignored_inputs = ['CALL'] # Nodes that can be added as input using the use_* methods @classproperty @@ -324,3 +334,44 @@ def get_code(self): from aiida.orm.code import Code return dict(self.get_inputs(node_type=Code, also_labels=True)).get( self._use_methods['code']['linkname'], None) + + @abc.abstractmethod + def has_finished_ok(self): + """ + Returns whether the Calculation has finished successfully. + """ + raise NotImplementedError + + @abc.abstractmethod + def has_failed(self): + """ + Returns whether the Calculation has failed. + """ + raise NotImplementedError + + def has_finished(self): + """ + Determine if the calculation is finished for whatever reason. + This may be because it finished successfully or because of a failure. + + :return: True if the job has finished running, False otherwise. + :rtype: bool + """ + return self.has_finished_ok() or self.has_failed() + + def _is_valid_cache(self): + return super(AbstractCalculation, self)._is_valid_cache() and self.has_finished_ok() + + def _get_objects_to_hash(self): + """ + Return a list of objects which should be included in the hash. + """ + res = super(AbstractCalculation, self)._get_objects_to_hash() + res.append({ + key: value.get_hash() + for key, value in self.get_inputs_dict( + link_type=LinkType.INPUT + ).items() + if key not in self._hash_ignored_inputs + }) + return res diff --git a/aiida/orm/implementation/general/calculation/inline.py b/aiida/orm/implementation/general/calculation/inline.py index a51bfb280c..67fe75ecad 100644 --- a/aiida/orm/implementation/general/calculation/inline.py +++ b/aiida/orm/implementation/general/calculation/inline.py @@ -21,6 +21,16 @@ class InlineCalculation(Calculation): for a simple calculation """ + _cacheable = True + + def get_desc(self): + """ + Returns a string with infos retrieved from a InlineCalculation node's + properties. + :return: description string + """ + return "{}()".format(self.get_function_name()) + def get_function_name(self): """ Get the function name. @@ -29,6 +39,13 @@ def get_function_name(self): """ return self.get_attr('function_name', None) + def has_finished_ok(self): + return self.is_sealed + + def has_failed(self): + # The InlineCalculation wrapper doesn't catch errors, which means the + # calculation is returned only if it is valid. + return False def make_inline(func): """ diff --git a/aiida/orm/implementation/general/calculation/job/__init__.py b/aiida/orm/implementation/general/calculation/job/__init__.py index 2aa60c3d6a..7cfe186281 100644 --- a/aiida/orm/implementation/general/calculation/job/__init__.py +++ b/aiida/orm/implementation/general/calculation/job/__init__.py @@ -13,7 +13,7 @@ import copy from aiida.utils import timezone -from aiida.common.utils import str_timedelta +from aiida.common.utils import str_timedelta, classproperty from aiida.common.datastructures import calc_states from aiida.common.exceptions import ModificationNotAllowed, MissingPluginError from aiida.common.links import LinkType @@ -27,7 +27,6 @@ # 'rerunnable', # 'resourceLimits', - _input_subfolder = 'raw_input' @@ -37,6 +36,30 @@ class AbstractJobCalculation(object): remotely on a job scheduler. """ + _cacheable = True + + @classproperty + def _hash_ignored_attributes(cls): + # _updatable_attributes are ignored automatically. + return super(AbstractJobCalculation, cls)._hash_ignored_attributes + [ + 'queue_name', + 'priority', + 'max_wallclock_seconds', + 'max_memory_kb', + ] + + def get_hash( + self, + ignore_errors=True, + ignored_folder_content=('raw_input',), + **kwargs + ): + return super(AbstractJobCalculation, self).get_hash( + ignore_errors=ignore_errors, + ignored_folder_content=ignored_folder_content, + **kwargs + ) + @classmethod def process(cls): from aiida.work.legacy.job_process import JobProcess @@ -61,7 +84,7 @@ def _init_internal_params(self): 'state', 'job_id', 'scheduler_state', 'scheduler_lastchecktime', 'last_jobinfo', 'remote_workdir', 'retrieve_list', - 'retrieve_singlefile_list' + 'retrieve_singlefile_list', 'retrieve_temporary_list' ) # Files in which the scheduler output and error will be stored. @@ -102,12 +125,21 @@ def store(self, *args, **kwargs): super(AbstractJobCalculation, self).store(*args, **kwargs) # I get here if the calculation was successfully stored. - self._set_state(calc_states.NEW) + # Set to new only if it is not already FINISHED (due to caching) + if not self.get_state() == calc_states.FINISHED: + self._set_state(calc_states.NEW) # Important to return self to allow the one-liner # c = Calculation().store() return self + def _add_outputs_from_cache(self, cache_node): + self._set_state(calc_states.PARSING) + super(AbstractJobCalculation, self)._add_outputs_from_cache( + cache_node=cache_node + ) + self._set_state(cache_node.get_state()) + def _validate(self): """ Verify if all the input nodes are present and valid. @@ -627,16 +659,6 @@ def _is_running(self): calc_states.PARSING ] - def has_finished(self): - """ - Determine if the calculation is finished for whatever reason. - This may be because it finished successfully or because of a failure. - - :return: True if the job has finished running, False otherwise. - :rtype: bool - """ - return self.has_finished_ok() or self.has_failed() - def has_finished_ok(self): """ Get whether the calculation is in the FINISHED status. @@ -736,7 +758,7 @@ def _set_retrieve_temporary_list(self, retrieve_temporary_list): 'strings or lists/tuples' ) - if (not (isinstance(item[0], basestring)) or + if (not (isinstance(item[0], basestring)) or not (isinstance(item[1], basestring)) or not (isinstance(item[2], int))): raise ValueError( diff --git a/aiida/orm/implementation/general/node.py b/aiida/orm/implementation/general/node.py index 50f818b0bb..ed0a871d5e 100644 --- a/aiida/orm/implementation/general/node.py +++ b/aiida/orm/implementation/general/node.py @@ -9,24 +9,29 @@ ########################################################################### from abc import ABCMeta, abstractmethod, abstractproperty -import collections -import logging import os import types +import logging +import importlib +import collections +try: + import pathlib +except ImportError: + import pathlib2 as pathlib from aiida.common.exceptions import (InternalError, ModificationNotAllowed, UniquenessError, ValidationError) from aiida.common.folders import SandboxFolder from aiida.common.utils import abstractclassmethod from aiida.common.utils import combomethod - +from aiida.common.caching import get_use_cache from aiida.common.links import LinkType from aiida.common.lang import override from aiida.common.old_pluginloader import get_query_type_string from aiida.backends.utils import validate_attribute_key _NO_DEFAULT = tuple() - +_HASH_EXTRA_KEY = '_aiida_hash' def clean_value(value): """ @@ -140,6 +145,12 @@ def __new__(cls, name, bases, attrs): # See documentation in the set() method. _set_incompatibilities = [] + # A list of attribute names that will be ignored when creating the hash. + _hash_ignored_attributes = [] + + # Flag that determines whether the class can be cached. + _cacheable = True + def get_desc(self): """ Returns a string with infos retrieved from a node's @@ -1128,9 +1139,9 @@ def iterextras(self): # added (in particular, we do not even have an ID to use!) # Return without value, meaning that this is an empty generator return - yield # Needed after return to convert it to a generator - for _ in self._db_iterextras(): - yield _ + yield # Needed after return to convert it to a generator + for extra in self._db_iterextras(): + yield extra def iterattrs(self): """ @@ -1255,7 +1266,7 @@ def _increment_version_number_db(self): pass @abstractmethod - def copy(self): + def copy(self, **kwargs): """ Return a copy of the current object to work with, not stored yet. @@ -1427,7 +1438,7 @@ def get_abs_path(self, path=None, section=None): section, reset_limit=True).get_abs_path( path, check_existence=True) - def store_all(self, with_transaction=True): + def store_all(self, with_transaction=True, use_cache=None): """ Store the node, together with all input links, if cached, and also the linked nodes, if they were not stored yet. @@ -1452,10 +1463,10 @@ def store_all(self, with_transaction=True): "unstored parents, cannot proceed (only direct parents " "can be unstored and will be stored by store_all, not " "grandparents or other ancestors".format(parent_node.uuid)) - return self._db_store_all(with_transaction) + return self._db_store_all(with_transaction, use_cache=use_cache) @abstractmethod - def _db_store_all(self, with_transaction=True): + def _db_store_all(self, with_transaction=True, use_cache=None): """ Store the node, together with all input links, if cached, and also the linked nodes, if they were not stored yet. @@ -1463,6 +1474,9 @@ def _db_store_all(self, with_transaction=True): :parameter with_transaction: if False, no transaction is used. This is meant to be used ONLY if the outer calling function has already a transaction open! + + :param use_cache: Determines whether caching is used to find an equivalent node. + :type use_cache: bool """ pass @@ -1523,7 +1537,7 @@ def _store_cached_input_links(self, with_transaction=True): """ pass - def store(self, with_transaction=True): + def store(self, with_transaction=True, use_cache=None): """ Store a new node in the DB, also saving its repository directory and attributes. @@ -1552,8 +1566,17 @@ def store(self, with_transaction=True): # the case. self._check_are_parents_stored() - # call implementation-dependent store method - self._db_store(with_transaction) + # Get default for use_cache if it's not set explicitly. + if use_cache is None: + use_cache = get_use_cache(type(self)) + # Retrieve the cached node. + same_node = self._get_same_node() if use_cache else None + if same_node is not None: + self._store_from_cache(same_node, with_transaction=with_transaction) + self._add_outputs_from_cache(same_node) + else: + # call implementation-dependent store method + self._db_store(with_transaction) # Set up autogrouping used by verdi run from aiida.orm.autogroup import current_autogroup, Autogroup, VERDIAUTOGROUP_TYPE @@ -1575,6 +1598,28 @@ def store(self, with_transaction=True): # n = Node().store() return self + def _store_from_cache(self, cache_node, with_transaction): + new_node = cache_node.copy(include_updatable_attrs=True) + inputlinks_cache = self._inputlinks_cache + # "impersonate" the copied node by getting all its attributes + self.__dict__ = new_node.__dict__ + # restore the input links + self._inputlinks_cache = inputlinks_cache + + # Make sure the node doesn't have any RETURN links + if cache_node.get_outputs(link_type=LinkType.RETURN): + raise ValueError("Cannot use cache from nodes with RETURN links.") + + self.store(with_transaction=with_transaction, use_cache=False) + self.set_extra('_aiida_cached_from', cache_node.uuid) + + def _add_outputs_from_cache(self, cache_node): + # add CREATE links + output_mapping = {} + for linkname, out_node in cache_node.get_outputs(also_labels=True, link_type=LinkType.CREATE): + new_node = out_node.copy(include_updatable_attrs=True).store() + new_node.add_link_from(self, label=linkname, link_type=LinkType.CREATE) + @abstractmethod def _db_store(self, with_transaction=True): """ @@ -1595,6 +1640,7 @@ def _db_store(self, with_transaction=True): """ pass + def __del__(self): """ Called only upon real object destruction from memory @@ -1604,6 +1650,95 @@ def __del__(self): if getattr(self, '_temp_folder', None) is not None: self._temp_folder.erase() + + def get_hash(self, ignore_errors=True, **kwargs): + """ + Making a hash based on my attributes + """ + from aiida.common.hashing import make_hash + try: + return make_hash(self._get_objects_to_hash(), **kwargs) + except Exception as e: + if ignore_errors: + return None + else: + raise e + + def _get_objects_to_hash(self): + """ + Return a list of objects which should be included in the hash. + """ + computer = self.get_computer() + return [ + importlib.import_module( + self.__module__.split('.', 1)[0] + ).__version__, + { + key: val for key, val in self.get_attrs().items() + if ( + (key not in self._hash_ignored_attributes) and + (key not in getattr(self, '_updatable_attributes', tuple())) + ) + }, + self.folder, + computer.uuid if computer is not None else None + ] + + def rehash(self): + """ + Re-generates the stored hash of the Node. + """ + self.set_extra(_HASH_EXTRA_KEY, self.get_hash()) + + def clear_hash(self): + """ + Sets the stored hash of the Node to None. + """ + self.set_extra(_HASH_EXTRA_KEY, None) + + def _get_same_node(self): + """ + Returns a stored node from which the current Node can be cached, meaning that the returned Node is a valid cache, and its ``_aiida_hash`` attribute matches ``self.get_hash()``. + + If there are multiple valid matches, the first one is returned. If no matches are found, ``None`` is returned. + + Note that after ``self`` is stored, this function can return ``self``. + """ + try: + return next(self._iter_all_same_nodes()) + except StopIteration: + return None + + def get_all_same_nodes(self): + """ + Return a list of stored nodes which match the type and hash of the current node. For the stored nodes, the ``_aiida_hash`` extra is checked to determine the hash, while ``self.get_hash()`` is executed on the current node. + + Only nodes which are a valid cache are returned. If the current node is already stored, it can be included in the returned list if ``self.get_hash()`` matches its ``_aiida_hash``. + """ + return list(self._iter_all_same_nodes()) + + def _iter_all_same_nodes(self): + """ + Returns an iterator of all same nodes. + """ + if not self._cacheable: + return iter(()) + + from aiida.orm.querybuilder import QueryBuilder + + hash_ = self.get_hash() + if hash_: + qb = QueryBuilder() + qb.append(self.__class__, filters={'extras._aiida_hash': hash_}, project='*', subclassing=False) + same_nodes = (n[0] for n in qb.iterall()) + return (n for n in same_nodes if n._is_valid_cache()) + + def _is_valid_cache(self): + """ + Subclass hook to exclude certain Nodes (e.g. failed calculations) from being considered in the caching process. + """ + return True + @property def out(self): """ diff --git a/aiida/orm/implementation/sqlalchemy/calculation/inline.py b/aiida/orm/implementation/sqlalchemy/calculation/inline.py index 3c24088421..940e69b8ca 100644 --- a/aiida/orm/implementation/sqlalchemy/calculation/inline.py +++ b/aiida/orm/implementation/sqlalchemy/calculation/inline.py @@ -95,6 +95,7 @@ def wrapped_function(*args, **kwargs): for v in retval.itervalues(): v.store(with_transaction=False) + c.seal() # Return the calculation and the return values return c, retval diff --git a/aiida/orm/implementation/sqlalchemy/node.py b/aiida/orm/implementation/sqlalchemy/node.py index 18602f9104..897bea6ccc 100644 --- a/aiida/orm/implementation/sqlalchemy/node.py +++ b/aiida/orm/implementation/sqlalchemy/node.py @@ -28,7 +28,7 @@ NotExistent, UniquenessError) from aiida.common.links import LinkType -from aiida.orm.implementation.general.node import AbstractNode, _NO_DEFAULT +from aiida.orm.implementation.general.node import AbstractNode, _NO_DEFAULT, _HASH_EXTRA_KEY from aiida.orm.implementation.sqlalchemy.computer import Computer from aiida.orm.implementation.sqlalchemy.group import Group from aiida.orm.implementation.sqlalchemy.utils import django_filter, \ @@ -483,7 +483,7 @@ def _increment_version_number_db(self): raise - def copy(self): + def copy(self, **kwargs): newobject = self.__class__() newobject.dbnode.type = self.dbnode.type # Inherit type newobject.dbnode.label = self.dbnode.label # Inherit label @@ -508,7 +508,7 @@ def id(self): def dbnode(self): return self._dbnode - def _db_store_all(self, with_transaction=True): + def _db_store_all(self, with_transaction=True, use_cache=None): """ Store the node, together with all input links, if cached, and also the linked nodes, if they were not stored yet. @@ -519,7 +519,7 @@ def _db_store_all(self, with_transaction=True): """ self._store_input_nodes() - self.store(with_transaction=False) + self.store(with_transaction=False, use_cache=use_cache) self._store_cached_input_links(with_transaction=False) from aiida.backends.sqlalchemy import get_scoped_session session = get_scoped_session() @@ -601,6 +601,8 @@ def _db_store(self, with_transaction=True): :parameter with_transaction: if False, no transaction is used. This is meant to be used ONLY if the outer calling function has already a transaction open! + + :param bool use_cache: Whether I attempt to find an equal node in the DB. """ from aiida.backends.sqlalchemy import get_scoped_session session = get_scoped_session() @@ -656,6 +658,7 @@ def _db_store(self, with_transaction=True): self._repository_folder.abspath, move=True, overwrite=True) raise + self.dbnode.set_extra(_HASH_EXTRA_KEY, self.get_hash()) return self diff --git a/aiida/orm/mixins.py b/aiida/orm/mixins.py index 08a7977067..9355ddeb8b 100644 --- a/aiida/orm/mixins.py +++ b/aiida/orm/mixins.py @@ -98,11 +98,12 @@ def iter_updatable_attrs(self): pass @override - def copy(self): + def copy(self, include_updatable_attrs=False): newobj = super(SealableWithUpdatableAttributes, self).copy() # Remove the updatable attributes - for k, v in self.iter_updatable_attrs(): - newobj._del_attr(k) + if not include_updatable_attrs: + for k, v in self.iter_updatable_attrs(): + newobj._del_attr(k) return newobj diff --git a/aiida/work/process.py b/aiida/work/process.py index 91f3b70142..3a5e7f5717 100644 --- a/aiida/work/process.py +++ b/aiida/work/process.py @@ -18,6 +18,7 @@ import plum.process from plum.process_monitor import MONITOR import plum.process_monitor +from plum.error import FastForwardError import voluptuous from abc import ABCMeta @@ -26,6 +27,7 @@ from aiida.common.lang import override, protected from aiida.common.log import LOG_LEVEL_REPORT from aiida.common.links import LinkType +from aiida.common import caching from aiida.utils.calculation import add_source_info from aiida.work.defaults import class_loader import aiida.work.util @@ -35,7 +37,6 @@ from aiida.orm.calculation.work import WorkCalculation - class DictSchema(object): def __init__(self, schema): self._schema = voluptuous.Schema(schema) @@ -330,7 +331,17 @@ def _create_and_setup_db_record(self): self._calc = self.create_db_record() self._setup_db_record() if self.inputs._store_provenance: - self.calc.store_all() + self.calc.store_all(use_cache=self._use_cache_enabled()) + if self.calc.has_finished_ok(): + self._state = plum.process.ProcessState.FINISHED + for name, value in self.calc.get_outputs_dict(link_type=LinkType.RETURN).items(): + if name.endswith('_{pk}'.format(pk=value.pk)): + continue + self.out(name, value) + # This is needed for JobProcess. In that case, the outputs are + # returned regardless of whether they end in '_pk' + for name, value in self.calc.get_outputs_dict(link_type=LinkType.CREATE).items(): + self.out(name, value) if self.calc.pk is not None: return self.calc.pk @@ -397,14 +408,16 @@ def _add_description_and_label(self): if label is not None: self._calc.label = label - def _can_fast_forward(self, inputs): - return False - - def _fast_forward(self): - node = None # Here we should find the old node - for k, v in node.get_output_dict(): - self.out(k, v) - + def _use_cache_enabled(self): + # First priority: inputs + try: + return self._parsed_inputs['_use_cache'] + # Second priority: config + except KeyError: + return ( + caching.get_use_cache(type(self)) or + caching.get_use_cache(type(self._calc)) + ) class FunctionProcess(Process): _func_args = None diff --git a/docs/requirements_for_rtd.txt b/docs/requirements_for_rtd.txt index a14e6b31bb..226521f1d3 100644 --- a/docs/requirements_for_rtd.txt +++ b/docs/requirements_for_rtd.txt @@ -23,17 +23,17 @@ paramiko==2.4.0 passlib==1.7.1 pathlib2==2.3.0 pip==9.0.1 -plumpy==0.7.10 +plumpy==0.7.11 portalocker==1.1.0 psutil==5.4.0 pycrypto==2.6.1 python-dateutil==2.6.0 python-mimeparse==0.1.4 pytz==2014.10 +pyyaml reentry==1.0.2 scipy<1.0.0 setuptools==36.6.0 -singledispatch==3.4.0.3 six==1.11.0 tabulate==0.7.5 tzlocal==1.3 diff --git a/docs/source/caching/index.rst b/docs/source/caching/index.rst new file mode 100644 index 0000000000..e730dbcaca --- /dev/null +++ b/docs/source/caching/index.rst @@ -0,0 +1,151 @@ +.. _caching: + +Caching +======= + +When working with AiiDA, you might sometimes re-run calculations which were already successfully executed. Because this can waste a lot of computational resources, you can enable AiiDA to **cache** calculations, which means that it will re-use existing calculations if a calculation with the same inputs is submitted again. + +When a calculation is cached, a copy of the original calculation is created. This copy will keep the input links of the new calculation. The outputs of the original calculation are also copied, and linked to the new calculation. This allows for the new calculation to be a separate Node in the provenance graph and, critically, preserves the acyclicity of the graph. + +Caching is also implemented for Data nodes. This is not very useful in practice (yet), but is an easy way to show how the caching mechanism works: + +.. ipython:: + :verbatim: + + In [1]: from __future__ import print_function + + In [2]: from aiida.orm.data.base import Str + + In [3]: n1 = Str('test string') + + In [4]: n1.store() + Out[4]: u'test string' + + In [5]: n2 = Str('test string') + + In [6]: n2.store(use_cache=True) + Out[6]: u'test string' + + In [7]: print('UUID of n1:', n1.uuid) + UUID of n1: 956109e1-4382-4240-a711-2a4f3b522122 + + In [8]: print('n2 is cached from:', n2.get_extra('_aiida_cached_from')) + n2 is cached from: 956109e1-4382-4240-a711-2a4f3b522122 + +As you can see, passing ``use_cache=True`` to the ``store`` method enables using the cache. The fact that ``n2`` was created from ``n1`` is stored in the ``_aiida_cached_from`` extra of ``n2``. + +When running a ``JobCalculation`` through the ``Process`` interface, you cannot directly set the ``use_cache`` flag when the calculation node is stored internally. Instead, you can pass the ``_use_cache`` flag to the ``run`` or ``submit`` method. + +Caching is **not** implemented for workchains and workfunctions. Unlike calculations, they can not only create new data nodes, but also return exsting ones. When copying a cached workchain, it's not clear which node should be returned without actually running the workchain. This is explained in more detail in the section :ref:`caching_provenance`. + +Configuration +------------- + +Of course, using caching would be quite tedious if you had to set ``use_cache`` manually everywhere. To fix this, the default for ``use_cache`` can be set in the ``.aiida/cache_config.yml`` file. You can specify a global default, or enable / disable caching for specific calculation or data classes. An example configuration file might look like this: + +.. code:: yaml + + profile-name: + default: False + enabled: + - aiida.orm.calculation.job.simpleplugins.templatereplacer.TemplatereplacerCalculation + - aiida.orm.data.base.Str + disabled: + - aiida.orm.data.base.Float + +This means that caching is enabled for ``TemplatereplacerCalculation`` and ``Str``, and disabled for all other classes. In this example, manually disabling ``aiida.orm.data.base.Float`` is actually not needed, since the ``default: False`` configuration means that caching is disabled for all classes unless it is manually enabled. Note also that the fully qualified class import name (e.g., ``aiida.orm.data.base.Str``) must be given, not just the class name (``Str``). This is to avoid accidentally matching classes with the same name. You can get this name by combining the module name and class name, or (usually) from the string representation of the class: + +.. ipython:: + :verbatim: + + In [1]: Str.__module__ + '.' + Str.__name__ + Out[1]: 'aiida.orm.data.base.Str' + + In [2]: str(Str) + Out[2]: "" + +Note that this is not the same as the type string stored in the database. + +.. _caching_matches: + +How are cached nodes matched? +----------------------------- + +To determine wheter a given node is identical to an existing one, a hash of the content of the node is created. If a node of the same class with the same hash already exists in the database, this is considered a cache match. You can manually check the hash of a given node with the :meth:`.get_hash() <.AbstractNode.get_hash>` method. Once a node is stored in the database, its hash is stored in the ``_aiida_hash`` extra, and this is used to find matching nodes. + +By default, this hash is created from: + +* all attributes of a node, except the ``_updatable_attributes`` +* the ``__version__`` of the module which defines the node class +* the content of the repository folder of the node +* the UUID of the computer, if the node has one + +In the case of calculations, the hashes of the inputs are also included. When developing calculation and data classes, there are some methods you can use to determine how the hash is created: + +* To ignore specific attributes, a ``Node`` subclass can have a ``_hash_ignored_attributes`` attribute. This is a list of attribute names which are ignored when creating the hash. +* For calculations, the ``_hash_ignored_inputs`` attribute lists inputs that should be ignored when creating the hash. +* To add things which should be considered in the hash, you can override the :meth:`_get_objects_to_hash <.AbstractNode._get_objects_to_hash>` method. Note that doing so overrides the behavior described above, so you should make sure to use the ``super()`` method. +* Pass a keyword argument to :meth:`.get_hash <.AbstractNode.get_hash>`. These are passed on to ``aiida.common.hashing.make_hash``. For example, the ``ignored_folder_content`` keyword is used by the :class:`JobCalculation <.AbstractJobCalculation>` to ignore the ``raw_input`` subfolder of its repository folder. + +Additionally, there are two methods you can use to disable caching for particular nodes: + +* The :meth:`._is_valid_cache` method determines whether a particular node can be used as a cache. This is used for example to disable caching from failed calculations. +* Node classes have a ``_cacheable`` attribute, which can be set to ``False`` to completely switch off caching for nodes of that class. This avoids performing queries for the hash altogether. + +There are two ways in which the hash match can go wrong: False negatives, where two nodes should have the same hash but do not, or false positives, where two different nodes have the same hash. It is important to understand that false negatives are **highly preferrable**, because they only increase the runtime of your calculations, as if caching was disabled. False positives however can break the logic of your calculations. Be mindful of this when modifying the caching behaviour of your calculation and data classes. + +.. _caching_error: + +What to do when caching is used when it shouldn't +------------------------------------------------- + +In general, the caching mechanism should trigger only when the output of a calculation will be exactly the same as if it is run again. However, there might be some edge cases where this is not true. + +For example, if the parser is in a different python module than the calculation, the version number used in the hash will not change when the parser is updated. While the "correct" solution to this problem is to increase the version number of a calculation when the behavior of its parser changes, there might still be cases (e.g. during development) when you manually want to stop a particular node from being cached. + +In such cases, you can follow these steps to disable caching: + +1. If you suspect that a node has been cached in error, check that it has a ``_aiida_cached_from`` extra. If that's not the case, it is not a problem of caching. +2. Get all nodes which match your node, and clear their hash: + + .. code:: python + + for n in node.get_all_same_nodes(): + n.clear_hash() +3. Run your calculation again. Now it should not use caching. + +If you instead think that there is a bug in the AiiDA implementation, please open an issue (with enough information to be able to reproduce the error, otherwise it is hard for us to help you) in the AiiDA GitHub repository: https://github.com/aiidateam/aiida_core/issues/new. + +.. _caching_provenance: + +Caching and the Provenance Graph +-------------------------------- + +The goal of the caching mechanism is to speed up AiiDA calculations by re-using duplicate calculations. However, the resulting provenance graph should be exactly the same as if caching was disabled. This has important consequences on the kind of caching operations that are possible. + +The provenance graph consists of nodes describing data, calculations and workchains, and links describing the relationship between these nodes. We have seen that the hash of a node is used to determine whether two nodes are equivalent. To successfully use a cached node however, we also need to know how the new node should be linked to its parents and children. + +In the case of a plain data node, this is simple: Copying a data node from an equivalent node should not change its links, so we just need to preserve the links which this new node already has. + +For calculations, the situation is a bit more complex: The node can have inputs and creates new data nodes as outputs. Again, the new node needs to keep its existing links. For the outputs, the calculation needs to create a copy of each node and link these as its outputs. This makes it look as if the calculation had produced these outputs itself, without caching. + +Finally, workchains can create links not only to nodes which they create themselves, but also to nodes created by a calculation that they called, or even their ancestors. This is where caching becomes impossible. Consider the following example (using workfunctions for simplicity): + +.. code:: python + + from aiida.orm.data.base import Int + from aiida.work.workfunction import workfunction + + @workfunction + def select(a, b): + return b + + d = Int(1) + r1 = select(d, d) + r2 = select(Int(1), Int(1)) + +The two ``select`` workfunctions have the same inputs as far as their hashes go. However, the first call uses the same input node twice, while the second one has two different inputs. If the second call should be cached from the first one, it is not clear which of the two input nodes should be returned. + +While this example might seem contrived, the conclusion is valid more generally: Because workchains can return nodes from their history, they cannot be cached. Since even two equivalent workchains (with the same inputs) can have a different history, there is no way to deduce which links should be created on the new workchain without actually running it. + +Overall, this limitation is acceptable: The runtime of AiiDA workchains is usually dominated by time spent inside expensive calculations. Since these can be avoided with the caching mechanism, it still improves the runtime and required computer resources a lot. diff --git a/docs/source/conf.py b/docs/source/conf.py index f60cd7fdf5..4b2c05469b 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -43,7 +43,7 @@ # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.imgmath', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.imgmath', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode', 'IPython.sphinxext.ipython_console_highlighting', 'IPython.sphinxext.ipython_directive'] todo_include_todos = True diff --git a/docs/source/developer_guide/caching.rst b/docs/source/developer_guide/caching.rst new file mode 100644 index 0000000000..9cca41f04b --- /dev/null +++ b/docs/source/developer_guide/caching.rst @@ -0,0 +1,12 @@ +Caching: implementation details ++++++++++++++++++++++++++++++++ + +This section covers some details of the caching mechanism which are not discussed in the :ref:`user guide `. If you are developing a plugin and want to modify the caching behavior of your classes, we recommend you read :ref:`this section ` first. + +Disabling caching for ``WorkCalculation`` +----------------------------------------- + +As discussed in the :ref:`user guide `, nodes which can have ``RETURN`` links cannot be cached. This is enforced on two levels: + +* The ``_cacheable`` property is set to ``False`` in the :class:`.AbstractCalculation`, and only re-enabled in :class:`.AbstractJobCalculation` and :class:`InlineCalculation <.general.calculation.inline.InlineCalculation>`. This means that a ``WorkCalculation`` will not be cached. +* The ``_store_from_cache`` method, which is used to "clone" an existing node, will raise an error if the existing node has any ``RETURN`` links. This extra safe-guard prevents cases where a user might incorrectly override the ``_cacheable`` property on a ``WorkCalculation`` subclass. diff --git a/docs/source/developer_guide/index.rst b/docs/source/developer_guide/index.rst index 5375e00e0b..fec4a139ae 100644 --- a/docs/source/developer_guide/index.rst +++ b/docs/source/developer_guide/index.rst @@ -22,3 +22,4 @@ Developer's guide ../verdi/properties database_schema control/index + caching diff --git a/docs/source/user_guide/index.rst b/docs/source/user_guide/index.rst index b2c8857f69..79ace8d9e7 100644 --- a/docs/source/user_guide/index.rst +++ b/docs/source/user_guide/index.rst @@ -8,5 +8,6 @@ User's guide ../get_started/index ../working_with_aiida/index ../tutorial/index + ../caching/index ../import_export/index - ../restapi/index \ No newline at end of file + ../restapi/index diff --git a/setup_requirements.py b/setup_requirements.py index fb74112c91..18c54714db 100644 --- a/setup_requirements.py +++ b/setup_requirements.py @@ -19,10 +19,10 @@ 'django-extensions==1.5.0', 'tzlocal==1.3', 'pytz==2014.10', + 'pyyaml', 'six==1.11.0', 'future==0.16.0', 'pathlib2==2.3.0', - 'singledispatch==3.4.0.3', # We need for the time being to stay with an old version # of celery, including the versions of the AMQP libraries below, # because the support for a SQLA broker has been dropped in later @@ -41,7 +41,7 @@ 'psutil==5.4.0', 'meld3==1.0.0', 'numpy==1.12.0', - 'plumpy==0.7.10', + 'plumpy==0.7.11', 'portalocker==1.1.0', 'SQLAlchemy==1.0.12', # upgrade to SQLalchemy 1.1.5 does break tests, see #465 'SQLAlchemy-Utils==0.31.2', @@ -70,6 +70,8 @@ ] extras_require = { + # Requirements for Python 2 only + ':python_version < "3"': ['chainmap', 'pathlib2', 'singledispatch >= 3.4.0.3'], # Requirements for ssh transport with authentification through Kerberos # token # N. B.: you need to install first libffi and MIT kerberos GSSAPI including header files.