From 24e446f9f548dbd7dac624c7ac4391398b2cbce6 Mon Sep 17 00:00:00 2001 From: Leopold Talirz Date: Fri, 16 Nov 2018 12:42:33 +0100 Subject: [PATCH 1/2] Added caching of entity collections and default user Now if a new collection is created via .objects it is cached for the next time in a lazy store. Similarly the default user for a User collection is cached after being created for the first time. This may lead to issues if the default user can change for a given profile during runtime, however for now we don't support this. --- aiida/backends/testbase.py | 3 +- aiida/backends/tests/__init__.py | 1 + .../tests/cmdline/commands/test_process.py | 4 +- aiida/backends/tests/orm/entities.py | 30 +++++++ aiida/backends/tests/work/test_futures.py | 17 ++-- aiida/backends/tests/work/test_rmq.py | 26 +++---- aiida/backends/tests/work/test_runners.py | 4 +- aiida/backends/tests/work/work_chain.py | 11 +-- aiida/cmdline/commands/cmd_process.py | 13 ++-- aiida/common/datastructures.py | 50 +++++++++--- aiida/daemon/runner.py | 3 +- aiida/{work => manage}/manager.py | 78 ++++++++++--------- aiida/orm/backends.py | 4 +- aiida/orm/entities.py | 53 +++++++++++-- aiida/orm/implementation/django/node.py | 1 + aiida/orm/implementation/sqlalchemy/node.py | 3 +- aiida/orm/users.py | 30 ++++--- aiida/work/__init__.py | 25 +++--- aiida/work/launch.py | 2 +- aiida/work/processes.py | 4 +- aiida/work/workfunctions.py | 3 +- docs/requirements_for_rtd.txt | 4 +- setup_requirements.py | 4 +- 23 files changed, 241 insertions(+), 132 deletions(-) create mode 100644 aiida/backends/tests/orm/entities.py rename aiida/{work => manage}/manager.py (79%) diff --git a/aiida/backends/testbase.py b/aiida/backends/testbase.py index 0fe4b71a9a..eb46a2a3bc 100644 --- a/aiida/backends/testbase.py +++ b/aiida/backends/testbase.py @@ -24,6 +24,7 @@ from aiida.common.exceptions import ConfigurationError, TestsNotAllowedError, InternalError from aiida.common.utils import classproperty from aiida import work +from aiida.manage.manager import AiiDAManager def check_if_tests_can_run(): @@ -113,7 +114,7 @@ def tearDown(self): self.__backend_instance.tearDown_method() # Clean up the loop we created in set up. # Call this after the instance tear down just in case it uses the loop - work.AiiDAManager.reset() + AiiDAManager.reset() loop = ioloop.IOLoop.current() if not loop._closing: loop.close() diff --git a/aiida/backends/tests/__init__.py b/aiida/backends/tests/__init__.py index 2ff30b1d46..0ff460b647 100644 --- a/aiida/backends/tests/__init__.py +++ b/aiida/backends/tests/__init__.py @@ -91,6 +91,7 @@ 'orm.computer': ['aiida.backends.tests.computer'], 'orm.data.frozendict': ['aiida.backends.tests.orm.data.frozendict'], 'orm.data.remote': ['aiida.backends.tests.orm.data.remote'], + 'orm.entities': ['aiida.backends.tests.orm.entities'], 'orm.log': ['aiida.backends.tests.orm.log'], 'orm.mixins': ['aiida.backends.tests.orm.mixins'], 'orm.node': ['aiida.backends.tests.orm.node.test_node'], diff --git a/aiida/backends/tests/cmdline/commands/test_process.py b/aiida/backends/tests/cmdline/commands/test_process.py index aad6426110..e38f3cbc94 100644 --- a/aiida/backends/tests/cmdline/commands/test_process.py +++ b/aiida/backends/tests/cmdline/commands/test_process.py @@ -29,7 +29,7 @@ from aiida.common.log import LOG_LEVEL_REPORT from aiida.orm.node.process import ProcessNode, WorkFunctionNode, WorkChainNode from aiida.work import test_utils -from aiida import work +from aiida.manage.manager import AiiDAManager def get_result_lines(result): @@ -50,7 +50,7 @@ def setUp(self): self.daemon_client = DaemonClient(profile) self.daemon_pid = subprocess.Popen( self.daemon_client.cmd_string.split(), stderr=sys.stderr, stdout=sys.stdout).pid - self.runner = work.AiiDAManager.create_runner(rmq_submit=True) + self.runner = AiiDAManager.create_runner(rmq_submit=True) self.cli_runner = CliRunner() def tearDown(self): diff --git a/aiida/backends/tests/orm/entities.py b/aiida/backends/tests/orm/entities.py new file mode 100644 index 0000000000..4426554eae --- /dev/null +++ b/aiida/backends/tests/orm/entities.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +########################################################################### +# Copyright (c), The AiiDA team. All rights reserved. # +# This file is part of the AiiDA code. # +# # +# The code is hosted on GitHub at https://github.com/aiidateam/aiida_core # +# For further information on the license, see the LICENSE.txt file # +# For further information please visit http://www.aiida.net # +########################################################################### +"""Test for general backend entities""" +from __future__ import division +from __future__ import print_function +from __future__ import absolute_import + +from aiida.backends.testbase import AiidaTestCase +from aiida import orm + + +class TestBackendEntitiesAndCollections(AiidaTestCase): + """Test backend entities and their collections""" + + def test_collections_cache(self): + """Make sure that we're not recreating collections each time .objects is called""" + # Check directly + user_collection = orm.User.objects + self.assertIs(user_collection, orm.User.objects) + + # Now check passing an explicit backend + backend = user_collection.backend + self.assertIs(user_collection, user_collection(backend)) diff --git a/aiida/backends/tests/work/test_futures.py b/aiida/backends/tests/work/test_futures.py index b02a6af4b9..51da201ecf 100644 --- a/aiida/backends/tests/work/test_futures.py +++ b/aiida/backends/tests/work/test_futures.py @@ -16,23 +16,22 @@ from tornado import gen from aiida.backends.testbase import AiidaTestCase -from aiida.work import futures -from aiida.work import test_utils from aiida import work +from aiida.manage.manager import AiiDAManager class TestWf(AiidaTestCase): TIMEOUT = datetime.timedelta(seconds=5.0) def test_calculation_future_broadcasts(self): - runner = work.AiiDAManager.get_runner() - process = test_utils.DummyProcess() + runner = AiiDAManager.get_runner() + process = work.test_utils.DummyProcess() # No polling - future = futures.CalculationFuture( + future = work.futures.CalculationFuture( pk=process.pid, poll_interval=None, - communicator=work.AiiDAManager.get_communicator()) + communicator=AiiDAManager.get_communicator()) work.run(process) calc_node = runner.run_until_complete(gen.with_timeout(self.TIMEOUT, future)) @@ -40,11 +39,11 @@ def test_calculation_future_broadcasts(self): self.assertEqual(process.calc.pk, calc_node.pk) def test_calculation_future_polling(self): - runner = work.AiiDAManager.get_runner() - process = test_utils.DummyProcess() + runner = AiiDAManager.get_runner() + process = work.test_utils.DummyProcess() # No communicator - future = futures.CalculationFuture( + future = work.futures.CalculationFuture( pk=process.pid, loop=runner.loop, poll_interval=0) diff --git a/aiida/backends/tests/work/test_rmq.py b/aiida/backends/tests/work/test_rmq.py index f4c8de4ea3..2fcc2abcdc 100644 --- a/aiida/backends/tests/work/test_rmq.py +++ b/aiida/backends/tests/work/test_rmq.py @@ -17,8 +17,8 @@ from aiida.backends.testbase import AiidaTestCase from aiida.orm.data.int import Int -from aiida.work import test_utils from aiida import work +from aiida.manage.manager import AiiDAManager class TestProcessControl(AiidaTestCase): @@ -33,8 +33,8 @@ def setUp(self): # These two need to share a common event loop otherwise the first will never send # the message while the daemon is running listening to intercept - self.runner = work.AiiDAManager.get_runner() - self.daemon_runner = work.AiiDAManager.create_daemon_runner(loop=self.runner.loop) + self.runner = AiiDAManager.get_runner() + self.daemon_runner = AiiDAManager.create_daemon_runner(loop=self.runner.loop) def tearDown(self): self.daemon_runner.close() @@ -44,7 +44,7 @@ def test_submit_simple(self): # Launch the process @gen.coroutine def do_submit(): - calc_node = work.submit(test_utils.DummyProcess) + calc_node = work.submit(work.test_utils.DummyProcess) yield self.wait_for_calc(calc_node) self.assertTrue(calc_node.is_finished_ok) @@ -58,7 +58,7 @@ def do_launch(): a = Int(5) b = Int(10) - calc_node = work.submit(test_utils.AddProcess, a=a, b=b) + calc_node = work.submit(work.test_utils.AddProcess, a=a, b=b) yield self.wait_for_calc(calc_node) self.assertTrue(calc_node.is_finished_ok) self.assertEqual(calc_node.process_state.value, plumpy.ProcessState.FINISHED.value) @@ -67,12 +67,12 @@ def do_launch(): def test_submit_bad_input(self): with self.assertRaises(ValueError): - work.submit(test_utils.AddProcess, a=Int(5)) + work.submit(work.test_utils.AddProcess, a=Int(5)) def test_exception_process(self): @gen.coroutine def do_exception(): - calc_node = work.submit(test_utils.ExceptionProcess) + calc_node = work.submit(work.test_utils.ExceptionProcess) yield self.wait_for_calc(calc_node) self.assertFalse(calc_node.is_finished_ok) @@ -83,11 +83,11 @@ def do_exception(): def test_pause(self): """Testing sending a pause message to the process.""" - controller = work.AiiDAManager.get_process_controller() + controller = AiiDAManager.get_process_controller() @gen.coroutine def do_pause(): - calc_node = work.submit(test_utils.WaitProcess) + calc_node = work.submit(work.test_utils.WaitProcess) while calc_node.process_state != work.ProcessState.WAITING: yield @@ -103,11 +103,11 @@ def do_pause(): def test_pause_play(self): """Test sending a pause and then a play message.""" - controller = work.AiiDAManager.get_process_controller() + controller = AiiDAManager.get_process_controller() @gen.coroutine def do_pause_play(): - calc_node = work.submit(test_utils.WaitProcess) + calc_node = work.submit(work.test_utils.WaitProcess) self.assertFalse(calc_node.paused) while calc_node.process_state != work.ProcessState.WAITING: yield @@ -129,11 +129,11 @@ def do_pause_play(): def test_kill(self): """Test sending a kill message.""" - controller = work.AiiDAManager.get_process_controller() + controller = AiiDAManager.get_process_controller() @gen.coroutine def do_kill(): - calc_node = work.submit(test_utils.WaitProcess) + calc_node = work.submit(work.test_utils.WaitProcess) self.assertFalse(calc_node.is_killed) while calc_node.process_state != work.ProcessState.WAITING: yield diff --git a/aiida/backends/tests/work/test_runners.py b/aiida/backends/tests/work/test_runners.py index 1b62d58184..8637356633 100644 --- a/aiida/backends/tests/work/test_runners.py +++ b/aiida/backends/tests/work/test_runners.py @@ -16,8 +16,8 @@ from aiida.backends.testbase import AiidaTestCase from aiida import work from aiida.daemon.workflowmanager import execute_steps -from aiida.work import runners from aiida.workflows.wf_demo import WorkflowDemo +from aiida.manage.manager import AiiDAManager class Proc(work.Process): @@ -33,7 +33,7 @@ class TestWorkchain(AiidaTestCase): def setUp(self): super(TestWorkchain, self).setUp() - self.runner = work.AiiDAManager.get_runner() + self.runner = AiiDAManager.get_runner() def tearDown(self): super(TestWorkchain, self).tearDown() diff --git a/aiida/backends/tests/work/work_chain.py b/aiida/backends/tests/work/work_chain.py index 8a6237f650..f29e13db5b 100644 --- a/aiida/backends/tests/work/work_chain.py +++ b/aiida/backends/tests/work/work_chain.py @@ -32,6 +32,7 @@ from aiida.work import ExitCode, Process from aiida.work.persistence import ObjectLoader from aiida.work.workchain import * +from aiida.manage.manager import AiiDAManager def run_until_paused(proc): @@ -563,7 +564,7 @@ def test_if_block_persistence(self): """ This test was created to capture issue #902 """ - runner = work.AiiDAManager.get_runner() + runner = AiiDAManager.get_runner() wc = IfTest() runner.schedule(wc) @@ -649,7 +650,7 @@ def result(self): def test_persisting(self): persister = plumpy.test_utils.TestPersister() - runner = work.AiiDAManager.get_runner() + runner = AiiDAManager.get_runner() workchain = Wf(runner=runner) work.run(workchain) @@ -817,7 +818,7 @@ def test_simple_run(self): Run the workchain which should hit the exception and therefore end up in the EXCEPTED state """ - runner = work.AiiDAManager.get_runner() + runner = AiiDAManager.get_runner() process = TestWorkChainAbort.AbortableWorkChain() @gen.coroutine @@ -843,7 +844,7 @@ def test_simple_kill_through_process(self): on the workchain itself. This should have the workchain end up in the KILLED state. """ - runner = work.AiiDAManager.get_runner() + runner = AiiDAManager.get_runner() process = TestWorkChainAbort.AbortableWorkChain() @gen.coroutine @@ -936,7 +937,7 @@ def test_simple_kill_through_process(self): Run the workchain for one step and then kill it. This should have the workchain and its children end up in the KILLED state. """ - runner = work.AiiDAManager.get_runner() + runner = AiiDAManager.get_runner() process = TestWorkChainAbortChildren.MainWorkChain(inputs={'kill': Bool(True)}) @gen.coroutine diff --git a/aiida/cmdline/commands/cmd_process.py b/aiida/cmdline/commands/cmd_process.py index bdfa64bd79..8359a2f1ad 100644 --- a/aiida/cmdline/commands/cmd_process.py +++ b/aiida/cmdline/commands/cmd_process.py @@ -18,6 +18,7 @@ from aiida.cmdline.utils import decorators, echo from aiida.cmdline.utils.query.calculation import CalculationQueryBuilder from aiida.common.log import LOG_LEVELS +from aiida.manage.manager import AiiDAManager @verdi.group('process') @@ -118,9 +119,8 @@ def process_status(processes): @decorators.only_if_daemon_running(echo.echo_warning, 'daemon is not running, so process may not be reachable') def process_kill(processes, timeout, wait): """Kill running processes.""" - from aiida import work - controller = work.AiiDAManager.get_process_controller() + controller = AiiDAManager.get_process_controller() futures = {} for process in processes: @@ -146,9 +146,8 @@ def process_kill(processes, timeout, wait): @decorators.only_if_daemon_running(echo.echo_warning, 'daemon is not running, so process may not be reachable') def process_pause(processes, timeout, wait): """Pause running processes.""" - from aiida import work - controller = work.AiiDAManager.get_process_controller() + controller = AiiDAManager.get_process_controller() futures = {} for process in processes: @@ -174,9 +173,8 @@ def process_pause(processes, timeout, wait): @decorators.only_if_daemon_running(echo.echo_warning, 'daemon is not running, so process may not be reachable') def process_play(processes, timeout, wait): """Play paused processes.""" - from aiida import work - controller = work.AiiDAManager.get_process_controller() + controller = AiiDAManager.get_process_controller() futures = {} for process in processes: @@ -198,13 +196,12 @@ def process_play(processes, timeout, wait): def process_watch(processes): """Watch the state transitions for a process.""" from kiwipy import BroadcastFilter - from aiida import work import concurrent.futures def _print(body, sender, subject, correlation_id): echo.echo('pk={}, subject={}, body={}, correlation_id={}'.format(sender, subject, body, correlation_id)) - communicator = work.AiiDAManager.get_communicator() + communicator = AiiDAManager.get_communicator() for process in processes: diff --git a/aiida/common/datastructures.py b/aiida/common/datastructures.py index c729007904..122c99af3e 100644 --- a/aiida/common/datastructures.py +++ b/aiida/common/datastructures.py @@ -15,9 +15,11 @@ from __future__ import absolute_import from aiida.common.extendeddicts import DefaultFieldsAttributeDict, Enumerate + class CalcState(Enumerate): pass + _sorted_datastates = ( 'NEW', # just created 'TOSUBMIT', # used by the executionmanager to submit new calculations scheduled to be submitted @@ -175,6 +177,7 @@ class CalcInfo(DefaultFieldsAttributeDict): class CodeRunmode(Enumerate): pass + # these are the possible ways to execute more than one code in the same scheduling job # if parallel, the codes will be executed as something like: # code1.x & @@ -279,15 +282,38 @@ class WorkflowDataValueType(Enumerate): wf_exit_call = "exit" wf_default_call = "none" -# TODO Improve/implement this! -# class DynResourcesInfo(AttributeDict): -# """ -# This object will contain a list of 'dynamical' resources to be -# passed from the code plugin to the ExecManager, containing -# things like -# * resources in the permanent repository, that will be simply -# linked locally (but copied remotely on the remote computer) -# to avoid a waste of permanent repository space -# * remote resources to be directly copied over only remotely -# """ -# pass + +class LazyStore(object): + """ + A container that provides a mapping to objects based on a key, if the object is not + found in the container when it is retrieved it will created using a provided factory + method + """ + + def __init__(self): + self._store = {} + + def get(self, key, factory): + """ + Get a value in the store based on the key, if it doesn't exist it will be created + using the factory method and returned + + :param key: the key of the object to get + :param factory: the factory used to create the object if necessary + :return: the object + """ + try: + return self._store[key] + except KeyError: + obj = factory() + self._store[key] = obj + return obj + + def pop(self, key): + """ + Pop an object from the store based on the given key + + :param key: the object key + :return: the object that was popped + """ + return self._store.pop(key) diff --git a/aiida/daemon/runner.py b/aiida/daemon/runner.py index 490a51bbb2..47584c949a 100644 --- a/aiida/daemon/runner.py +++ b/aiida/daemon/runner.py @@ -18,6 +18,7 @@ from aiida.common.log import configure_logging from aiida.daemon.client import get_daemon_client from aiida import work +from aiida.manage.manager import AiiDAManager logger = logging.getLogger(__name__) @@ -33,7 +34,7 @@ def start_daemon(): configure_logging(daemon=True, daemon_log_file=daemon_client.daemon_log_file) try: - runner = work.AiiDAManager.create_daemon_runner() + runner = AiiDAManager.create_daemon_runner() except Exception as exception: logger.exception('daemon runner failed to start') raise diff --git a/aiida/work/manager.py b/aiida/manage/manager.py similarity index 79% rename from aiida/work/manager.py rename to aiida/manage/manager.py index 177d46e9fb..20fea50c94 100644 --- a/aiida/work/manager.py +++ b/aiida/manage/manager.py @@ -13,13 +13,9 @@ from __future__ import absolute_import import functools -import kiwipy.rmq import plumpy -from aiida.work import rmq from aiida import utils -from . import persistence -from . import runners __all__ = ('AiiDAManager',) @@ -35,13 +31,17 @@ class AiiDAManager(object): to create objects based on the current settings. It is also a useful place to put objects where there can be a single 'global' (per profile) instance. + + Future plans: + * reset manager cache when loading a new profile + * move construct_backend() from orm.backends inside the AiiDAManager """ - _PROFILE = None - _COMMUNICATOR = None # type: kiwipy.rmq.RmqThreadCommunicator - _PROCESS_CONTROLLER = None # type: plumpy.RemoteProcessThreadController - _PERSISTER = None # type: aiida.work.AiiDAPersister - _RUNNER = None # type: aiida.work.Runer + _profile = None # type: aiida.common.profile.Profile + _communicator = None # type: kiwipy.rmq.RmqThreadCommunicator + _process_controller = None # type: plumpy.RemoteProcessThreadController + _persister = None # type: aiida.work.AiiDAPersister + _runner = None # type: aiida.work.Runner @classmethod def get_profile(cls): @@ -53,10 +53,10 @@ def get_profile(cls): """ from aiida.common import profile - if cls._PROFILE is None: - cls._PROFILE = profile.get_profile() + if cls._profile is None: + cls._profile = profile.get_profile() - return cls._PROFILE + return cls._profile @classmethod def get_persister(cls): @@ -66,10 +66,12 @@ def get_persister(cls): :return: the current persister instance :rtype: :class:`plumpy.Persister` """ - if cls._PERSISTER is None: - cls._PERSISTER = persistence.AiiDAPersister() + from aiida.work import persistence + + if cls._persister is None: + cls._persister = persistence.AiiDAPersister() - return cls._PERSISTER + return cls._persister @classmethod def get_communicator(cls): @@ -79,10 +81,10 @@ def get_communicator(cls): :return: a global communicator instance :rtype: :class:`kiwipy.Communicator` """ - if cls._COMMUNICATOR is None: - cls._COMMUNICATOR = cls.create_communicator() + if cls._communicator is None: + cls._communicator = cls.create_communicator() - return cls._COMMUNICATOR + return cls._communicator @classmethod def create_communicator(cls, task_prefetch_count=None): @@ -93,6 +95,8 @@ def create_communicator(cls, task_prefetch_count=None): :return: the communicator instance :rtype: :class:`~kiwipy.rmq.communicator.RmqThreadCommunicator` """ + from aiida.work import rmq + import kiwipy.rmq profile = cls.get_profile() if task_prefetch_count is None: @@ -128,10 +132,10 @@ def get_process_controller(cls): :return: the process controller instance :rtype: :class:`plumpy.RemoteProcessThreadController` """ - if cls._PROCESS_CONTROLLER is None: - cls._PROCESS_CONTROLLER = plumpy.RemoteProcessThreadController(cls.get_communicator()) + if cls._process_controller is None: + cls._process_controller = plumpy.RemoteProcessThreadController(cls.get_communicator()) - return cls._PROCESS_CONTROLLER + return cls._process_controller @classmethod def get_runner(cls): @@ -141,10 +145,10 @@ def get_runner(cls): :return: the global runner :rtype: :class:`aiida.work.Runner` """ - if cls._RUNNER is None: - cls._RUNNER = cls.create_runner() + if cls._runner is None: + cls._runner = cls.create_runner() - return cls._RUNNER + return cls._runner @classmethod def set_runner(cls, new_runner): @@ -154,10 +158,10 @@ def set_runner(cls, new_runner): :param new_runner: the new runner to use :type new_runner: :class:`aiida.work.Runner` """ - if cls._RUNNER is not None: - cls._RUNNER.close() + if cls._runner is not None: + cls._runner.close() - cls._RUNNER = new_runner + cls._runner = new_runner @classmethod def create_runner(cls, with_persistence=True, **kwargs): @@ -169,6 +173,8 @@ def create_runner(cls, with_persistence=True, **kwargs): :return: a new runner instance :rtype: :class:`aiida.work.Runner` """ + from aiida.work import runners + profile = cls.get_profile() poll_interval = 0.0 if profile.is_test_profile else profile.get_option('runner.poll.interval') @@ -194,6 +200,7 @@ def create_daemon_runner(cls, loop=None): :return: a runner configured to work in the daemon configuration :rtype: :class:`aiida.work.Runner` """ + from aiida.work import rmq, persistence runner = cls.create_runner(rmq_submit=True, loop=loop) runner_loop = runner.loop @@ -216,15 +223,16 @@ def reset(cls): """ Reset the global settings entirely and release any global objects """ - if cls._COMMUNICATOR is not None: - cls._COMMUNICATOR.stop() - if cls._RUNNER is not None: - cls._RUNNER.stop() + if cls._communicator is not None: + cls._communicator.stop() + if cls._runner is not None: + cls._runner.stop() - cls._PROFILE = None - cls._COMMUNICATOR = None - cls._PROCESS_CONTROLLER = None - cls._RUNNER = None + cls._profile = None + cls._persister = None + cls._communicator = None + cls._process_controller = None + cls._runner = None def __init__(self): """Can't instantiate this class""" diff --git a/aiida/orm/backends.py b/aiida/orm/backends.py index f683db5c85..6149c5cd2d 100644 --- a/aiida/orm/backends.py +++ b/aiida/orm/backends.py @@ -12,6 +12,8 @@ from __future__ import print_function from __future__ import absolute_import +from aiida.backends import settings + __all__ = 'construct_backend', 'CollectionEntry' _DJANGO_BACKEND = None @@ -27,7 +29,7 @@ def construct_backend(backend_type=None): """ # pylint: disable=global-statement if backend_type is None: - from aiida.backends import settings + backend_type = settings.BACKEND if backend_type == 'django': diff --git a/aiida/orm/entities.py b/aiida/orm/entities.py index 2f028e7d30..6561cffab2 100644 --- a/aiida/orm/entities.py +++ b/aiida/orm/entities.py @@ -12,16 +12,38 @@ from __future__ import print_function from __future__ import absolute_import +import typing + from aiida.common import exceptions +from aiida.common import datastructures from aiida.common.utils import classproperty, type_check +from . import backends __all__ = ('Entity', 'Collection') +EntityType = typing.TypeVar('EntityType') # pylint: disable=invalid-name + -class Collection(object): +class Collection(typing.Generic[EntityType]): """Container class that represents the collection of objects of a particular type.""" + # A store for any backend specific collections that already exist + _COLLECTIONS = datastructures.LazyStore() + + @classmethod + def get_collection(cls, entity_type, backend): + """ + Get the collection for a given entity type and backend instance + + :param entity_type: the entity type e.g. User, Computer, etc + :param backend: the backend instance to get the collection for + :return: the collection instance + """ + # Lazily get the collection i.e. create only if we haven't done so yet + return cls._COLLECTIONS.get((entity_type, backend), lambda: entity_type.Collection(backend, entity_type)) + def __init__(self, backend, entity_class): + """Construct a new entity collection""" # assert issubclass(entity_class, Entity), "Must provide an entity type" if backend is None: from . import backends @@ -40,7 +62,7 @@ def __call__(self, backend): # Special case if they actually want the same collection return self - return self.__class__(backend, self._entity_type) + return self.get_collection(self.entity_type, backend) @property def backend(self): @@ -114,8 +136,8 @@ def all(self): class Entity(object): """An AiiDA entity""" - _BACKEND = None - _OBJECTS = None + _backend = None + _objects = None # Define out collection type Collection = Collection @@ -126,12 +148,27 @@ def objects(cls, backend=None): # pylint: disable=no-self-use, no-self-argument Get an collection for objects of this type. :param backend: the optional backend to use (otherwise use default) - :return: an object that can be used to access entites of this type + :return: an object that can be used to access entities of this type """ - from . import backends + new_backend = backend or cls._backend or backends.construct_backend() + + if type(cls._backend) != type(new_backend): # pylint: disable=unidiomatic-typecheck + # if the backend has changed, update cache + cls._backend = new_backend + cls._objects = cls.Collection(cls._backend, cls) + else: + # if not, use cache (if already populated) + cls._objects = cls._objects or cls.Collection(cls._backend, cls) + + return cls._objects - backend = backend or backends.construct_backend() - return cls.Collection(backend, cls) + @classmethod + def reset(cls): + """ + Reset the backend and objects cache. + """ + cls._backend = None + cls._backendobjectS = None @classmethod def get(cls, **kwargs): diff --git a/aiida/orm/implementation/django/node.py b/aiida/orm/implementation/django/node.py index 1be70bf722..5b70dfe9e4 100644 --- a/aiida/orm/implementation/django/node.py +++ b/aiida/orm/implementation/django/node.py @@ -26,6 +26,7 @@ from aiida.common.utils import get_new_uuid, type_check from aiida.orm.implementation.general.node import AbstractNode, _HASH_EXTRA_KEY from . import computer as computers +from aiida.manage.manager import AiiDAManager class Node(AbstractNode): diff --git a/aiida/orm/implementation/sqlalchemy/node.py b/aiida/orm/implementation/sqlalchemy/node.py index 0aa7112e9c..d5a9365454 100644 --- a/aiida/orm/implementation/sqlalchemy/node.py +++ b/aiida/orm/implementation/sqlalchemy/node.py @@ -20,7 +20,6 @@ from aiida.backends.sqlalchemy.models.node import DbNode, DbLink from aiida.backends.sqlalchemy.models.comment import DbComment -from aiida.backends.sqlalchemy.models.computer import DbComputer from aiida.backends.sqlalchemy.utils import flag_modified from aiida.common.utils import get_new_uuid from aiida.common.folders import RepositoryFolder @@ -29,6 +28,7 @@ from aiida.common.utils import type_check from aiida.orm.implementation.general.node import AbstractNode, _HASH_EXTRA_KEY from aiida.orm.implementation.sqlalchemy.utils import get_attr +from aiida.manage.manager import AiiDAManager from . import computer as computers @@ -69,6 +69,7 @@ def __init__(self, **kwargs): else: user = orm.User.objects(backend=self._backend).get_default().backend_entity + if user is None: raise RuntimeError("Could not find a default user") diff --git a/aiida/orm/users.py b/aiida/orm/users.py index 53155ea12e..73f9391da5 100644 --- a/aiida/orm/users.py +++ b/aiida/orm/users.py @@ -31,12 +31,16 @@ class User(entities.Entity): backend needs to provide a concrete version. """ - # pylint: disable=invalid-name - class Collection(entities.Collection): """ The collection of users stored in a backend """ + UNDEFINED = 'UNDEFINED' + _default_user = None # type: aiida.orm.user.User + + def __init__(self, *args, **kwargs): + super(User.Collection, self).__init__(*args, **kwargs) + self._default_user = self.UNDEFINED def get_or_create(self, **kwargs): """ @@ -59,16 +63,18 @@ def get_default(self): :return: The default user :rtype: :class:`aiida.orm.User` """ - from aiida.common.utils import get_configured_user_email - - email = get_configured_user_email() - if not email: - return None - - try: - return self.get(email=email) - except (exceptions.MultipleObjectsError, exceptions.NotExistent): - return None + if self._default_user is self.UNDEFINED: + from aiida.common.utils import get_configured_user_email + email = get_configured_user_email() + if not email: + self._default_user = None + + try: + self._default_user = self.get(email=email) + except (exceptions.MultipleObjectsError, exceptions.NotExistent): + self._default_user = None + + return self._default_user REQUIRED_FIELDS = ['first_name', 'last_name', 'institution'] diff --git a/aiida/work/__init__.py b/aiida/work/__init__.py index 1b4c78b0d7..778bf203c6 100644 --- a/aiida/work/__init__.py +++ b/aiida/work/__init__.py @@ -16,7 +16,6 @@ from .futures import * from .job_processes import * from .launch import * -from .manager import * from .persistence import * from .processes import * from .process_function import * @@ -25,17 +24,15 @@ from .utils import * from .workchain import * - __all__ = ( - exceptions.__all__ + - exit_code.__all__ + - futures.__all__ + - job_processes.__all__ + - launch.__all__ + - manager.__all__ + - persistence.__all__ + - processes.__all__ + - rmq.__all__ + - runners.__all__ + - utils.__all__ + - workchain.__all__) + exceptions.__all__ + + exit_code.__all__ + + futures.__all__ + + job_processes.__all__ + + launch.__all__ + + persistence.__all__ + + processes.__all__ + + rmq.__all__ + + runners.__all__ + + utils.__all__ + + workchain.__all__) diff --git a/aiida/work/launch.py b/aiida/work/launch.py index 4a19afce79..b3fc31c3c5 100644 --- a/aiida/work/launch.py +++ b/aiida/work/launch.py @@ -12,7 +12,7 @@ from __future__ import print_function from __future__ import absolute_import -from . import manager +from aiida.manage import manager from . import processes from . import utils diff --git a/aiida/work/processes.py b/aiida/work/processes.py index 6d7b0ab79e..f0755bc524 100644 --- a/aiida/work/processes.py +++ b/aiida/work/processes.py @@ -125,7 +125,7 @@ def get_or_create_db_record(cls): return cls._calc_class() def __init__(self, inputs=None, logger=None, runner=None, parent_pid=None, enable_persistence=True): - from . import manager + from aiida.manage import manager self._runner = runner if runner is not None else manager.AiiDAManager.get_runner() @@ -187,7 +187,7 @@ def get_provenance_inputs_iterator(self): @override def load_instance_state(self, saved_state, load_context): - from . import manager + from aiida.manage import manager if 'runner' in load_context: self._runner = load_context.runner diff --git a/aiida/work/workfunctions.py b/aiida/work/workfunctions.py index 29a6df142a..c5823438fb 100644 --- a/aiida/work/workfunctions.py +++ b/aiida/work/workfunctions.py @@ -17,6 +17,7 @@ from aiida.common.warnings import AiidaDeprecationWarning as DeprecationWarning # pylint: disable=redefined-builtin from .process_function import workfunction -warnings.warn('this module has been deprecated, import directly from `aiida.work` instead', DeprecationWarning) # pylint: disable=no-member +warnings.warn('this module has been deprecated, import directly from `aiida.work` instead', + DeprecationWarning) # pylint: disable=no-member __all__ = ('workfunction',) diff --git a/docs/requirements_for_rtd.txt b/docs/requirements_for_rtd.txt index b1c813ed70..1f83a19d84 100644 --- a/docs/requirements_for_rtd.txt +++ b/docs/requirements_for_rtd.txt @@ -13,7 +13,7 @@ PyYAML==3.12 Pygments==2.2.0 SQLAlchemy-Utils==0.33.0 SQLAlchemy==1.2.12 -Sphinx==1.7.7 +Sphinx==1.8.2 aldjemy==0.8.0 alembic==0.9.9 anyjson==0.3.3 @@ -61,7 +61,7 @@ simplejson==3.16.0 singledispatch>=3.4.0.3; python_version<"3.5" six==1.11.0 spglib==1.10.3.65 -sphinx-rtd-theme==0.3.1 +sphinx-rtd-theme==0.4.2 sqlalchemy-diff==0.1.3 sqlalchemy-migrate==0.11.0 tabulate==0.8.2 diff --git a/setup_requirements.py b/setup_requirements.py index f19d081ada..364143cf2e 100644 --- a/setup_requirements.py +++ b/setup_requirements.py @@ -80,12 +80,12 @@ ], # Requirements to building documentation 'docs': [ - 'Sphinx==1.7.7', + 'Sphinx==1.8.2', 'Pygments==2.2.0', 'docutils==0.14', 'Jinja2==2.10', 'MarkupSafe==1.0', - 'sphinx-rtd-theme==0.3.1', # Required by readthedocs + 'sphinx-rtd-theme==0.4.2', # Required by readthedocs ], # Requirements for non-core functionalities that rely on external atomic manipulation/processing software 'atomic_tools': [ From 2ff392aa4c66c384f4dc69809d16fcdd45d00b42 Mon Sep 17 00:00:00 2001 From: Martin Uhrin Date: Mon, 26 Nov 2018 21:57:00 +0100 Subject: [PATCH 2/2] Moved AiiDAManager to be returned by get_manager() This way we have greater control over its creation and we don't have to implement a reset method which starts to get complicate. We just throw the object away and instsantiate a new one when we need to reset * Removed construct_backend() (replaced by get_manager().get_backend()) * Changed all tests to not specify the backend, they should just do what user facing code does which is use the default one and if we ever want to test with different backends we just implement a function to load a new backend from a profile globally * Moved getting default user email to the `Profile` class --- .ci/test_fixtures.py | 8 +- .ci/test_plugin_testcase.py | 2 +- .pre-commit-config.yaml | 1 - aiida/backends/djsite/db/models.py | 20 +-- aiida/backends/djsite/db/subtests/generic.py | 4 +- aiida/backends/djsite/db/testbase.py | 35 +--- aiida/backends/sqlalchemy/models/authinfo.py | 4 +- aiida/backends/sqlalchemy/models/user.py | 4 +- aiida/backends/sqlalchemy/tests/nodes.py | 3 +- aiida/backends/sqlalchemy/tests/session.py | 16 +- aiida/backends/sqlalchemy/tests/testbase.py | 27 +-- aiida/backends/testbase.py | 13 +- aiida/backends/testimplbase.py | 14 +- .../tests/cmdline/commands/test_calcjob.py | 14 +- .../cmdline/commands/test_calculation.py | 9 +- .../tests/cmdline/commands/test_code.py | 10 +- .../tests/cmdline/commands/test_computer.py | 44 +++-- .../tests/cmdline/commands/test_data.py | 13 +- .../tests/cmdline/commands/test_export.py | 3 +- .../tests/cmdline/commands/test_node.py | 2 +- .../tests/cmdline/commands/test_process.py | 4 +- .../tests/cmdline/commands/test_user.py | 8 +- .../tests/cmdline/commands/test_workflow.py | 16 +- .../tests/cmdline/params/types/test_group.py | 6 +- aiida/backends/tests/computer.py | 8 +- aiida/backends/tests/daemon.py | 3 +- aiida/backends/tests/export_and_import.py | 24 +-- aiida/backends/tests/generic.py | 24 +-- aiida/backends/tests/nodes.py | 20 ++- aiida/backends/tests/query.py | 2 +- aiida/backends/tests/restapi.py | 2 +- aiida/backends/tests/work/test_futures.py | 9 +- aiida/backends/tests/work/test_rmq.py | 13 +- aiida/backends/tests/work/test_runners.py | 4 +- aiida/backends/tests/work/test_utils.py | 3 +- aiida/backends/tests/work/work_chain.py | 12 +- aiida/backends/tests/workflows.py | 1 - aiida/cmdline/commands/cmd_computer.py | 6 +- aiida/cmdline/commands/cmd_data/cmd_bands.py | 4 +- aiida/cmdline/commands/cmd_process.py | 10 +- aiida/cmdline/commands/cmd_user.py | 6 +- aiida/common/__init__.py | 8 + aiida/common/datastructures.py | 1 + aiida/common/profile.py | 74 ++++++++- aiida/common/utils.py | 31 +--- aiida/control/profile.py | 6 +- aiida/daemon/runner.py | 6 +- aiida/manage/__init__.py | 17 ++ aiida/manage/database/integrity.py | 5 +- aiida/manage/manager.py | 154 ++++++++++-------- aiida/orm/__init__.py | 9 +- aiida/orm/authinfos.py | 4 +- aiida/orm/backends.py | 64 -------- aiida/orm/comments.py | 4 +- aiida/orm/computers.py | 4 +- aiida/orm/data/remote.py | 2 +- aiida/orm/entities.py | 33 +--- aiida/orm/groups.py | 4 +- aiida/orm/implementation/django/node.py | 3 +- aiida/orm/implementation/django/workflow.py | 4 +- aiida/orm/implementation/general/node.py | 5 +- aiida/orm/implementation/general/workflow.py | 11 +- .../orm/implementation/sqlalchemy/__init__.py | 1 - .../orm/implementation/sqlalchemy/authinfo.py | 1 - aiida/orm/implementation/sqlalchemy/node.py | 1 - .../orm/implementation/sqlalchemy/workflow.py | 11 +- aiida/orm/logs.py | 4 +- aiida/orm/querybuilder.py | 24 +-- aiida/orm/users.py | 34 ++-- aiida/restapi/translator/code.py | 2 +- aiida/restapi/translator/computer.py | 5 +- aiida/restapi/translator/group.py | 4 +- aiida/restapi/translator/node.py | 6 +- aiida/restapi/translator/user.py | 4 +- aiida/transport/cli.py | 4 +- aiida/utils/fixtures.py | 73 +++++---- aiida/work/launch.py | 10 +- aiida/work/process_function.py | 5 +- aiida/work/processes.py | 4 +- aiida/work/workfunctions.py | 3 +- aiida/workflows/test.py | 2 +- docs/source/developer_guide/cookbook.rst | 5 +- docs/source/nitpick-exceptions | 3 + 83 files changed, 501 insertions(+), 585 deletions(-) delete mode 100644 aiida/orm/backends.py diff --git a/.ci/test_fixtures.py b/.ci/test_fixtures.py index b0fd332db8..38dc89aac3 100644 --- a/.ci/test_fixtures.py +++ b/.ci/test_fixtures.py @@ -32,15 +32,12 @@ def setUp(self): def test_create_db_cluster(self): self.fixture_manager.create_db_cluster() - self.assertTrue( - pgtest.is_server_running(self.fixture_manager.pg_cluster.cluster)) + self.assertTrue(pgtest.is_server_running(self.fixture_manager.pg_cluster.cluster)) def test_create_aiida_db(self): self.fixture_manager.create_db_cluster() self.fixture_manager.create_aiida_db() - self.assertTrue( - self.fixture_manager.postgres.db_exists( - self.fixture_manager.db_name)) + self.assertTrue(self.fixture_manager.postgres.db_exists(self.fixture_manager.db_name)) def test_create_use_destroy_profile(self): """ @@ -55,6 +52,7 @@ def test_create_use_destroy_profile(self): from aiida import is_dbenv_loaded with Capturing() as output: self.fixture_manager.create_profile() + self.assertTrue(self.fixture_manager.root_dir_ok, msg=output) self.assertTrue(self.fixture_manager.config_dir_ok, msg=output) self.assertTrue(self.fixture_manager.repo_ok, msg=output) diff --git a/.ci/test_plugin_testcase.py b/.ci/test_plugin_testcase.py index ccee82f74d..e3191291b1 100644 --- a/.ci/test_plugin_testcase.py +++ b/.ci/test_plugin_testcase.py @@ -78,7 +78,7 @@ def test_data_loaded(self): self.assertTrue(is_dbenv_loaded()) self.assertEqual(orm.load_node(self.data_pk).uuid, self.data.uuid) - self.assertEqual(orm.Computer.objects(self.backend).get(name='localhost').uuid, self.computer.uuid) + self.assertEqual(orm.Computer.objects.get(name='localhost').uuid, self.computer.uuid) def test_tear_down(self): """ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 898746c346..bf25375fd6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -189,7 +189,6 @@ aiida/common/links.py| aiida/common/orbital/__init__.py| aiida/common/orbital/realhydrogen.py| - aiida/common/profile.py| aiida/common/test_extendeddicts.py| aiida/common/test_logging.py| aiida/control/tests/test_postgres.py| diff --git a/aiida/backends/djsite/db/models.py b/aiida/backends/djsite/db/models.py index 4688d3973b..0b800ab99b 100644 --- a/aiida/backends/djsite/db/models.py +++ b/aiida/backends/djsite/db/models.py @@ -41,6 +41,7 @@ # load_dbenv() function). SCHEMA_VERSION = migrations.current_schema_version() + class AiidaQuerySet(QuerySet): def iterator(self): for obj in super(AiidaQuerySet, self).iterator(): @@ -54,7 +55,6 @@ def __iter__(self): return (x.get_aiida_class() for x in super(AiidaQuerySet, self).__iter__()) - def __getitem__(self, key): """Get item for [] operator @@ -120,8 +120,8 @@ class DbUser(AbstractBaseUser, PermissionsMixin): def get_aiida_class(self): from aiida.orm.implementation.django.users import DjangoUser - from aiida.orm.backends import construct_backend - return DjangoUser.from_dbmodel(self, construct_backend()) + from aiida.manage import get_manager + return DjangoUser.from_dbmodel(self, get_manager().get_backend()) @python_2_unicode_compatible @@ -745,7 +745,7 @@ def create_value(cls, key, value, subspecifier_value=None, bulk_create() call). """ import datetime - + import aiida.utils.json as json from aiida.utils.timezone import is_naive, make_aware, get_current_timezone @@ -1315,8 +1315,8 @@ def __str__(self): def get_aiida_class(self): from aiida.orm.implementation.django.groups import DjangoGroup - from aiida.orm.backends import construct_backend - return DjangoGroup.from_dbmodel(self, construct_backend()) + from aiida.manage import get_manager + return DjangoGroup.from_dbmodel(self, get_manager().get_backend()) @python_2_unicode_compatible @@ -1399,8 +1399,8 @@ def get_dbcomputer(cls, computer): def get_aiida_class(self): from aiida.orm.implementation.django.computer import DjangoComputer - from aiida.orm.backends import construct_backend - return DjangoComputer.from_dbmodel(self, construct_backend()) + from aiida.manage import get_manager + return DjangoComputer.from_dbmodel(self, get_manager().get_backend()) def _get_val_from_metadata(self, key): import aiida.utils.json as json @@ -1452,8 +1452,8 @@ def __str__(self): def get_aiida_class(self): from aiida.orm.implementation.django.authinfo import DjangoAuthInfo - from aiida.orm.backends import construct_backend - return DjangoAuthInfo.from_dbmodel(self, construct_backend()) + from aiida.manage import get_manager + return DjangoAuthInfo.from_dbmodel(self, get_manager().get_backend()) @python_2_unicode_compatible diff --git a/aiida/backends/djsite/db/subtests/generic.py b/aiida/backends/djsite/db/subtests/generic.py index 30230f9408..d775cfebe6 100644 --- a/aiida/backends/djsite/db/subtests/generic.py +++ b/aiida/backends/djsite/db/subtests/generic.py @@ -34,7 +34,7 @@ def test_deletion(self): workdir='/tmp/aiida').store() # # This should be possible, because nothing is using this computer - self.backend.computers.delete(newcomputer.id) + orm.Computer.objects.delete(newcomputer.id) calc_params = { 'computer': self.computer, @@ -47,7 +47,7 @@ def test_deletion(self): # This should fail, because there is at least a calculation # using this computer (the one created just above) with self.assertRaises(InvalidOperation): - self.backend.computers.delete(self.computer.id) + orm.Computer.objects.delete(self.computer.id) class TestGroupsDjango(AiidaTestCase): diff --git a/aiida/backends/djsite/db/testbase.py b/aiida/backends/djsite/db/testbase.py index 1be5968caa..377edbe0b9 100644 --- a/aiida/backends/djsite/db/testbase.py +++ b/aiida/backends/djsite/db/testbase.py @@ -40,38 +40,9 @@ class DjangoTests(AiidaTestImplementation): def setUpClass_method(self): self.clean_db() self.backend = DjangoBackend() - self.insert_data() - - def setUp_method(self): - pass - - def tearDown_method(self): - pass - - def insert_data(self): - """ - Insert default data into the DB. - """ - from django.core.exceptions import ObjectDoesNotExist # pylint: disable=import-error, no-name-in-module - - from aiida.backends.djsite.db.models import DbUser - from aiida.common.utils import get_configured_user_email - # We create the user only once: - # Otherwise, get_automatic_user() will fail when the - # user is recreated because it caches the user! - # In any case, store it in self.user though - try: - self.user = DbUser.objects.get(email=get_configured_user_email()) - except ObjectDoesNotExist: - self.user = DbUser.objects.create_user(get_configured_user_email(), 'fakepwd') - # Reqired by the calling class - self.user_email = self.user.email - - super(DjangoTests, self).insert_data() def clean_db(self): from aiida.backends.djsite.db.models import (DbComputer, DbUser, DbWorkflow, DbWorkflowStep, DbWorkflowData) - from aiida.common.utils import get_configured_user_email # Complicated way to make sure we 'unwind' all the relationships # between workflows and their children. @@ -99,9 +70,7 @@ def clean_db(self): DbNode.objects.all().delete() # pylint: disable=no-member - # I delete all the users except the default user. - # See discussion in setUpClass - DbUser.objects.exclude(email=get_configured_user_email()).delete() + DbUser.objects.all().delete() # pylint: disable=no-member DbComputer.objects.all().delete() @@ -123,8 +92,6 @@ def tearDownClass_method(self): "the repository. Repository path: " "{}".format(REPOSITORY_PATH)) - self.clean_db() - # I clean the test repository shutil.rmtree(REPOSITORY_PATH, ignore_errors=True) os.makedirs(REPOSITORY_PATH) diff --git a/aiida/backends/sqlalchemy/models/authinfo.py b/aiida/backends/sqlalchemy/models/authinfo.py index 9b6e2bb0bd..9155d09ecb 100644 --- a/aiida/backends/sqlalchemy/models/authinfo.py +++ b/aiida/backends/sqlalchemy/models/authinfo.py @@ -55,5 +55,5 @@ def __str__(self): def get_aiida_class(self): from aiida.orm.implementation.sqlalchemy.authinfo import SqlaAuthInfo - from aiida.orm.backends import construct_backend - return SqlaAuthInfo.from_dbmodel(dbmodel=self, backend=construct_backend()) + from aiida.manage import get_manager + return SqlaAuthInfo.from_dbmodel(dbmodel=self, backend=get_manager().get_backend()) diff --git a/aiida/backends/sqlalchemy/models/user.py b/aiida/backends/sqlalchemy/models/user.py index d6d16e9e06..c239f3843d 100644 --- a/aiida/backends/sqlalchemy/models/user.py +++ b/aiida/backends/sqlalchemy/models/user.py @@ -50,5 +50,5 @@ def __str__(self): def get_aiida_class(self): from aiida.orm.implementation.sqlalchemy.users import SqlaUser - from aiida.orm.backends import construct_backend - return SqlaUser.from_dbmodel(self, construct_backend()) + from aiida.manage import get_manager + return SqlaUser.from_dbmodel(self, get_manager().get_backend()) diff --git a/aiida/backends/sqlalchemy/tests/nodes.py b/aiida/backends/sqlalchemy/tests/nodes.py index 71677d8d4a..3816e373bc 100644 --- a/aiida/backends/sqlalchemy/tests/nodes.py +++ b/aiida/backends/sqlalchemy/tests/nodes.py @@ -307,7 +307,6 @@ def test_multiple_node_creation(self): """ from aiida.backends.sqlalchemy.models.node import DbNode from aiida.common.utils import get_new_uuid - from aiida.orm.implementation.sqlalchemy import users as users import aiida.backends.sqlalchemy backend = self.backend @@ -336,7 +335,7 @@ def test_multiple_node_creation(self): "UUID in the session/DB.") # Get the automatic user - dbuser = orm.User.objects(self.backend).get_default().backend_entity.dbmodel + dbuser = orm.User.objects.get_default().backend_entity.dbmodel # Create a new node but now add it to the session node_uuid = get_new_uuid() node = DbNode(user=dbuser, uuid=node_uuid, type=None) diff --git a/aiida/backends/sqlalchemy/tests/session.py b/aiida/backends/sqlalchemy/tests/session.py index 82d988c1dc..8b45b0b577 100644 --- a/aiida/backends/sqlalchemy/tests/session.py +++ b/aiida/backends/sqlalchemy/tests/session.py @@ -14,12 +14,11 @@ from __future__ import division from __future__ import print_function from __future__ import absolute_import -import os from sqlalchemy.orm import sessionmaker import aiida.backends from aiida.backends.testbase import AiidaTestCase -from aiida.common.utils import get_configured_user_email +from aiida.manage import get_manager class TestSessionSqla(AiidaTestCase): @@ -65,7 +64,8 @@ def test_session_update_and_expiration_1(self): session = aiida.backends.sqlalchemy.get_scoped_session() - user = self.backend.users.create(email=get_configured_user_email()) + email = get_manager().get_profile().default_user_email + user = self.backend.users.create(email=email) session.add(user.dbmodel) session.commit() @@ -89,14 +89,14 @@ def test_session_update_and_expiration_2(self): expire_on_commit=True & committing computer and code objects with their built-in store function. """ - from aiida.backends.sqlalchemy.models.user import DbUser from aiida.orm.data.code import Code session = aiida.backends.sqlalchemy.get_scoped_session() self.set_connection(expire_on_commit=True) - user = self.backend.users.create(email=get_configured_user_email()) + email = get_manager().get_profile().default_user_email + user = self.backend.users.create(email=email) session.add(user.dbmodel) session.commit() @@ -124,7 +124,8 @@ def test_session_update_and_expiration_3(self): session = aiida.backends.sqlalchemy.get_scoped_session() - user = self.backend.users.create(email=get_configured_user_email()) + email = get_manager().get_profile().default_user_email + user = self.backend.users.create(email=email) session.add(user.dbmodel) session.commit() @@ -154,7 +155,8 @@ def test_session_update_and_expiration_4(self): session = aiida.backends.sqlalchemy.get_scoped_session() - user = self.backend.users.create(email=get_configured_user_email()) + email = get_manager().get_profile().default_user_email + user = self.backend.users.create(email=email) session.add(user.dbmodel) session.commit() diff --git a/aiida/backends/sqlalchemy/tests/testbase.py b/aiida/backends/sqlalchemy/tests/testbase.py index 71c4803f75..fe6e478187 100644 --- a/aiida/backends/sqlalchemy/tests/testbase.py +++ b/aiida/backends/sqlalchemy/tests/testbase.py @@ -22,7 +22,7 @@ from aiida.backends.sqlalchemy.models.user import DbUser from aiida.backends.sqlalchemy.utils import install_tc from aiida.backends.testimplbase import AiidaTestImplementation -from aiida.common.utils import get_configured_user_email +from aiida.manage import get_manager from aiida.orm.implementation.sqlalchemy.backend import SqlaBackend # Querying for expired objects automatically doesn't seem to work. @@ -65,7 +65,6 @@ def setUpClass_method(self): else: self.clean_db() self.backend = SqlaBackend() - self.insert_data() def setUp_method(self): pass @@ -73,26 +72,6 @@ def setUp_method(self): def tearDown_method(self): pass - def insert_data(self): - """ - Insert default data into the DB. - """ - email = get_configured_user_email() - - has_user = DbUser.query.filter(DbUser.email == email).first() - if not has_user: - self.user = DbUser(get_configured_user_email(), "foo", "bar", - "tests") - self.test_session.add(self.user) - self.test_session.commit() - else: - self.user = has_user - - # Required by the calling class - self.user_email = self.user.email - - super(SqlAlchemyTests, self).insert_data() - @staticmethod def inject_computer(f): @functools.wraps(f) @@ -140,7 +119,7 @@ def clean_db(self): # delete computers and users self.test_session.query(DbNode).delete() - # # Delete the users + # Delete the users self.test_session.query(DbUser).delete() # Delete the computers @@ -162,8 +141,6 @@ def tearDownClass_method(self): "the repository. Repository path: " "{}".format(REPOSITORY_PATH)) - self.clean_db() - self.test_session.close() self.test_session = None diff --git a/aiida/backends/testbase.py b/aiida/backends/testbase.py index eb46a2a3bc..8770ba0d23 100644 --- a/aiida/backends/testbase.py +++ b/aiida/backends/testbase.py @@ -23,8 +23,7 @@ from aiida.backends.tests import get_db_test_list from aiida.common.exceptions import ConfigurationError, TestsNotAllowedError, InternalError from aiida.common.utils import classproperty -from aiida import work -from aiida.manage.manager import AiiDAManager +from aiida.manage import get_manager, reset_manager def check_if_tests_can_run(): @@ -90,8 +89,6 @@ def get_backend_class(cls): @classmethod def setUpClass(cls, *args, **kwargs): - from aiida.orm.backends import construct_backend - # Note: this will raise an exception, that will be seen as a test # failure. To be safe, you should do the same check also in the tearDownClass # to avoid that it is run @@ -99,7 +96,8 @@ def setUpClass(cls, *args, **kwargs): cls.__backend_instance = cls.get_backend_class()() cls.__backend_instance.setUpClass_method(*args, **kwargs) - cls.backend = construct_backend() + cls.backend = cls.__backend_instance.backend + cls.insert_data() cls._class_was_setup = True @@ -114,7 +112,7 @@ def tearDown(self): self.__backend_instance.tearDown_method() # Clean up the loop we created in set up. # Call this after the instance tear down just in case it uses the loop - AiiDAManager.reset() + reset_manager() loop = ioloop.IOLoop.current() if not loop._closing: loop.close() @@ -136,6 +134,8 @@ def clean_db(cls): raise InvalidOperation("You cannot call clean_db before running the setUpClass") cls.__backend_instance.clean_db() + # Make sure to reset the backend when cleaning the database + reset_manager() @classproperty def computer(cls): @@ -156,6 +156,7 @@ def tearDownClass(cls, *args, **kwargs): # Double check for double security to avoid to run the tearDown # if this is not a test profile check_if_tests_can_run() + cls.clean_db() cls.__backend_instance.tearDownClass_method(*args, **kwargs) def assertClickResultNoException(self, cli_result): diff --git a/aiida/backends/testimplbase.py b/aiida/backends/testimplbase.py index 789009a27d..550f740d19 100644 --- a/aiida/backends/testimplbase.py +++ b/aiida/backends/testimplbase.py @@ -15,7 +15,8 @@ import six from aiida.common.exceptions import InternalError -from aiida.orm import Computer +from aiida import orm +from aiida.manage import get_manager @six.add_metaclass(ABCMeta) @@ -41,6 +42,8 @@ class AiidaTestImplementation(object): # This should be set by the implementing class in setUpClass_method() backend = None # type: aiida.orm.Backend computer = None # type: aiida.orm.Computer + user = None # type: aiida.orm.User + user_email = None # type: str @abstractmethod def setUpClass_method(self): @@ -51,15 +54,12 @@ def setUpClass_method(self): """ pass - @abstractmethod def setUp_method(self): pass - @abstractmethod def tearDown_method(self): pass - @abstractmethod def tearDownClass_method(self): """ This class implements the tear down methods (e.g. cleans up the DB). @@ -77,7 +77,7 @@ def insert_data(self): """ This method inserts default data into the database. """ - self.computer = Computer( + self.computer = orm.Computer( name='localhost', hostname='localhost', transport_type='local', @@ -86,6 +86,10 @@ def insert_data(self): backend=self.backend ).store() + email = get_manager().get_profile().default_user_email + self.user = orm.User(email=email).store() + self.user_email = email + def get_computer(self): """ A ORM Computer object present in the DB diff --git a/aiida/backends/tests/cmdline/commands/test_calcjob.py b/aiida/backends/tests/cmdline/commands/test_calcjob.py index 146ae67fef..f6facb373b 100644 --- a/aiida/backends/tests/cmdline/commands/test_calcjob.py +++ b/aiida/backends/tests/cmdline/commands/test_calcjob.py @@ -40,20 +40,16 @@ def setUpClass(cls, *args, **kwargs): from aiida import orm cls.computer = orm.Computer( - name='comp', - hostname='localhost', - transport_type='local', - scheduler_type='direct', - workdir='/tmp/aiida', - backend=cls.backend).store() + name='comp', hostname='localhost', transport_type='local', scheduler_type='direct', + workdir='/tmp/aiida').store() cls.code = orm.Code(remote_computer_exec=(cls.computer, '/bin/true')).store() cls.group = orm.Group(name='test_group').store() cls.node = Node().store() cls.calcs = [] - user = orm.User.objects(cls.backend).get_default() - authinfo = orm.AuthInfo(computer=cls.computer, user=user, backend=cls.backend) + user = orm.User.objects.get_default() + authinfo = orm.AuthInfo(computer=cls.computer, user=user) authinfo.store() # Create 13 CalcJobNodes (one for each CalculationState) @@ -108,7 +104,7 @@ def setUpClass(cls, *args, **kwargs): # Get the imported ArithmeticAddCalculation node ArithmeticAddCalculation = CalculationFactory('arithmetic.add') - calcjobs = orm.QueryBuilder(backend=cls.backend).append(ArithmeticAddCalculation).all()[0] + calcjobs = orm.QueryBuilder().append(ArithmeticAddCalculation).all()[0] cls.arithmetic_job = calcjobs[0] def setUp(self): diff --git a/aiida/backends/tests/cmdline/commands/test_calculation.py b/aiida/backends/tests/cmdline/commands/test_calculation.py index a2038b34be..1ce2078a3d 100644 --- a/aiida/backends/tests/cmdline/commands/test_calculation.py +++ b/aiida/backends/tests/cmdline/commands/test_calculation.py @@ -45,16 +45,15 @@ def setUpClass(cls, *args, **kwargs): hostname='localhost', transport_type='local', scheduler_type='direct', - workdir='/tmp/aiida', - backend=cls.backend).store() + workdir='/tmp/aiida').store() cls.code = orm.Code(remote_computer_exec=(cls.computer, '/bin/true')).store() cls.group = orm.Group(name='test_group').store() cls.node = Node().store() cls.calcs = [] - user = orm.User.objects(cls.backend).get_default() - authinfo = orm.AuthInfo(computer=cls.computer, user=user, backend=cls.backend) + user = orm.User.objects.get_default() + authinfo = orm.AuthInfo(computer=cls.computer, user=user) authinfo.store() # Create 13 CalcJobNodes (one for each CalculationState) @@ -109,7 +108,7 @@ def setUpClass(cls, *args, **kwargs): # Get the imported ArithmeticAddCalculation node ArithmeticAddCalculation = CalculationFactory('arithmetic.add') - calculations = orm.QueryBuilder(backend=cls.backend).append(ArithmeticAddCalculation).all()[0] + calculations = orm.QueryBuilder().append(ArithmeticAddCalculation).all()[0] cls.arithmetic_job = calculations[0] def setUp(self): diff --git a/aiida/backends/tests/cmdline/commands/test_code.py b/aiida/backends/tests/cmdline/commands/test_code.py index b392009f7d..9595743371 100644 --- a/aiida/backends/tests/cmdline/commands/test_code.py +++ b/aiida/backends/tests/cmdline/commands/test_code.py @@ -34,11 +34,10 @@ def setUpClass(cls, *args, **kwargs): hostname='localhost', transport_type='local', scheduler_type='direct', - workdir='/tmp/aiida', - backend=cls.backend).store() + workdir='/tmp/aiida').store() def setUp(self): - self.comp = orm.Computer.objects(self.backend).get(name='comp') + self.comp = orm.Computer.objects.get(name='comp') self.cli_runner = CliRunner() self.this_folder = os.path.dirname(__file__) @@ -123,12 +122,11 @@ def setUpClass(cls, *args, **kwargs): hostname='localhost', transport_type='local', scheduler_type='direct', - workdir='/tmp/aiida', - backend=cls.backend).store() + workdir='/tmp/aiida').store() def setUp(self): from aiida import orm - self.comp = orm.Computer.objects(self.backend).get(name='comp') + self.comp = orm.Computer.objects.get(name='comp') try: code = orm.Code.get_from_string('code') diff --git a/aiida/backends/tests/cmdline/commands/test_computer.py b/aiida/backends/tests/cmdline/commands/test_computer.py index 559b7f42a9..4b69974f22 100644 --- a/aiida/backends/tests/cmdline/commands/test_computer.py +++ b/aiida/backends/tests/cmdline/commands/test_computer.py @@ -130,7 +130,7 @@ def test_interactive(self): result = self.cli_runner.invoke(computer_setup, input=user_input) self.assertIsNone(result.exception, msg="There was an unexpected exception. Output: {}".format(result.output)) - new_computer = orm.Computer.objects(self.backend).get(name=label) + new_computer = orm.Computer.objects.get(name=label) self.assertIsInstance(new_computer, orm.Computer) self.assertEqual(new_computer.description, options_dict['description']) @@ -171,7 +171,7 @@ def test_mixed(self): result = self.cli_runner.invoke(computer_setup, options, input=user_input) self.assertIsNone(result.exception, msg="There was an unexpected exception. Output: {}".format(result.output)) - new_computer = orm.Computer.objects(self.backend).get(name=label) + new_computer = orm.Computer.objects.get(name=label) self.assertIsInstance(new_computer, orm.Computer) self.assertEqual(new_computer.description, options_dict_full['description']) @@ -198,7 +198,7 @@ def test_noninteractive(self): result = self.cli_runner.invoke(computer_setup, options) self.assertIsNone(result.exception, result.output[-1000:]) - new_computer = orm.Computer.objects(self.backend).get(name=options_dict['label']) + new_computer = orm.Computer.objects.get(name=options_dict['label']) self.assertIsInstance(new_computer, orm.Computer) self.assertEqual(new_computer.description, options_dict['description']) @@ -234,7 +234,7 @@ def test_noninteractive_disabled(self): result = self.cli_runner.invoke(computer_setup, options) self.assertIsNone(result.exception, result.output[-1000:]) - new_computer = orm.Computer.objects(self.backend).get(name=options_dict['label']) + new_computer = orm.Computer.objects.get(name=options_dict['label']) self.assertIsInstance(new_computer, orm.Computer) self.assertFalse(new_computer.is_enabled()) @@ -253,7 +253,7 @@ def test_noninteractive_enabled(self): result = self.cli_runner.invoke(computer_setup, options) self.assertIsNone(result.exception, result.output[-1000:]) - new_computer = orm.Computer.objects(self.backend).get(name=options_dict['label']) + new_computer = orm.Computer.objects.get(name=options_dict['label']) self.assertIsInstance(new_computer, orm.Computer) self.assertTrue(new_computer.is_enabled()) @@ -268,7 +268,7 @@ def test_noninteractive_optional_default_mpiprocs(self): self.assertIsNone(result.exception, result.output[-1000:]) - new_computer = orm.Computer.objects(self.backend).get(name=options_dict['label']) + new_computer = orm.Computer.objects.get(name=options_dict['label']) self.assertIsInstance(new_computer, orm.Computer) self.assertIsNone(new_computer.get_default_mpiprocs_per_machine()) @@ -283,7 +283,7 @@ def test_noninteractive_optional_default_mpiprocs_2(self): self.assertIsNone(result.exception, result.output[-1000:]) - new_computer = orm.Computer.objects(self.backend).get(name=options_dict['label']) + new_computer = orm.Computer.objects.get(name=options_dict['label']) self.assertIsInstance(new_computer, orm.Computer) self.assertIsNone(new_computer.get_default_mpiprocs_per_machine()) @@ -351,11 +351,9 @@ class TestVerdiComputerConfigure(AiidaTestCase): def setUp(self): """Prepare computer builder with common properties.""" - from aiida.orm.backends import construct_backend from aiida.control.computer import ComputerBuilder - self.backend = construct_backend() self.cli_runner = CliRunner() - self.user = orm.User.objects(self.backend).get_default() + self.user = orm.User.objects.get_default() self.comp_builder = ComputerBuilder(label='test_comp_setup') self.comp_builder.hostname = 'localhost' self.comp_builder.description = 'Test Computer' @@ -459,7 +457,7 @@ def test_ssh_ni_username(self): ['ssh', comp.label, '--non-interactive', '--username={}'.format(username)], catch_exceptions=False) self.assertTrue(comp.is_user_configured(self.user), msg=result.output) - self.assertEqual(orm.AuthInfo.objects(self.backend).get( + self.assertEqual(orm.AuthInfo.objects.get( dbcomputer_id=comp.id, aiidauser_id=self.user.id).get_auth_params()['username'], username) @@ -507,8 +505,7 @@ def setUpClass(cls, *args, **kwargs): hostname='localhost', transport_type='local', scheduler_type='direct', - workdir='/tmp/aiida', - backend=cls.backend) + workdir='/tmp/aiida') cls.comp.set_default_mpiprocs_per_machine(1) cls.comp.store() @@ -516,7 +513,7 @@ def setUp(self): """ Prepare the computer and user """ - self.user = orm.User.objects(self.backend).get_default() + self.user = orm.User.objects.get_default() # I need to configure the computer here; being 'local', # there should not be any options asked here @@ -685,9 +682,9 @@ def test_computer_rename(self): # Check that the name really was changed # The old name should not be available with self.assertRaises(NotExistent): - orm.Computer.objects(self.backend).get(name='comp_cli_test_computer') + orm.Computer.objects.get(name='comp_cli_test_computer') # The new name should be avilable - orm.Computer.objects(self.backend).get(name='renamed_test_computer') + orm.Computer.objects.get(name='renamed_test_computer') # Now change the name back options = ['renamed_test_computer', 'comp_cli_test_computer'] @@ -698,9 +695,9 @@ def test_computer_rename(self): # Check that the name really was changed # The old name should not be available with self.assertRaises(NotExistent): - orm.Computer.objects(self.backend).get(name='renamed_test_computer') + orm.Computer.objects.get(name='renamed_test_computer') # The new name should be avilable - orm.Computer.objects(self.backend).get(name='comp_cli_test_computer') + orm.Computer.objects.get(name='comp_cli_test_computer') def test_computer_delete(self): """ @@ -710,8 +707,7 @@ def test_computer_delete(self): # Setup a computer to delete during the test comp = orm.Computer(name='computer_for_test_delete', hostname='localhost', - transport_type='local', scheduler_type='direct', workdir='/tmp/aiida', - backend=self.backend).store() + transport_type='local', scheduler_type='direct', workdir='/tmp/aiida').store() # See if the command complains about not getting an invalid computer options = ['non_existent_computer_name'] @@ -723,10 +719,10 @@ def test_computer_delete(self): options = ['computer_for_test_delete'] result = self.cli_runner.invoke(computer_delete, options) # Exception should be not be raised - self.assertIsNone(result.exception, result.output) + self.assertClickResultNoException(result) # Check that the computer really was deleted with self.assertRaises(NotExistent): - orm.Computer.objects(self.backend).get(name='computer_for_test_delete') + orm.Computer.objects.get(name='computer_for_test_delete') def test_computer_duplicate_interactive(self): os.environ['VISUAL'] = 'sleep 1; vim -cwq' @@ -737,7 +733,7 @@ def test_computer_duplicate_interactive(self): catch_exceptions=False) self.assertIsNone(result.exception, result.output) - new_computer = orm.Computer.objects(self.backend).get(name=label) + new_computer = orm.Computer.objects.get(name=label) self.assertEquals(self.comp.description, new_computer.description) self.assertEquals(self.comp.get_hostname(), new_computer.get_hostname()) self.assertEquals(self.comp.get_transport_type(), new_computer.get_transport_type()) @@ -755,7 +751,7 @@ def test_computer_duplicate_non_interactive(self): ['--non-interactive', '--label=' + label, str(self.comp.pk)]) self.assertIsNone(result.exception, result.output) - new_computer = orm.Computer.objects(self.backend).get(name=label) + new_computer = orm.Computer.objects.get(name=label) self.assertEquals(self.comp.description, new_computer.description) self.assertEquals(self.comp.get_hostname(), new_computer.get_hostname()) self.assertEquals(self.comp.get_transport_type(), new_computer.get_transport_type()) diff --git a/aiida/backends/tests/cmdline/commands/test_data.py b/aiida/backends/tests/cmdline/commands/test_data.py index 259d6bcd68..d12db8cbae 100644 --- a/aiida/backends/tests/cmdline/commands/test_data.py +++ b/aiida/backends/tests/cmdline/commands/test_data.py @@ -426,8 +426,8 @@ class TestVerdiDataRemote(AiidaTestCase): @classmethod def setUpClass(cls): super(TestVerdiDataRemote, cls).setUpClass() - user = orm.User.objects(cls.backend).get_default() - orm.AuthInfo(cls.computer, user, backend=cls.backend).store() + user = orm.User.objects.get_default() + orm.AuthInfo(cls.computer, user).store() def setUp(self): comp = self.computer @@ -552,8 +552,7 @@ def setUpClass(cls): hostname='localhost', transport_type='local', scheduler_type='direct', - workdir='/tmp/aiida', - backend=cls.backend).store() + workdir='/tmp/aiida').store() cls.ids = cls.create_trajectory_data() def setUp(self): @@ -649,8 +648,7 @@ def setUpClass(cls): hostname='localhost', transport_type='local', scheduler_type='direct', - workdir='/tmp/aiida', - backend=cls.backend).store() + workdir='/tmp/aiida').store() cls.ids = cls.create_structure_data() def setUp(self): @@ -768,8 +766,7 @@ def setUpClass(cls): hostname='localhost', transport_type='local', scheduler_type='direct', - workdir='/tmp/aiida', - backend=cls.backend).store() + workdir='/tmp/aiida').store() cls.ids = cls.create_cif_data() diff --git a/aiida/backends/tests/cmdline/commands/test_export.py b/aiida/backends/tests/cmdline/commands/test_export.py index c0ea2a25f8..926abcbd03 100644 --- a/aiida/backends/tests/cmdline/commands/test_export.py +++ b/aiida/backends/tests/cmdline/commands/test_export.py @@ -68,8 +68,7 @@ def setUpClass(cls, *args, **kwargs): hostname='localhost', transport_type='local', scheduler_type='direct', - workdir='/tmp/aiida', - backend=cls.backend).store() + workdir='/tmp/aiida').store() cls.code = orm.Code(remote_computer_exec=(cls.computer, '/bin/true')).store() cls.group = orm.Group(name='test_group').store() diff --git a/aiida/backends/tests/cmdline/commands/test_node.py b/aiida/backends/tests/cmdline/commands/test_node.py index 4fd1480086..e868656609 100644 --- a/aiida/backends/tests/cmdline/commands/test_node.py +++ b/aiida/backends/tests/cmdline/commands/test_node.py @@ -426,7 +426,7 @@ class TestVerdiDataRemote(AiidaTestCase): def setUpClass(cls): from aiida import orm super(TestVerdiDataRemote, cls).setUpClass() - user = orm.User.objects(cls.backend).get_default() + user = orm.User.objects.get_default() authinfo = orm.AuthInfo(cls.computer, user) authinfo.store() diff --git a/aiida/backends/tests/cmdline/commands/test_process.py b/aiida/backends/tests/cmdline/commands/test_process.py index e38f3cbc94..9ced9afc59 100644 --- a/aiida/backends/tests/cmdline/commands/test_process.py +++ b/aiida/backends/tests/cmdline/commands/test_process.py @@ -29,7 +29,7 @@ from aiida.common.log import LOG_LEVEL_REPORT from aiida.orm.node.process import ProcessNode, WorkFunctionNode, WorkChainNode from aiida.work import test_utils -from aiida.manage.manager import AiiDAManager +from aiida.manage import get_manager def get_result_lines(result): @@ -50,7 +50,7 @@ def setUp(self): self.daemon_client = DaemonClient(profile) self.daemon_pid = subprocess.Popen( self.daemon_client.cmd_string.split(), stderr=sys.stderr, stdout=sys.stdout).pid - self.runner = AiiDAManager.create_runner(rmq_submit=True) + self.runner = get_manager().create_runner(rmq_submit=True) self.cli_runner = CliRunner() def tearDown(self): diff --git a/aiida/backends/tests/cmdline/commands/test_user.py b/aiida/backends/tests/cmdline/commands/test_user.py index 0aade3874c..48e04470fd 100644 --- a/aiida/backends/tests/cmdline/commands/test_user.py +++ b/aiida/backends/tests/cmdline/commands/test_user.py @@ -37,12 +37,12 @@ class TestVerdiUserCommand(AiidaTestCase): def setUp(self): super(TestVerdiUserCommand, self).setUp() - created, user = orm.User.objects(self.backend).get_or_create(email=user_1['email']) + created, user = orm.User.objects.get_or_create(email=user_1['email']) for key, value in user_1.items(): if key != 'email': setattr(user, key, user_1[key]) if created: - orm.User(backend=self.backend, **user_1).store() + orm.User(**user_1).store() self.cli_runner = CliRunner() def test_user_list(self): @@ -72,7 +72,7 @@ def test_user_create(self): self.assertTrue(user_2['email'] in result.output) self.assertTrue("is already present" not in result.output) - user_obj = orm.User.objects(self.backend).get(email=user_2['email']) + user_obj = orm.User.objects.get(email=user_2['email']) for key, val in user_2.items(): self.assertEqual(val, getattr(user_obj, key)) @@ -93,7 +93,7 @@ def test_user_update(self): result = CliRunner().invoke(cmd_user.configure, cli_options, catch_exceptions=False) self.assertTrue(email in result.output) - user_model = orm.User.objects(self.backend).get(email=email) + user_model = orm.User.objects.get(email=email) # Check it's all been changed to user2's attributes except the email for key, value in user_2.items(): diff --git a/aiida/backends/tests/cmdline/commands/test_workflow.py b/aiida/backends/tests/cmdline/commands/test_workflow.py index 2f3aa3e89a..30d5272562 100644 --- a/aiida/backends/tests/cmdline/commands/test_workflow.py +++ b/aiida/backends/tests/cmdline/commands/test_workflow.py @@ -18,10 +18,6 @@ format_pk -def debug_msg(result): - return ''.join(format_exception(*result.exc_info)) - - def format_wf_for_list(workflow): return '{} {}'.format(workflow.__class__.__name__, format_pk(workflow)) @@ -48,21 +44,21 @@ def setUpClass(cls): def test_workflow_list_default(self): result = self.cli_runner.invoke(workflow_list, []) - self.assertIsNone(result.exception, msg=debug_msg(result)) + self.assertClickResultNoException(result) self.assertIn(format_wf_for_list(self.workflow), result.output) self.assertIn(format_wf_for_list(self.other_workflow), result.output) self.assertNotIn(format_wf_for_list(self.done_workflow), result.output) def test_workflow_list_workflows(self): result = self.cli_runner.invoke(workflow_list, ['--workflows={}'.format(self.workflow.pk)]) - self.assertIsNone(result.exception, msg=debug_msg(result)) + self.assertClickResultNoException(result) self.assertIn(format_wf_for_list(self.workflow), result.output) self.assertNotIn(format_wf_for_list(self.other_workflow), result.output) self.assertNotIn(format_wf_for_list(self.done_workflow), result.output) def test_workflow_list_states(self): result = self.cli_runner.invoke(workflow_list, ['--all-states']) - self.assertIsNone(result.exception, msg=debug_msg(result)) + self.assertClickResultNoException(result) self.assertIn(format_wf_for_list(self.workflow), result.output) self.assertIn(format_wf_for_list(self.other_workflow), result.output) self.assertIn(format_wf_for_list(self.done_workflow), result.output) @@ -74,14 +70,14 @@ def test_workflow_list_depth(self): results.append(self.cli_runner.invoke(workflow_list, ['--depth=2'])) for result in results: - self.assertIsNone(result.exception, msg=debug_msg(result)) + self.assertClickResultNoException(result) self.assertLess(len(results[0].output), len(results[1].output)) self.assertLess(len(results[1].output), len(results[2].output)) def test_workflow_report(self): result = self.cli_runner.invoke(workflow_report, [str(self.super_workflow.uuid)]) - self.assertIsNone(result.exception, msg=debug_msg(result)) + self.assertClickResultNoException(result) self.assertIn(format_pk(self.super_workflow), result.output) def test_workflow_kill(self): @@ -102,5 +98,5 @@ def test_workflow_kill(self): def test_workflow_logshow(self): result = self.cli_runner.invoke(workflow_logshow, [str(self.super_workflow.pk)]) - self.assertIsNone(result.exception, msg=debug_msg(result)) + self.assertClickResultNoException(result) self.assertIn(format_pk(self.super_workflow), result.output) diff --git a/aiida/backends/tests/cmdline/params/types/test_group.py b/aiida/backends/tests/cmdline/params/types/test_group.py index dea968e484..462ba18858 100644 --- a/aiida/backends/tests/cmdline/params/types/test_group.py +++ b/aiida/backends/tests/cmdline/params/types/test_group.py @@ -29,9 +29,9 @@ def setUpClass(cls): super(TestGroupParamType, cls).setUpClass() cls.param = GroupParamType() - cls.entity_01 = Group(name='group_01', backend=cls.backend).store() - cls.entity_02 = Group(name=str(cls.entity_01.pk), backend=cls.backend).store() - cls.entity_03 = Group(name=str(cls.entity_01.uuid), backend=cls.backend).store() + cls.entity_01 = Group(name='group_01').store() + cls.entity_02 = Group(name=str(cls.entity_01.pk)).store() + cls.entity_03 = Group(name=str(cls.entity_01.uuid)).store() def test_get_by_id(self): """ diff --git a/aiida/backends/tests/computer.py b/aiida/backends/tests/computer.py index d257450415..c420335e0e 100644 --- a/aiida/backends/tests/computer.py +++ b/aiida/backends/tests/computer.py @@ -54,18 +54,17 @@ def test_delete(self): hostname='aaa', transport_type='local', scheduler_type='pbspro', - workdir='/tmp/aiida', - backend=self.backend).store() + workdir='/tmp/aiida').store() comp_pk = new_comp.pk - check_computer = orm.Computer.objects(self.backend).get(id=comp_pk) + check_computer = orm.Computer.objects.get(id=comp_pk) self.assertEquals(comp_pk, check_computer.pk) Computer.objects.delete(comp_pk) with self.assertRaises(NotExistent): - orm.Computer.objects(self.backend).get(id=comp_pk) + orm.Computer.objects.get(id=comp_pk) class TestComputerConfigure(AiidaTestCase): @@ -74,7 +73,6 @@ def setUp(self): """Prepare current user and computer builder with common properties.""" from aiida.control.computer import ComputerBuilder - backend = self.backend self.comp_builder = ComputerBuilder(label='test', description='Test Computer', enabled=True, hostname='localhost') self.comp_builder.scheduler = 'direct' diff --git a/aiida/backends/tests/daemon.py b/aiida/backends/tests/daemon.py index b6d35d6ca7..b1edb1a0df 100644 --- a/aiida/backends/tests/daemon.py +++ b/aiida/backends/tests/daemon.py @@ -26,9 +26,8 @@ class TestDaemonBasic(AiidaTestCase): def test_workflow_fast_kill(self): from aiida.cmdline.commands.workflow import Workflow as WfCmd - from aiida.orm.backends import construct_backend - backend = construct_backend() + backend = self.backend params = dict() params['nmachine'] = 2 diff --git a/aiida/backends/tests/export_and_import.py b/aiida/backends/tests/export_and_import.py index 40e1f6d3cc..4c9b97b9f7 100644 --- a/aiida/backends/tests/export_and_import.py +++ b/aiida/backends/tests/export_and_import.py @@ -466,14 +466,16 @@ def test_5(self): from aiida.orm.importexport import export from aiida.common.datastructures import calc_states from aiida.common.links import LinkType - from aiida.common.utils import get_configured_user_email - + from aiida.manage import get_manager + + manager = get_manager() + # Creating a folder for the import/export files temp_folder = tempfile.mkdtemp() try: # Create another user new_email = "newuser@new.n" - user = orm.User(email=new_email, backend=self.backend).store() + user = orm.User(email=new_email).store() # Create a structure data node that has a calculation as output sd1 = StructureData() @@ -525,8 +527,7 @@ def test_5(self): node = load_node(uuid=uuid) self.assertEquals(node.get_user().email, new_email) for uuid in uuids_u2: - self.assertEquals(load_node(uuid).get_user().email, - get_configured_user_email()) + self.assertEquals(load_node(uuid).get_user().email, manager.get_profile().default_user_email) finally: # Deleting the created temporary folder shutil.rmtree(temp_folder, ignore_errors=True) @@ -549,14 +550,16 @@ def test_6(self): from aiida.orm.importexport import export from aiida.common.datastructures import calc_states from aiida.common.links import LinkType - from aiida.common.utils import get_configured_user_email + from aiida.manage import get_manager + + manager = get_manager() # Creating a folder for the import/export files temp_folder = tempfile.mkdtemp() try: # Create another user new_email = "newuser@new.n" - user = orm.User(email=new_email, backend=self.backend).store() + user = orm.User(email=new_email).store() # Create a structure data node that has a calculation as output sd1 = StructureData() @@ -631,8 +634,7 @@ def test_6(self): for uuid in uuids1: self.assertEquals(load_node(uuid).get_user().email, new_email) for uuid in uuids2: - self.assertEquals(load_node(uuid).get_user().email, - get_configured_user_email()) + self.assertEquals(load_node(uuid).get_user().email, manager.get_profile().default_user_email) finally: # Deleting the created temporary folder @@ -660,7 +662,7 @@ def test_7(self): try: # Create another user new_email = "newuser@new.n" - user = orm.User(email=new_email, backend=self.backend) + user = orm.User(email=new_email) user.store() # Create a structure data node that has a calculation as output @@ -725,7 +727,7 @@ def test_group_export(self): try: # Create another user new_email = "newuser@new.n" - user = orm.User(email=new_email, backend=self.backend) + user = orm.User(email=new_email) user.store() # Create a structure data node diff --git a/aiida/backends/tests/generic.py b/aiida/backends/tests/generic.py index 910f74b551..250c07d3f7 100644 --- a/aiida/backends/tests/generic.py +++ b/aiida/backends/tests/generic.py @@ -163,8 +163,8 @@ def test_creation(self): n = orm.Node() stored_n = orm.Node().store() - g = orm.Group(name='testgroup', backend=self.backend) - self.addCleanup(lambda: orm.Group.objects(self.backend).delete(g.id)) + g = orm.Group(name='testgroup') + self.addCleanup(lambda: orm.Group.objects.delete(g.id)) with self.assertRaises(ModificationNotAllowed): # g unstored @@ -195,12 +195,12 @@ def test_description(self): n = orm.Node().store() - g1 = Group(name='testgroupdescription1', description="g1", backend=self.backend).store() - self.addCleanup(lambda: orm.Group.objects(self.backend).delete(g1.id)) + g1 = Group(name='testgroupdescription1', description="g1").store() + self.addCleanup(lambda: orm.Group.objects.delete(g1.id)) g1.add_nodes(n) - g2 = Group(name='testgroupdescription2', description="g2", backend=self.backend) - self.addCleanup(lambda: orm.Group.objects(self.backend).delete(g2.id)) + g2 = Group(name='testgroupdescription2', description="g2") + self.addCleanup(lambda: orm.Group.objects.delete(g2.id)) # Preliminary checks self.assertTrue(g1.is_stored) @@ -237,8 +237,8 @@ def test_add_nodes(self): n7 = orm.Node().store() n8 = orm.Node().store() - g = orm.Group(name='test_adding_nodes', backend=self.backend).store() - self.addCleanup(lambda: orm.Group.objects(self.backend).delete(g.pk)) + g = orm.Group(name='test_adding_nodes').store() + self.addCleanup(lambda: orm.Group.objects.delete(g.pk)) g.store() # Single node g.add_nodes(n1) @@ -275,7 +275,7 @@ def test_remove_nodes(self): n_out = orm.Node().store() g = Group(name='test_remove_nodes').store() - self.addCleanup(lambda: orm.Group.objects(self.backend).delete(g.id)) + self.addCleanup(lambda: orm.Group.objects.delete(g.id)) # Add initial nodes g.add_nodes([n1, n2, n3, n4, n5, n6, n7, n8]) @@ -320,14 +320,14 @@ def test_name_desc(self): self.assertEquals(g.description, 'some desc') # To avoid to find it in further tests - orm.Group.objects(self.backend).delete(g.pk) + orm.Group.objects.delete(g.pk) def test_delete(self): from aiida.common.exceptions import NotExistent n = orm.Node().store() - g = orm.Group(name='testgroup3', description='some other desc', backend=self.backend).store() + g = orm.Group(name='testgroup3', description='some other desc').store() gcopy = orm.Group.get(name='testgroup3') self.assertEquals(g.uuid, gcopy.uuid) @@ -335,7 +335,7 @@ def test_delete(self): g.add_nodes(n) self.assertEquals(len(g.nodes), 1) - orm.Group.objects(self.backend).delete(g.pk) + orm.Group.objects.delete(g.pk) with self.assertRaises(NotExistent): # The group does not exist anymore diff --git a/aiida/backends/tests/nodes.py b/aiida/backends/tests/nodes.py index 9d2df9fd6f..f7695ebce5 100644 --- a/aiida/backends/tests/nodes.py +++ b/aiida/backends/tests/nodes.py @@ -114,7 +114,8 @@ def test_folder_file_different(self): f2 = self.create_folderdata_with_empty_folder() assert ( - f1.folder.get_subfolder('path').get_content_list() == f2.folder.get_subfolder('path').get_content_list()) + f1.folder.get_subfolder('path').get_content_list() == f2.folder.get_subfolder( + 'path').get_content_list()) assert f1.get_hash() != f2.get_hash() def test_folder_same(self): @@ -1184,7 +1185,7 @@ def test_comments(self): from aiida.utils import timezone from time import sleep - user = User.objects(self.backend).get_default() + user = User.objects.get_default() a = Node() with self.assertRaises(ModificationNotAllowed): @@ -1272,22 +1273,22 @@ def test_get_subclass_from_pk(self): # Check that you can load it with a simple integer id. a2 = Node.get_subclass_from_pk(a1.id) self.assertEquals(a1.id, a2.id, "The ids of the stored and loaded node" - "should be equal (since it should be " - "the same node") + "should be equal (since it should be " + "the same node") if six.PY2: # In Python 3, int is always long (enough) # Check that you can load it with an id of type long a3 = Node.get_subclass_from_pk(long(a1.id)) self.assertEquals(a1.id, a3.id, "The ids of the stored and loaded node" - "should be equal (since it should be " - "the same node") + "should be equal (since it should be " + "the same node") # Check that it manages to load the node even if the id is # passed as a string. a4 = Node.get_subclass_from_pk(str(a1.id)) self.assertEquals(a1.id, a4.id, "The ids of the stored and loaded node" - "should be equal (since it should be " - "the same node") + "should be equal (since it should be " + "the same node") # Check that a ValueError exception is raised when a string that can # not be casted to integer is passed. @@ -1834,6 +1835,7 @@ def test_multiple_create_links(self): def test_valid_links(self): import tempfile + from aiida import orm from aiida.orm import DataFactory from aiida.orm.node.process import CalcJobNode from aiida.orm.code import Code @@ -1852,7 +1854,7 @@ def test_valid_links(self): code.set_remote_computer_exec((self.computer, '/bin/true')) code.store() - unsavedcomputer = self.backend.computers.create(name='localhost2', hostname='localhost') + unsavedcomputer = orm.Computer(name='localhost2', hostname='localhost') with self.assertRaises(ValueError): # I need to save the localhost entry first diff --git a/aiida/backends/tests/query.py b/aiida/backends/tests/query.py index 6769ba8265..65a0a78a4e 100644 --- a/aiida/backends/tests/query.py +++ b/aiida/backends/tests/query.py @@ -792,7 +792,7 @@ def test_joins3_user_group(self): # Create another user new_email = "newuser@new.n" - user = orm.User(email=new_email, backend=self.backend).store() + user = orm.User(email=new_email).store() # Create a group that belongs to that user from aiida.orm.groups import Group diff --git a/aiida/backends/tests/restapi.py b/aiida/backends/tests/restapi.py index e5e5065456..5093430d82 100644 --- a/aiida/backends/tests/restapi.py +++ b/aiida/backends/tests/restapi.py @@ -113,7 +113,7 @@ def setUpClass(cls, *args, **kwargs): }] for dummy_computer in dummy_computers: - computer = orm.Computer(backend=cls.backend, **dummy_computer) + computer = orm.Computer(**dummy_computer) computer.store() # Prepare typical REST responses diff --git a/aiida/backends/tests/work/test_futures.py b/aiida/backends/tests/work/test_futures.py index 51da201ecf..514b45b0f0 100644 --- a/aiida/backends/tests/work/test_futures.py +++ b/aiida/backends/tests/work/test_futures.py @@ -17,21 +17,22 @@ from aiida.backends.testbase import AiidaTestCase from aiida import work -from aiida.manage.manager import AiiDAManager +from aiida.manage import get_manager class TestWf(AiidaTestCase): TIMEOUT = datetime.timedelta(seconds=5.0) def test_calculation_future_broadcasts(self): - runner = AiiDAManager.get_runner() + manager = get_manager() + runner = manager.get_runner() process = work.test_utils.DummyProcess() # No polling future = work.futures.CalculationFuture( pk=process.pid, poll_interval=None, - communicator=AiiDAManager.get_communicator()) + communicator=manager.get_communicator()) work.run(process) calc_node = runner.run_until_complete(gen.with_timeout(self.TIMEOUT, future)) @@ -39,7 +40,7 @@ def test_calculation_future_broadcasts(self): self.assertEqual(process.calc.pk, calc_node.pk) def test_calculation_future_polling(self): - runner = AiiDAManager.get_runner() + runner = get_manager().get_runner() process = work.test_utils.DummyProcess() # No communicator diff --git a/aiida/backends/tests/work/test_rmq.py b/aiida/backends/tests/work/test_rmq.py index 2fcc2abcdc..2ca7ca557f 100644 --- a/aiida/backends/tests/work/test_rmq.py +++ b/aiida/backends/tests/work/test_rmq.py @@ -18,7 +18,7 @@ from aiida.backends.testbase import AiidaTestCase from aiida.orm.data.int import Int from aiida import work -from aiida.manage.manager import AiiDAManager +from aiida.manage import get_manager class TestProcessControl(AiidaTestCase): @@ -33,8 +33,9 @@ def setUp(self): # These two need to share a common event loop otherwise the first will never send # the message while the daemon is running listening to intercept - self.runner = AiiDAManager.get_runner() - self.daemon_runner = AiiDAManager.create_daemon_runner(loop=self.runner.loop) + manager = get_manager() + self.runner = manager.get_runner() + self.daemon_runner = manager.create_daemon_runner(loop=self.runner.loop) def tearDown(self): self.daemon_runner.close() @@ -83,7 +84,7 @@ def do_exception(): def test_pause(self): """Testing sending a pause message to the process.""" - controller = AiiDAManager.get_process_controller() + controller = get_manager().get_process_controller() @gen.coroutine def do_pause(): @@ -103,7 +104,7 @@ def do_pause(): def test_pause_play(self): """Test sending a pause and then a play message.""" - controller = AiiDAManager.get_process_controller() + controller = get_manager().get_process_controller() @gen.coroutine def do_pause_play(): @@ -129,7 +130,7 @@ def do_pause_play(): def test_kill(self): """Test sending a kill message.""" - controller = AiiDAManager.get_process_controller() + controller = get_manager().get_process_controller() @gen.coroutine def do_kill(): diff --git a/aiida/backends/tests/work/test_runners.py b/aiida/backends/tests/work/test_runners.py index 8637356633..64f6cf443a 100644 --- a/aiida/backends/tests/work/test_runners.py +++ b/aiida/backends/tests/work/test_runners.py @@ -17,7 +17,7 @@ from aiida import work from aiida.daemon.workflowmanager import execute_steps from aiida.workflows.wf_demo import WorkflowDemo -from aiida.manage.manager import AiiDAManager +from aiida.manage import get_manager class Proc(work.Process): @@ -33,7 +33,7 @@ class TestWorkchain(AiidaTestCase): def setUp(self): super(TestWorkchain, self).setUp() - self.runner = AiiDAManager.get_runner() + self.runner = get_manager().get_runner() def tearDown(self): super(TestWorkchain, self).tearDown() diff --git a/aiida/backends/tests/work/test_utils.py b/aiida/backends/tests/work/test_utils.py index ae0dcb753f..17c4fd8db1 100644 --- a/aiida/backends/tests/work/test_utils.py +++ b/aiida/backends/tests/work/test_utils.py @@ -32,8 +32,7 @@ def setUpClass(cls, *args, **kwargs): super(TestExponentialBackoffRetry, cls).setUpClass(*args, **kwargs) cls.authinfo = orm.AuthInfo( computer=cls.computer, - user=orm.User.objects(cls.backend).get_default(), - backend=cls.backend) + user=orm.User.objects.get_default()) cls.authinfo.store() def test_exponential_backoff_success(self): diff --git a/aiida/backends/tests/work/work_chain.py b/aiida/backends/tests/work/work_chain.py index f29e13db5b..0d70158842 100644 --- a/aiida/backends/tests/work/work_chain.py +++ b/aiida/backends/tests/work/work_chain.py @@ -32,7 +32,7 @@ from aiida.work import ExitCode, Process from aiida.work.persistence import ObjectLoader from aiida.work.workchain import * -from aiida.manage.manager import AiiDAManager +from aiida.manage import get_manager def run_until_paused(proc): @@ -564,7 +564,7 @@ def test_if_block_persistence(self): """ This test was created to capture issue #902 """ - runner = AiiDAManager.get_runner() + runner = get_manager().get_runner() wc = IfTest() runner.schedule(wc) @@ -650,7 +650,7 @@ def result(self): def test_persisting(self): persister = plumpy.test_utils.TestPersister() - runner = AiiDAManager.get_runner() + runner = get_manager().get_runner() workchain = Wf(runner=runner) work.run(workchain) @@ -818,7 +818,7 @@ def test_simple_run(self): Run the workchain which should hit the exception and therefore end up in the EXCEPTED state """ - runner = AiiDAManager.get_runner() + runner = get_manager().get_runner() process = TestWorkChainAbort.AbortableWorkChain() @gen.coroutine @@ -844,7 +844,7 @@ def test_simple_kill_through_process(self): on the workchain itself. This should have the workchain end up in the KILLED state. """ - runner = AiiDAManager.get_runner() + runner = get_manager().get_runner() process = TestWorkChainAbort.AbortableWorkChain() @gen.coroutine @@ -937,7 +937,7 @@ def test_simple_kill_through_process(self): Run the workchain for one step and then kill it. This should have the workchain and its children end up in the KILLED state. """ - runner = AiiDAManager.get_runner() + runner = get_manager().get_runner() process = TestWorkChainAbortChildren.MainWorkChain(inputs={'kill': Bool(True)}) @gen.coroutine diff --git a/aiida/backends/tests/workflows.py b/aiida/backends/tests/workflows.py index f387bc51ef..def5b3fa8e 100644 --- a/aiida/backends/tests/workflows.py +++ b/aiida/backends/tests/workflows.py @@ -14,7 +14,6 @@ from aiida.backends.testbase import AiidaTestCase from aiida.backends.utils import get_workflow_list from aiida.common.datastructures import wf_states -from aiida.orm.backends import construct_backend from aiida.workflows.test import WFTestEmpty from aiida import orm from aiida.orm.implementation import get_workflow_info diff --git a/aiida/cmdline/commands/cmd_computer.py b/aiida/cmdline/commands/cmd_computer.py index 0ad09457d8..fec4eff6de 100644 --- a/aiida/cmdline/commands/cmd_computer.py +++ b/aiida/cmdline/commands/cmd_computer.py @@ -603,14 +603,12 @@ def computer_delete(computer): it. """ from aiida.common.exceptions import InvalidOperation - from aiida.orm.backends import construct_backend - - backend = construct_backend() + from aiida import orm compname = computer.name try: - backend.computers.delete(computer.id) + orm.Computer.objects.delete(computer.id) except InvalidOperation as error: echo.echo_critical(str(error)) diff --git a/aiida/cmdline/commands/cmd_data/cmd_bands.py b/aiida/cmdline/commands/cmd_data/cmd_bands.py index 07acd25b43..e251d3b7b3 100644 --- a/aiida/cmdline/commands/cmd_data/cmd_bands.py +++ b/aiida/cmdline/commands/cmd_data/cmd_bands.py @@ -46,11 +46,11 @@ def bands(): @options.FORMULA_MODE() def bands_list(elements, elements_exclusive, raw, formula_mode, past_days, groups, all_users): """List BandsData objects.""" - from aiida.orm import construct_backend + from aiida.manage import get_manager from tabulate import tabulate from argparse import Namespace - backend = construct_backend() + backend = get_manager().get_backend() args = Namespace() args.element = elements diff --git a/aiida/cmdline/commands/cmd_process.py b/aiida/cmdline/commands/cmd_process.py index 8359a2f1ad..fae1b9fdb7 100644 --- a/aiida/cmdline/commands/cmd_process.py +++ b/aiida/cmdline/commands/cmd_process.py @@ -18,7 +18,7 @@ from aiida.cmdline.utils import decorators, echo from aiida.cmdline.utils.query.calculation import CalculationQueryBuilder from aiida.common.log import LOG_LEVELS -from aiida.manage.manager import AiiDAManager +from aiida.manage import get_manager @verdi.group('process') @@ -120,7 +120,7 @@ def process_status(processes): def process_kill(processes, timeout, wait): """Kill running processes.""" - controller = AiiDAManager.get_process_controller() + controller = get_manager().get_process_controller() futures = {} for process in processes: @@ -147,7 +147,7 @@ def process_kill(processes, timeout, wait): def process_pause(processes, timeout, wait): """Pause running processes.""" - controller = AiiDAManager.get_process_controller() + controller = get_manager().get_process_controller() futures = {} for process in processes: @@ -174,7 +174,7 @@ def process_pause(processes, timeout, wait): def process_play(processes, timeout, wait): """Play paused processes.""" - controller = AiiDAManager.get_process_controller() + controller = get_manager().get_process_controller() futures = {} for process in processes: @@ -201,7 +201,7 @@ def process_watch(processes): def _print(body, sender, subject, correlation_id): echo.echo('pk={}, subject={}, body={}, correlation_id={}'.format(sender, subject, body, correlation_id)) - communicator = AiiDAManager.get_communicator() + communicator = get_manager().get_communicator() for process in processes: diff --git a/aiida/cmdline/commands/cmd_user.py b/aiida/cmdline/commands/cmd_user.py index 33a2ff5bcd..f8d15cafb7 100644 --- a/aiida/cmdline/commands/cmd_user.py +++ b/aiida/cmdline/commands/cmd_user.py @@ -113,12 +113,14 @@ def user_list(color): List all the users. :param color: Show the list using colors """ - from aiida.common.utils import get_configured_user_email from aiida.common.exceptions import ConfigurationError + from aiida.manage import get_manager from aiida import orm + manager = get_manager() + try: - current_user = get_configured_user_email() + current_user = manager.get_profile().default_user_email except ConfigurationError: current_user = None diff --git a/aiida/common/__init__.py b/aiida/common/__init__.py index fbb55ae73b..cf680af245 100644 --- a/aiida/common/__init__.py +++ b/aiida/common/__init__.py @@ -7,7 +7,15 @@ # For further information on the license, see the LICENSE.txt file # # For further information please visit http://www.aiida.net # ########################################################################### +"""AiiDA common functionality""" + from __future__ import division from __future__ import print_function from __future__ import absolute_import from aiida.common.log import AIIDA_LOGGER + +# pylint: disable=wildcard-import + +from .profile import * + +__all__ = profile.__all__ diff --git a/aiida/common/datastructures.py b/aiida/common/datastructures.py index 122c99af3e..2f4ddf3ef1 100644 --- a/aiida/common/datastructures.py +++ b/aiida/common/datastructures.py @@ -13,6 +13,7 @@ from __future__ import division from __future__ import print_function from __future__ import absolute_import + from aiida.common.extendeddicts import DefaultFieldsAttributeDict, Enumerate diff --git a/aiida/common/profile.py b/aiida/common/profile.py index d22a0bc54e..57ffe6bf1d 100644 --- a/aiida/common/profile.py +++ b/aiida/common/profile.py @@ -7,6 +7,7 @@ # For further information on the license, see the LICENSE.txt file # # For further information please visit http://www.aiida.net # ########################################################################### +"""AiiDA profile related code""" from __future__ import division from __future__ import print_function from __future__ import absolute_import @@ -14,7 +15,9 @@ from aiida.backends import settings from aiida.common import setup +from aiida.common import exceptions +__all__ = 'Profile', 'get_current_profile_name', 'get_profile_config' CONFIG_DIR = setup.AIIDA_CONFIG_FOLDER DAEMON_DIR = 'daemon' @@ -47,7 +50,6 @@ def get_profile_config(name=None): :raises MissingConfigurationError: if the configuration file cannot be found :raises ProfileConfigurationError: if the name is not found in the configuration file """ - from aiida.common.exceptions import MissingConfigurationError, ProfileConfigurationError from aiida.common.setup import get_config if name is None: @@ -55,13 +57,13 @@ def get_profile_config(name=None): try: config = get_config() - except MissingConfigurationError: - raise MissingConfigurationError('could not load the configuration file') + except exceptions.MissingConfigurationError: + raise exceptions.MissingConfigurationError('could not load the configuration file') try: profile = config['profiles'][name] except KeyError: - raise ProfileConfigurationError('invalid profile name "{}"'.format(name)) + raise exceptions.ProfileConfigurationError('invalid profile name "{}"'.format(name)) return profile @@ -96,38 +98,92 @@ def __init__(self, name, config): self._config = config # Currently, whether a profile is a test profile is solely determined by its name starting with 'test_' - if self.name.startswith('test_'): - self._test_profile = True - else: - self._test_profile = False + self._test_profile = bool(self.name.startswith('test_')) - def get_option(self, option): + @staticmethod + def get_option(option): """Return the value of an option of this profile.""" from aiida.common.setup import get_property return get_property(option) @property def config(self): + """ + Get the profile configuration dictionary + + :return: the profile configuration + :rtype: dict + """ return self._config @property def name(self): + """ + Get the profile name + + :return: the profile name + :rtype: str + """ return self._name @property def uuid(self): + """ + Get the UUID of this profile + + :return: the profile UUID + """ return self.config[setup.PROFILE_UUID_KEY] @property def rmq_prefix(self): + """ + Get the RMQ prefix + + :return: the rmq prefix string + :rtype: str + """ return self._RMQ_PREFIX.format(uuid=self.uuid) @property def is_test_profile(self): + """ + Is this a test profile + + :return: True if test profile, False otherwise + :rtype: bool + """ return self._test_profile + @property + def default_user_email(self): + """ + Return the email (that is used as the username) configured during the + first verdi install. + + :return: the currently configured user email address + :rtype: str + """ + from aiida.common.setup import DEFAULT_USER_CONFIG_FIELD + + try: + email = self.config[DEFAULT_USER_CONFIG_FIELD] + # I do not catch the error in case of missing configuration, because + # it is already a ConfigurationError + except KeyError: + raise exceptions.ConfigurationError( + "No '{}' key found in the AiiDA configuration file".format(DEFAULT_USER_CONFIG_FIELD)) + + return email + @property def filepaths(self): + """ + Get the filepaths used by this profile + + :return: a dictionary of filepaths + :rtype: dict + """ return { 'circus': { 'log': CIRCUS_LOG_FILE_TEMPLATE.format(self.name), diff --git a/aiida/common/utils.py b/aiida/common/utils.py index 56277a7256..ad22580bbc 100644 --- a/aiida/common/utils.py +++ b/aiida/common/utils.py @@ -77,25 +77,6 @@ def __init__(self, callable): # pylint: disable=redefined-builtin super(abstractstaticmethod, self).__init__(callable) -def get_configured_user_email(): - """ - Return the email (that is used as the username) configured during the - first verdi install. - """ - from aiida.common.setup import get_profile_config, DEFAULT_USER_CONFIG_FIELD - from aiida.backends import settings - - try: - profile_conf = get_profile_config(settings.AIIDADB_PROFILE) - email = profile_conf[DEFAULT_USER_CONFIG_FIELD] - # I do not catch the error in case of missing configuration, because - # it is already a ConfigurationError - except KeyError: - raise ConfigurationError("No '{}' key found in the AiiDA configuration file".format(DEFAULT_USER_CONFIG_FIELD)) - - return email - - def get_new_uuid(): """ Return a new UUID (typically to be used for new nodes). @@ -976,12 +957,12 @@ def _prettify_label_agr(cls, label): """ label = ( - label + label .replace('GAMMA', r'\xG\f{}') .replace('DELTA', r'\xD\f{}') .replace('LAMBDA', r'\xL\f{}') .replace('SIGMA', r'\xS\f{}') - ) # yapf:disable + ) # yapf:disable return re.sub(r'_(.?)', r'\\s\1\\N', label) @classmethod @@ -1008,12 +989,12 @@ def _prettify_label_gnuplot(cls, label): """ label = ( - label + label .replace(u'GAMMA', u'Γ') .replace(u'DELTA', u'Δ') .replace(u'LAMBDA', u'Λ') .replace(u'SIGMA', u'Σ') - ) # yapf:disable + ) # yapf:disable return re.sub(r'_(.?)', r'_{\1}', label) @classmethod @@ -1040,12 +1021,12 @@ def _prettify_label_latex(cls, label): """ label = ( - label + label .replace('GAMMA', r'$\Gamma$') .replace('DELTA', r'$\Delta$') .replace('LAMBDA', r'$\Lambda$') .replace('SIGMA', r'$\Sigma$') - ) # yapf:disable + ) # yapf:disable label = re.sub(r'_(.?)', r'$_{\1}$', label) # label += r"$_{\vphantom{0}}$" diff --git a/aiida/control/profile.py b/aiida/control/profile.py index 117802abc8..dc03055cec 100644 --- a/aiida/control/profile.py +++ b/aiida/control/profile.py @@ -155,16 +155,18 @@ def setup_profile(profile, only_config, set_default=False, non_interactive=False load_dbenv() from aiida.common.setup import DEFAULT_AIIDA_USER + from aiida.manage import get_manager from aiida import orm + manager = get_manager() + if not orm.User.objects.find({'email': DEFAULT_AIIDA_USER}): echo.echo("Installing default AiiDA user...") nuser = orm.User(email=DEFAULT_AIIDA_USER, first_name="AiiDA", last_name="Daemon") nuser.is_active = True nuser.store() - from aiida.common.utils import get_configured_user_email - email = get_configured_user_email() + email = manager.get_profile().default_user_email echo.echo("Starting user configuration for {}...".format(email)) if email == DEFAULT_AIIDA_USER: echo.echo("You set up AiiDA using the default Daemon email ({}),".format(email)) diff --git a/aiida/daemon/runner.py b/aiida/daemon/runner.py index 47584c949a..2619a55b96 100644 --- a/aiida/daemon/runner.py +++ b/aiida/daemon/runner.py @@ -18,7 +18,7 @@ from aiida.common.log import configure_logging from aiida.daemon.client import get_daemon_client from aiida import work -from aiida.manage.manager import AiiDAManager +from aiida.manage import get_manager logger = logging.getLogger(__name__) @@ -34,12 +34,12 @@ def start_daemon(): configure_logging(daemon=True, daemon_log_file=daemon_client.daemon_log_file) try: - runner = AiiDAManager.create_daemon_runner() + runner = get_manager().create_daemon_runner() except Exception as exception: logger.exception('daemon runner failed to start') raise - def shutdown_daemon(num, frame): + def shutdown_daemon(_num, _frame): logger.info('Received signal to shut down the daemon runner') runner.close() diff --git a/aiida/manage/__init__.py b/aiida/manage/__init__.py index a7e3fad50c..b3f990853b 100644 --- a/aiida/manage/__init__.py +++ b/aiida/manage/__init__.py @@ -7,3 +7,20 @@ # For further information on the license, see the LICENSE.txt file # # For further information please visit http://www.aiida.net # ########################################################################### +""" +A module to bring together the different parts of AiIDA: + + * backend + * profile/settings + * daemon/workflow runner + * etc. +""" +from __future__ import division +from __future__ import print_function +from __future__ import absolute_import + +# pylint: disable=wildcard-import + +from .manager import * + +__all__ = manager.__all__ # pylint: disable=undefined-variable diff --git a/aiida/manage/database/integrity.py b/aiida/manage/database/integrity.py index 5c46320d87..fbaa841fa2 100644 --- a/aiida/manage/database/integrity.py +++ b/aiida/manage/database/integrity.py @@ -13,6 +13,7 @@ from __future__ import absolute_import import uuid as UUID +from aiida.manage import get_manager from aiida.common.exceptions import IntegrityError @@ -33,9 +34,7 @@ def get_duplicate_node_uuids(): :return: list of tuples of (pk, uuid) of nodes with duplicate UUIDs """ - from aiida.orm.backends import construct_backend - - backend = construct_backend() + backend = get_manager().get_backend() duplicates = backend.query_manager.get_duplicate_node_uuids() return duplicates diff --git a/aiida/manage/manager.py b/aiida/manage/manager.py index 20fea50c94..7b89c95d0d 100644 --- a/aiida/manage/manager.py +++ b/aiida/manage/manager.py @@ -16,11 +16,14 @@ import plumpy from aiida import utils +from aiida import common -__all__ = ('AiiDAManager',) +__all__ = 'get_manager', 'reset_manager' +_manager = None # pylint: disable=invalid-name -class AiiDAManager(object): + +class Manager(object): """ Manager singleton to provide global versions of commonly used profile/settings related objects and methods to facilitate their construction. @@ -34,32 +37,42 @@ class AiiDAManager(object): Future plans: * reset manager cache when loading a new profile - * move construct_backend() from orm.backends inside the AiiDAManager """ - _profile = None # type: aiida.common.profile.Profile - _communicator = None # type: kiwipy.rmq.RmqThreadCommunicator - _process_controller = None # type: plumpy.RemoteProcessThreadController - _persister = None # type: aiida.work.AiiDAPersister - _runner = None # type: aiida.work.Runner + def get_backend(self): + """ + Get the database backend - @classmethod - def get_profile(cls): + :return: the database backend + :rtype: :class:`aiida.orm.Backend` + """ + if self._backend is None: + from aiida.backends.profile import BACKEND_DJANGO, BACKEND_SQLA + backend_type = self.get_profile().config.get('AIIDADB_BACKEND') + if backend_type == BACKEND_DJANGO: + from aiida.orm.implementation.django.backend import DjangoBackend + self._backend = DjangoBackend() + elif backend_type == BACKEND_SQLA: + from aiida.orm.implementation.sqlalchemy.backend import SqlaBackend + self._backend = SqlaBackend() + else: + raise RuntimeError("Invalid backend type in profile: {}".format(backend_type)) + + return self._backend + + def get_profile(self): """ Get the current profile :return: current loaded profile instance :rtype: :class:`~aiida.common.profile.Profile` """ - from aiida.common import profile - - if cls._profile is None: - cls._profile = profile.get_profile() + if self._profile is None: + self._profile = common.profile.get_profile() - return cls._profile + return self._profile - @classmethod - def get_persister(cls): + def get_persister(self): """ Get the persister @@ -68,26 +81,24 @@ def get_persister(cls): """ from aiida.work import persistence - if cls._persister is None: - cls._persister = persistence.AiiDAPersister() + if self._persister is None: + self._persister = persistence.AiiDAPersister() - return cls._persister + return self._persister - @classmethod - def get_communicator(cls): + def get_communicator(self): """ Get the communicator :return: a global communicator instance :rtype: :class:`kiwipy.Communicator` """ - if cls._communicator is None: - cls._communicator = cls.create_communicator() + if self._communicator is None: + self._communicator = self.create_communicator() - return cls._communicator + return self._communicator - @classmethod - def create_communicator(cls, task_prefetch_count=None): + def create_communicator(self, task_prefetch_count=None): """ Create a Communicator @@ -97,7 +108,7 @@ def create_communicator(cls, task_prefetch_count=None): """ from aiida.work import rmq import kiwipy.rmq - profile = cls.get_profile() + profile = self.get_profile() if task_prefetch_count is None: task_prefetch_count = rmq._RMQ_TASK_PREFETCH_COUNT # pylint: disable=protected-access @@ -124,47 +135,43 @@ def create_communicator(cls, task_prefetch_count=None): testing_mode=testing_mode, ) - @classmethod - def get_process_controller(cls): + def get_process_controller(self): """ Get a process controller :return: the process controller instance :rtype: :class:`plumpy.RemoteProcessThreadController` """ - if cls._process_controller is None: - cls._process_controller = plumpy.RemoteProcessThreadController(cls.get_communicator()) + if self._process_controller is None: + self._process_controller = plumpy.RemoteProcessThreadController(self.get_communicator()) - return cls._process_controller + return self._process_controller - @classmethod - def get_runner(cls): + def get_runner(self): """ Get a runner that is based on the current profile settings and can be used globally by the code. :return: the global runner :rtype: :class:`aiida.work.Runner` """ - if cls._runner is None: - cls._runner = cls.create_runner() + if self._runner is None: + self._runner = self.create_runner() - return cls._runner + return self._runner - @classmethod - def set_runner(cls, new_runner): + def set_runner(self, new_runner): """ Set the currently used runner :param new_runner: the new runner to use :type new_runner: :class:`aiida.work.Runner` """ - if cls._runner is not None: - cls._runner.close() + if self._runner is not None: + self._runner.close() - cls._runner = new_runner + self._runner = new_runner - @classmethod - def create_runner(cls, with_persistence=True, **kwargs): + def create_runner(self, with_persistence=True, **kwargs): """ Create a new runner @@ -175,7 +182,7 @@ def create_runner(cls, with_persistence=True, **kwargs): """ from aiida.work import runners - profile = cls.get_profile() + profile = self.get_profile() poll_interval = 0.0 if profile.is_test_profile else profile.get_option('runner.poll.interval') settings = {'rmq_submit': False, 'poll_interval': poll_interval} @@ -183,15 +190,14 @@ def create_runner(cls, with_persistence=True, **kwargs): if 'communicator' not in settings: # Only call get_communicator if we have to as it will lazily create - settings['communicator'] = cls.get_communicator() + settings['communicator'] = self.get_communicator() if with_persistence and 'persister' not in settings: - settings['persister'] = cls.get_persister() + settings['persister'] = self.get_persister() return runners.Runner(**settings) - @classmethod - def create_daemon_runner(cls, loop=None): + def create_daemon_runner(self, loop=None): """ Create a new daemon runner. This is used by workers when the daemon is running and in testing. @@ -201,13 +207,13 @@ def create_daemon_runner(cls, loop=None): :rtype: :class:`aiida.work.Runner` """ from aiida.work import rmq, persistence - runner = cls.create_runner(rmq_submit=True, loop=loop) + runner = self.create_runner(rmq_submit=True, loop=loop) runner_loop = runner.loop # Listen for incoming launch requests task_receiver = rmq.ProcessLauncher( loop=runner_loop, - persister=cls.get_persister(), + persister=self.get_persister(), load_context=plumpy.LoadSaveContext(runner=runner), loader=persistence.get_object_loader()) @@ -218,22 +224,40 @@ def callback(*args, **kwargs): return runner - @classmethod - def reset(cls): + def close(self): """ Reset the global settings entirely and release any global objects """ - if cls._communicator is not None: - cls._communicator.stop() - if cls._runner is not None: - cls._runner.stop() + if self._communicator is not None: + self._communicator.stop() + if self._runner is not None: + self._runner.stop() - cls._profile = None - cls._persister = None - cls._communicator = None - cls._process_controller = None - cls._runner = None + self._profile = None + self._persister = None + self._communicator = None + self._process_controller = None + self._runner = None def __init__(self): - """Can't instantiate this class""" - raise NotImplementedError("Can't instantiate") + super(Manager, self).__init__() + self._backend = None # type: aiida.orm.Backend + self._profile = None # type: aiida.common.profile.Profile + self._communicator = None # type: kiwipy.rmq.RmqThreadCommunicator + self._process_controller = None # type: plumpy.RemoteProcessThreadController + self._persister = None # type: aiida.work.AiiDAPersister + self._runner = None # type: aiida.work.Runner + + +def get_manager(): + global _manager # pylint: disable=invalid-name, global-statement + if _manager is None: + _manager = Manager() + return _manager + + +def reset_manager(): + global _manager # pylint: disable=invalid-name, global-statement + if _manager is not None: + _manager.close() + _manager = None diff --git a/aiida/orm/__init__.py b/aiida/orm/__init__.py index 9e3aaf81a5..f390e97054 100644 --- a/aiida/orm/__init__.py +++ b/aiida/orm/__init__.py @@ -13,12 +13,12 @@ from __future__ import division from __future__ import print_function from __future__ import absolute_import -from aiida.orm.data import * -from aiida.orm.data.code import Code -from aiida.orm.workflow import Workflow + +from .data import * +from .data.code import Code +from .workflow import Workflow from .authinfos import * -from .backends import * from .calculation import * from .computers import * from .entities import * @@ -41,7 +41,6 @@ __all__ = (_local + authinfos.__all__ + - backends.__all__ + calculation.__all__ + computers.__all__ + entities.__all__ + diff --git a/aiida/orm/authinfos.py b/aiida/orm/authinfos.py index 03a07fa720..f7f65415d1 100644 --- a/aiida/orm/authinfos.py +++ b/aiida/orm/authinfos.py @@ -14,7 +14,7 @@ from aiida.transport import TransportFactory from aiida.common.exceptions import (ConfigurationError, MissingPluginError) -from . import backends +from aiida.manage import get_manager from . import entities from . import users @@ -48,7 +48,7 @@ def __init__(self, computer, user, backend=None): :param user: a User instance :return: an AuthInfo object associated with the given computer and user """ - backend = backend or backends.construct_backend() + backend = backend or get_manager().get_backend() model = backend.authinfos.create(computer=computer.backend_entity, user=user.backend_entity) super(AuthInfo, self).__init__(model) diff --git a/aiida/orm/backends.py b/aiida/orm/backends.py deleted file mode 100644 index 6149c5cd2d..0000000000 --- a/aiida/orm/backends.py +++ /dev/null @@ -1,64 +0,0 @@ -# -*- coding: utf-8 -*- -########################################################################### -# Copyright (c), The AiiDA team. All rights reserved. # -# This file is part of the AiiDA code. # -# # -# The code is hosted on GitHub at https://github.com/aiidateam/aiida_core # -# For further information on the license, see the LICENSE.txt file # -# For further information please visit http://www.aiida.net # -########################################################################### -"""Module for backend related methods and classes""" -from __future__ import division -from __future__ import print_function -from __future__ import absolute_import - -from aiida.backends import settings - -__all__ = 'construct_backend', 'CollectionEntry' - -_DJANGO_BACKEND = None -_SQLA_BACKEND = None - - -def construct_backend(backend_type=None): - """ - Construct a concrete backend instance based on the backend_type or use the global backend value if not specified. - - :param backend_type: get a backend instance based on the specified type (or default) - :return: :class:`aiida.orm.implementation.Backend` - """ - # pylint: disable=global-statement - if backend_type is None: - - backend_type = settings.BACKEND - - if backend_type == 'django': - global _DJANGO_BACKEND - if _DJANGO_BACKEND is None: - from aiida.orm.implementation.django.backend import DjangoBackend - _DJANGO_BACKEND = DjangoBackend() - return _DJANGO_BACKEND - elif backend_type == 'sqlalchemy': - global _SQLA_BACKEND - if _SQLA_BACKEND is None: - from aiida.orm.implementation.sqlalchemy.backend import SqlaBackend - _SQLA_BACKEND = SqlaBackend() - return _SQLA_BACKEND - else: - raise ValueError("The specified backend {} is currently not implemented".format(backend_type)) - - -class CollectionEntry(object): - """Class that represents an entry within a collection of entries of a particular backend entity.""" - - def __init__(self, backend=None): - """ - :param backend: The backend instance - :type backend: :class:`aiida.orm.implementation.backends.Backend` - """ - self._backend = backend or construct_backend() - - @property - def backend(self): - """Return the backend.""" - return self._backend diff --git a/aiida/orm/comments.py b/aiida/orm/comments.py index f75bfa8cc9..10901c164a 100644 --- a/aiida/orm/comments.py +++ b/aiida/orm/comments.py @@ -12,7 +12,7 @@ from __future__ import print_function from __future__ import absolute_import -from . import backends +from aiida.manage import get_manager from . import entities from . import users @@ -53,7 +53,7 @@ def __init__(self, node, user, content=None, backend=None): :param content: the comment content :return: a Comment object associated to the given node and user """ - backend = backend or backends.construct_backend() + backend = backend or get_manager().get_backend() model = backend.comments.create(node=node, user=user.backend_entity, content=content) super(Comment, self).__init__(model) diff --git a/aiida/orm/computers.py b/aiida/orm/computers.py index e0441c5e0b..9b4da0fda8 100644 --- a/aiida/orm/computers.py +++ b/aiida/orm/computers.py @@ -18,7 +18,7 @@ from aiida import transport, scheduler from aiida.common import exceptions -from . import backends +from aiida.manage import get_manager from . import entities from . import users @@ -173,7 +173,7 @@ def __init__(self, backend=None): """Construct a new computer""" # pylint: disable=too-many-arguments - backend = backend or backends.construct_backend() + backend = backend or get_manager().get_backend() model = backend.computers.create( name=name, hostname=hostname, diff --git a/aiida/orm/data/remote.py b/aiida/orm/data/remote.py index dfeeb612e1..43091ae276 100644 --- a/aiida/orm/data/remote.py +++ b/aiida/orm/data/remote.py @@ -188,4 +188,4 @@ def _validate(self): raise ValidationError("Remote computer not set.") def _get_authinfo(self): - return AuthInfo.objects(backend=self._backend).get(dbcomputer=self.get_computer(), aiidauser=self.get_user()) + return AuthInfo.objects.get(dbcomputer=self.get_computer(), aiidauser=self.get_user()) diff --git a/aiida/orm/entities.py b/aiida/orm/entities.py index 6561cffab2..a66222e7d1 100644 --- a/aiida/orm/entities.py +++ b/aiida/orm/entities.py @@ -17,7 +17,7 @@ from aiida.common import exceptions from aiida.common import datastructures from aiida.common.utils import classproperty, type_check -from . import backends +from aiida.manage import get_manager __all__ = ('Entity', 'Collection') @@ -44,11 +44,8 @@ def get_collection(cls, entity_type, backend): def __init__(self, backend, entity_class): """Construct a new entity collection""" - # assert issubclass(entity_class, Entity), "Must provide an entity type" - if backend is None: - from . import backends - backend = backend or backends.construct_backend() - self._backend = backend + assert issubclass(entity_class, Entity), "Must provide an entity type" + self._backend = backend or get_manager().get_backend() self._entity_type = entity_class def __call__(self, backend): @@ -136,10 +133,9 @@ def all(self): class Entity(object): """An AiiDA entity""" - _backend = None _objects = None - # Define out collection type + # Define our collection type Collection = Collection @classproperty @@ -150,25 +146,8 @@ def objects(cls, backend=None): # pylint: disable=no-self-use, no-self-argument :param backend: the optional backend to use (otherwise use default) :return: an object that can be used to access entities of this type """ - new_backend = backend or cls._backend or backends.construct_backend() - - if type(cls._backend) != type(new_backend): # pylint: disable=unidiomatic-typecheck - # if the backend has changed, update cache - cls._backend = new_backend - cls._objects = cls.Collection(cls._backend, cls) - else: - # if not, use cache (if already populated) - cls._objects = cls._objects or cls.Collection(cls._backend, cls) - - return cls._objects - - @classmethod - def reset(cls): - """ - Reset the backend and objects cache. - """ - cls._backend = None - cls._backendobjectS = None + backend = backend or get_manager().get_backend() + return cls.Collection.get_collection(cls, backend) @classmethod def get(cls, **kwargs): diff --git a/aiida/orm/groups.py b/aiida/orm/groups.py index c51d2cb878..b6feb15a0f 100644 --- a/aiida/orm/groups.py +++ b/aiida/orm/groups.py @@ -14,7 +14,7 @@ from aiida.common import exceptions from aiida.common.utils import type_check -from . import backends +from aiida.manage import get_manager from . import entities from . import users @@ -173,7 +173,7 @@ def __init__(self, name, user=None, description='', type_string='', backend=None :param type_string: a string identifying the type of group (by default, an empty string, indicating an user-defined group. """ - backend = backend or backends.construct_backend() + backend = backend or get_manager().get_backend() user = user or users.User.objects(backend).get_default() type_check(user, users.User) diff --git a/aiida/orm/implementation/django/node.py b/aiida/orm/implementation/django/node.py index 5b70dfe9e4..732815f42e 100644 --- a/aiida/orm/implementation/django/node.py +++ b/aiida/orm/implementation/django/node.py @@ -26,7 +26,6 @@ from aiida.common.utils import get_new_uuid, type_check from aiida.orm.implementation.general.node import AbstractNode, _HASH_EXTRA_KEY from . import computer as computers -from aiida.manage.manager import AiiDAManager class Node(AbstractNode): @@ -124,7 +123,7 @@ def __init__(self, **kwargs): # uuid, self.__class__.__name__, e.message)) else: # TODO: allow to get the user from the parameters - user = orm.User.objects(backend=self._backend).get_default().backend_entity + user = orm.User.objects.get_default().backend_entity self._dbnode = DbNode(user=user.dbmodel, uuid=get_new_uuid(), type=self._plugin_type_string) diff --git a/aiida/orm/implementation/django/workflow.py b/aiida/orm/implementation/django/workflow.py index 67a379fad4..f334a43919 100644 --- a/aiida/orm/implementation/django/workflow.py +++ b/aiida/orm/implementation/django/workflow.py @@ -51,9 +51,9 @@ def __init__(self, **kwargs): """ from aiida.backends.djsite.db.models import DbWorkflow from aiida import orm - from aiida.orm.backends import construct_backend + from aiida.manage import get_manager - self._backend = construct_backend() + self._backend = get_manager().get_backend() self._to_be_stored = True self._logger = logger.getChild(self.__class__.__name__) diff --git a/aiida/orm/implementation/general/node.py b/aiida/orm/implementation/general/node.py index 2a7cee7392..396bf4bccb 100644 --- a/aiida/orm/implementation/general/node.py +++ b/aiida/orm/implementation/general/node.py @@ -31,6 +31,7 @@ from aiida.common.links import LinkType from aiida.common.utils import abstractclassmethod, sql_string_match from aiida.common.utils import combomethod, classproperty +from aiida.manage import get_manager from aiida.plugins.loader import get_query_type_from_type_string, get_type_string_from_class from aiida.orm.utils import links @@ -303,8 +304,6 @@ def __init__(self, **kwargs): loaded from the database. (It is not possible to assign a uuid to a new Node.) """ - from aiida.orm.backends import construct_backend - self._to_be_stored = True self._attrs_cache = {} @@ -314,7 +313,7 @@ def __init__(self, **kwargs): self._temp_folder = None self._repo_folder = None - self._backend = construct_backend() + self._backend = get_manager().get_backend() def __repr__(self): return '<{}: {}>'.format(self.__class__.__name__, str(self)) diff --git a/aiida/orm/implementation/general/workflow.py b/aiida/orm/implementation/general/workflow.py index a0a10a3f8b..4aae6ac0a4 100644 --- a/aiida/orm/implementation/general/workflow.py +++ b/aiida/orm/implementation/general/workflow.py @@ -24,7 +24,6 @@ from aiida.common.utils import str_timedelta from aiida.common import AIIDA_LOGGER from aiida.orm.node.process import CalcJobNode -from aiida.orm.backends import construct_backend from aiida.utils import timezone from aiida.common.log import create_logger_adapter from aiida.common.utils import abstractclassmethod @@ -95,7 +94,6 @@ def __init__(self, **kwargs): the given uuid. """ super(AbstractWorkflow, self).__init__() - self._backend = construct_backend() def __repr__(self): return '<{}: {}>'.format(self.__class__.__name__, str(self)) @@ -234,11 +232,6 @@ def is_stored(self): """ return not self._to_be_stored - @property - def backend(self): - """Get the backend of this workflow""" - return self._backend - def get_folder_list(self, subfolder='.'): """ Get the the list of files/directory in the repository of the object. @@ -590,7 +583,7 @@ def wrapper(self, *args, **kwargs): # self.get_steps(wrapped_method).set_nextcall(wf_exit_call) - automatic_user = orm.User.objects(self._backend).get_default().backend_entity + automatic_user = orm.User.objects.get_default().backend_entity method_step, created = self.dbworkflowinstance.steps.get_or_create( name=wrapped_method, user=automatic_user.dbmodel) try: @@ -666,7 +659,7 @@ def next(self, next_method): # arround # Retrieve the caller method - user = orm.User.objects(self._backend).get_default().backend_entity.dbmodel + user = orm.User.objects.get_default().backend_entity.dbmodel method_step = self.dbworkflowinstance.steps.get(name=caller_method, user=user) # Attach calculations diff --git a/aiida/orm/implementation/sqlalchemy/__init__.py b/aiida/orm/implementation/sqlalchemy/__init__.py index f2951f79ac..43003866d3 100644 --- a/aiida/orm/implementation/sqlalchemy/__init__.py +++ b/aiida/orm/implementation/sqlalchemy/__init__.py @@ -8,7 +8,6 @@ # For further information please visit http://www.aiida.net # ########################################################################### - from .backend import * from .groups import * from .users import * diff --git a/aiida/orm/implementation/sqlalchemy/authinfo.py b/aiida/orm/implementation/sqlalchemy/authinfo.py index 28bfc6dfb6..fa71984207 100644 --- a/aiida/orm/implementation/sqlalchemy/authinfo.py +++ b/aiida/orm/implementation/sqlalchemy/authinfo.py @@ -14,7 +14,6 @@ from aiida.common import exceptions from aiida.common.utils import type_check from aiida.orm.implementation.authinfos import BackendAuthInfo, BackendAuthInfoCollection -from aiida.orm.backends import construct_backend from . import entities from . import computer as computers from . import users as users diff --git a/aiida/orm/implementation/sqlalchemy/node.py b/aiida/orm/implementation/sqlalchemy/node.py index d5a9365454..6a3dc5a536 100644 --- a/aiida/orm/implementation/sqlalchemy/node.py +++ b/aiida/orm/implementation/sqlalchemy/node.py @@ -28,7 +28,6 @@ from aiida.common.utils import type_check from aiida.orm.implementation.general.node import AbstractNode, _HASH_EXTRA_KEY from aiida.orm.implementation.sqlalchemy.utils import get_attr -from aiida.manage.manager import AiiDAManager from . import computer as computers diff --git a/aiida/orm/implementation/sqlalchemy/workflow.py b/aiida/orm/implementation/sqlalchemy/workflow.py index 861e86badb..ca62b9277c 100644 --- a/aiida/orm/implementation/sqlalchemy/workflow.py +++ b/aiida/orm/implementation/sqlalchemy/workflow.py @@ -52,9 +52,6 @@ def __init__(self, **kwargs): the given uuid. """ from aiida import orm - from aiida.orm.backends import construct_backend - - self._backend = construct_backend() self._to_be_stored = True @@ -121,7 +118,7 @@ def __init__(self, **kwargs): if isinstance(params, dict): self.set_params(params) - user = orm.User.objects(self._backend).get_default().backend_entity + user = orm.User.objects.get_default().backend_entity # This stores the MD5 as well, to test in case the workflow has # been modified after the launch @@ -459,7 +456,7 @@ def get_step(self, step_method): raise InternalError("Cannot query a step with name {0}, reserved string".format(step_method_name)) step_list = self.dbworkflowinstance.steps - automatic_user = orm.User.objects(self._backend).get_default().backend_entity + automatic_user = orm.User.objects.get_default().backend_entity step = [_ for _ in step_list if _.name == step_method_name and _.user == automatic_user.dbmodel] try: return step[0] @@ -625,7 +622,7 @@ def wrapper(self, *args, **kwargs): # self.get_steps(wrapped_method).set_nextcall(wf_exit_call) - user = orm.User.objects(self._backend).get_default().backend_entity + user = orm.User.objects.get_default().backend_entity method_step, created = self.dbworkflowinstance._get_or_create_step(name=wrapped_method, user=user.dbmodel) @@ -701,7 +698,7 @@ def next(self, next_method): # with particular filters, in order to avoid repetition of all the code # arround - automatic_user = orm.User.objects(self._backend).get_default().backend_entity + automatic_user = orm.User.objects.get_default().backend_entity # Retrieve the caller method method_step, _ = self.dbworkflowinstance._get_or_create_step(name=caller_method, user=automatic_user.dbmodel) diff --git a/aiida/orm/logs.py b/aiida/orm/logs.py index c4201c71b1..a85b64d395 100644 --- a/aiida/orm/logs.py +++ b/aiida/orm/logs.py @@ -13,7 +13,7 @@ from __future__ import absolute_import from aiida.utils import timezone -from . import backends +from aiida.manage import get_manager from . import entities from . import node @@ -98,7 +98,7 @@ def delete_many(self, filters): def __init__(self, time, loggername, levelname, objname, objpk=None, message='', metadata=None, backend=None): # pylint: disable=too-many-arguments """Construct a new computer""" - backend = backend or backends.construct_backend() + backend = backend or get_manager().get_backend() model = backend.logs.create( time=time, loggername=loggername, diff --git a/aiida/orm/querybuilder.py b/aiida/orm/querybuilder.py index 8bd828617d..0a90090eb0 100644 --- a/aiida/orm/querybuilder.py +++ b/aiida/orm/querybuilder.py @@ -37,11 +37,8 @@ from aiida.common.exceptions import InputValidationError # The way I get column as a an attribute to the orm class from aiida.common.links import LinkType +from aiida.manage import get_manager from aiida.orm.node import Node -from aiida.orm import backends -from aiida.orm import computers -from aiida.orm import users -from aiida.orm import authinfos from aiida.orm.utils import convert from . import authinfos @@ -52,7 +49,7 @@ from . import logs from . import users -__all__ = ('QueryBuilder', ) +__all__ = ('QueryBuilder',) _LOGGER = logging.getLogger(__name__) @@ -239,7 +236,7 @@ def __init__(self, backend=None, **kwargs): check :func:`QueryBuilder.order_by` for more information. """ - backend = backend or backends.construct_backend() + backend = backend or get_manager().get_backend() self._impl = backend.query() # A list storing the path being traversed by the query @@ -552,7 +549,6 @@ def append(self, if not isinstance(type, six.string_types): raise InputValidationError("{} was passed as type, but is not a string".format(type)) - ormclass, ormclasstype, query_type_string = self._get_ormclass(cls, type) # TAG ################################# @@ -667,8 +663,8 @@ def append(self, "{} is not a valid keyword " "for joining specification\n" "Valid keywords are: " - "{}".format( - key, spec_to_function_map + ['cls', 'type', 'tag', 'autotag', 'filters', 'project'])) + "{}".format(key, + spec_to_function_map + ['cls', 'type', 'tag', 'autotag', 'filters', 'project'])) elif joining_keyword: raise InputValidationError("You already specified joining specification {}\n" "But you now also want to specify {}" @@ -902,6 +898,7 @@ def _add_type_filter(self, tagspec, query_type_string, plugin_type_string, subcl """ Add a filter on the type based on the query_type_string """ + def get_type_filter(q, p): if subclassing: return {'like': '{}%'.format(q)} @@ -1458,7 +1455,8 @@ def _join_node_comment(self, joined_entity, entity_to_join, isouterjoin): :param entity_to_join: aliased comment """ self._check_dbentities((joined_entity, self._impl.Node), (entity_to_join, self._impl.Comment), 'with_node') - self._query = self._query.join(entity_to_join, joined_entity.id == entity_to_join.dbnode_id, isouter=isouterjoin) + self._query = self._query.join( + entity_to_join, joined_entity.id == entity_to_join.dbnode_id, isouter=isouterjoin) def _join_comment_node(self, joined_entity, entity_to_join, isouterjoin): """ @@ -1466,7 +1464,8 @@ def _join_comment_node(self, joined_entity, entity_to_join, isouterjoin): :param entity_to_join: aliased node """ self._check_dbentities((joined_entity, self._impl.Comment), (entity_to_join, self._impl.Node), 'with_comment') - self._query = self._query.join(entity_to_join, joined_entity.dbnode_id == entity_to_join.id, isouter=isouterjoin) + self._query = self._query.join( + entity_to_join, joined_entity.dbnode_id == entity_to_join.id, isouter=isouterjoin) def _get_function_map(self): """ @@ -1768,6 +1767,7 @@ def except_if_input_to(self, calc_class): :returns: self """ + def build_counterquery(calc_class): if issubclass(calc_class, self.Node): orm_calc_class = calc_class @@ -2146,6 +2146,7 @@ def _deprecate(self, function, deprecated_name, preferred_name, version='1.0.0a5 :param preferred_name: the new name which is preferred :param version: aiida version for which this takes effect. """ + def wrapper(*args, **kwargs): """ Decorator to print a deprecation warning @@ -2158,4 +2159,5 @@ def wrapper(*args, **kwargs): AiidaDeprecationWarning, stacklevel=2) return function(*args, **kwargs) + return wrapper diff --git a/aiida/orm/users.py b/aiida/orm/users.py index 73f9391da5..9be4f73b14 100644 --- a/aiida/orm/users.py +++ b/aiida/orm/users.py @@ -18,25 +18,21 @@ from aiida.common.hashing import is_password_usable from aiida.common import exceptions from aiida.utils.email import normalize_email - -from . import backends +from aiida.manage import get_manager from . import entities __all__ = ('User',) class User(entities.Entity): - """ - This is the base class for User information in AiiDA. An implementing - backend needs to provide a concrete version. - """ + """AiiDA User""" class Collection(entities.Collection): """ - The collection of users stored in a backend - """ + The collection of users stored in a backend + """ UNDEFINED = 'UNDEFINED' - _default_user = None # type: aiida.orm.user.User + _default_user = None # type: aiida.orm.User def __init__(self, *args, **kwargs): super(User.Collection, self).__init__(*args, **kwargs) @@ -44,13 +40,14 @@ def __init__(self, *args, **kwargs): def get_or_create(self, **kwargs): """ - Get the existing user with a given email address or create an unstored one - - :param kwargs: The properties of the user to get or create - :return: The corresponding user object - :rtype: :class:`aiida.orm.User` - :raises: :class:`aiida.common.exceptions.MultipleObjectsError`, :class:`aiida.common.exceptions.NotExistent` - """ + Get the existing user with a given email address or create an unstored one + + :param kwargs: The properties of the user to get or create + :return: The corresponding user object + :rtype: :class:`aiida.orm.User` + :raises: :class:`aiida.common.exceptions.MultipleObjectsError`, + :class:`aiida.common.exceptions.NotExistent` + """ try: return False, self.get(**kwargs) except exceptions.NotExistent: @@ -64,8 +61,7 @@ def get_default(self): :rtype: :class:`aiida.orm.User` """ if self._default_user is self.UNDEFINED: - from aiida.common.utils import get_configured_user_email - email = get_configured_user_email() + email = get_manager().get_profile().default_user_email if not email: self._default_user = None @@ -79,7 +75,7 @@ def get_default(self): REQUIRED_FIELDS = ['first_name', 'last_name', 'institution'] def __init__(self, email, first_name='', last_name='', institution='', backend=None): - backend = backend or backends.construct_backend() + backend = backend or get_manager().get_backend() email = normalize_email(email) backend_entity = backend.users.create(email, first_name, last_name, institution) super(User, self).__init__(backend_entity) diff --git a/aiida/restapi/translator/code.py b/aiida/restapi/translator/code.py index 49bac0e119..c8db344af7 100644 --- a/aiida/restapi/translator/code.py +++ b/aiida/restapi/translator/code.py @@ -15,6 +15,7 @@ from __future__ import print_function from __future__ import absolute_import from aiida.restapi.translator.node import NodeTranslator +from aiida.orm.code import Code class CodeTranslator(NodeTranslator): @@ -25,7 +26,6 @@ class CodeTranslator(NodeTranslator): # A label associated to the present class (coincides with the resource name) __label__ = "codes" # The AiiDA class one-to-one associated to the present class - from aiida.orm.code import Code _aiida_class = Code # The string name of the AiiDA class _aiida_type = "code.Code" diff --git a/aiida/restapi/translator/computer.py b/aiida/restapi/translator/computer.py index dec3ce7f70..c096b4b496 100644 --- a/aiida/restapi/translator/computer.py +++ b/aiida/restapi/translator/computer.py @@ -15,18 +15,17 @@ from __future__ import print_function from __future__ import absolute_import from aiida.restapi.translator.base import BaseTranslator +from aiida import orm class ComputerTranslator(BaseTranslator): """ Translator relative to resource 'computers' and aiida class Computer """ - # A label associated to the present class (coincides with the resource name) __label__ = "computers" # The AiiDA class one-to-one associated to the present class - from aiida.orm import Computer - _aiida_class = Computer + _aiida_class = orm.Computer # The string name of the AiiDA class _aiida_type = "Computer" # The string associated to the AiiDA class in the query builder lexicon diff --git a/aiida/restapi/translator/group.py b/aiida/restapi/translator/group.py index 0ed99deed1..414258c727 100644 --- a/aiida/restapi/translator/group.py +++ b/aiida/restapi/translator/group.py @@ -15,6 +15,7 @@ from __future__ import print_function from __future__ import absolute_import from aiida.restapi.translator.base import BaseTranslator +from aiida import orm class GroupTranslator(BaseTranslator): @@ -25,8 +26,7 @@ class GroupTranslator(BaseTranslator): # A label associated to the present class (coincides with the resource name) __label__ = "groups" # The AiiDA class one-to-one associated to the present class - from aiida.orm import Group - _aiida_class = Group + _aiida_class = orm.Group # The string name of the AiiDA class _aiida_type = "groups.Group" # The string associated to the AiiDA class in the query builder lexicon diff --git a/aiida/restapi/translator/node.py b/aiida/restapi/translator/node.py index 5a1170e153..144257b951 100644 --- a/aiida/restapi/translator/node.py +++ b/aiida/restapi/translator/node.py @@ -16,6 +16,7 @@ InvalidOperation from aiida.restapi.common.exceptions import RestValidationError from aiida.restapi.translator.base import BaseTranslator +from aiida.manage import get_manager from aiida import orm @@ -28,8 +29,7 @@ class NodeTranslator(BaseTranslator): # A label associated to the present class (coincides with the resource name) __label__ = "nodes" # The AiiDA class one-to-one associated to the present class - from aiida.orm.node import Node - _aiida_class = Node + _aiida_class = orm.Node # The string name of the AiiDA class _aiida_type = "node.Node" # The string associated to the AiiDA class in the query builder lexicon @@ -123,7 +123,7 @@ def __init__(self, Class=None, **kwargs): """ self._subclasses = self._get_subclasses() - self._backend = orm.construct_backend() + self._backend = get_manager().get_backend() def set_query_type(self, query_type, diff --git a/aiida/restapi/translator/user.py b/aiida/restapi/translator/user.py index 7065f1fe31..f1a80aaf80 100644 --- a/aiida/restapi/translator/user.py +++ b/aiida/restapi/translator/user.py @@ -13,6 +13,7 @@ from __future__ import print_function from __future__ import absolute_import from aiida.restapi.translator.base import BaseTranslator +from aiida import orm class UserTranslator(BaseTranslator): @@ -23,8 +24,7 @@ class UserTranslator(BaseTranslator): # A label associated to the present class (coincides with the resource name) __label__ = "users" # The AiiDA class one-to-one associated to the present class - from aiida.orm import User - _aiida_class = User + _aiida_class = orm.User # The string name of the AiiDA class _aiida_type = "User" # The string associated to the AiiDA class in the query builder lexicon diff --git a/aiida/transport/cli.py b/aiida/transport/cli.py index f7a5da4666..25538662ec 100644 --- a/aiida/transport/cli.py +++ b/aiida/transport/cli.py @@ -21,6 +21,7 @@ from aiida.cmdline.utils.decorators import with_dbenv from aiida.cmdline.utils import echo from aiida.common.exceptions import NotExistent +from aiida.manage import get_manager TRANSPORT_PARAMS = [] @@ -38,13 +39,12 @@ def match_comp_transport(ctx, param, computer, transport_type): @with_dbenv() def configure_computer_main(computer, user, **kwargs): """Configure a computer via the CLI.""" - from aiida.common.utils import get_configured_user_email from aiida import orm user = user or orm.User.objects.get_default() echo.echo_info('Configuring computer {} for user {}.'.format(computer.name, user.email)) - if user.email != get_configured_user_email(): + if user.email != get_manager().get_profile().default_user_email: echo.echo_info('Configuring different user, defaults may not be appropriate.') computer.configure(user=user, **kwargs) diff --git a/aiida/utils/fixtures.py b/aiida/utils/fixtures.py index daaac3bb55..1ec3975cfa 100644 --- a/aiida/utils/fixtures.py +++ b/aiida/utils/fixtures.py @@ -46,6 +46,8 @@ from aiida.backends.profile import BACKEND_DJANGO, BACKEND_SQLA from aiida.common import setup as aiida_cfg from aiida.backends import settings as backend_settings +from aiida.manage import get_manager, reset_manager +from aiida.common import exceptions class FixtureError(Exception): @@ -64,8 +66,7 @@ class FixtureManager(object): """ Manage the life cycle of a completely separated and temporary AiiDA environment - * No previously created database of profile is required to run tests using this - environment + * No previously created database of profile is required to run tests using this environment * Tests using this environment will never pollute the user's work environment Example:: @@ -111,6 +112,8 @@ def test_my_stuff(test_data): """ + _test_case = None + def __init__(self): self.db_params = {} self.fs_env = {'repo': 'test_repo', 'config': '.aiida'} @@ -140,8 +143,7 @@ def _backend(self): """ if self.__backend is None: # Lazy load the backend so we don't do it too early (i.e. before load_dbenv()) - from aiida.orm.backends import construct_backend - self.__backend = construct_backend() + self.__backend = get_manager().get_backend() return self.__backend def create_db_cluster(self): @@ -189,15 +191,26 @@ def create_profile(self): setup_profile(profile=profile_name, only_config=False, non_interactive=True, **self.profile) aiida_cfg.set_default_profile(profile_name) self.__is_running_on_test_profile = True + self._create_test_case() + self.init_db() def reset_db(self): """Cleans all data from the database between tests""" - if not self.__is_running_on_test_profile: - raise FixtureError('No test profile has been set up yet, can not reset the db') - if self.profile_info['backend'] == BACKEND_DJANGO: - self.__clean_db_django() - elif self.profile_info['backend'] == BACKEND_SQLA: - self.__clean_db_sqla() + + self._test_case.clean_db() + reset_manager() + self.init_db() + + @staticmethod + def init_db(): + """Initialise the database state""" + # Create the default user + from aiida import orm + try: + orm.User(email=get_manager().get_profile().default_user_email).store() + except exceptions.IntegrityError: + # The default user already exists, no problem + pass @property def profile(self): @@ -362,29 +375,21 @@ def destroy_all(self): if 'profile' in self._backup: backend_settings.AIIDADB_PROFILE = self._backup['profile'] - @staticmethod - def __clean_db_django(): - from aiida.backends.djsite.db.testbase import DjangoTests - DjangoTests().clean_db() - - def __clean_db_sqla(self): - """Clean database for sqlalchemy backend""" - from aiida.backends.sqlalchemy.tests.testbase import SqlAlchemyTests - from aiida.backends.sqlalchemy import get_scoped_session - from aiida import orm - - user = orm.User.objects(self._backend).get(email=self.email) - new_user = orm.User(email=user.email, backend=self._backend) - new_user.first_name = user.first_name - new_user.last_name = user.last_name - new_user.institution = user.institution - - sqla_testcase = SqlAlchemyTests() - sqla_testcase.test_session = get_scoped_session() - sqla_testcase.clean_db() + def _create_test_case(self): + """ + Create the test case for the correct backend which will be used to clean up + """ + if not self.__is_running_on_test_profile: + raise FixtureError('No test profile has been set up yet, cannot create appropriate test case') + if self.profile_info['backend'] == BACKEND_DJANGO: + from aiida.backends.djsite.db.testbase import DjangoTests + self._test_case = DjangoTests() + elif self.profile_info['backend'] == BACKEND_SQLA: + from aiida.backends.sqlalchemy.tests.testbase import SqlAlchemyTests + from aiida.backends.sqlalchemy import get_scoped_session - # that deleted our user, we need to recreate it - new_user.store() + self._test_case = SqlAlchemyTests() + self._test_case.test_session = get_scoped_session() def has_profile_open(self): return self.__is_running_on_test_profile @@ -445,14 +450,12 @@ def test_my_plugin(self): @classmethod def setUpClass(cls): - from aiida.orm.backends import construct_backend - cls.fixture_manager = _GLOBAL_FIXTURE_MANAGER if not cls.fixture_manager.has_profile_open(): raise ValueError( "Fixture mananger has no open profile. Please use aiida.utils.fixtures.TestRunner to run these tests.") - cls.backend = construct_backend() + cls.backend = get_manager().get_backend() def tearDown(self): self.fixture_manager.reset_db() diff --git a/aiida/work/launch.py b/aiida/work/launch.py index b3fc31c3c5..0e3f6dab52 100644 --- a/aiida/work/launch.py +++ b/aiida/work/launch.py @@ -31,7 +31,7 @@ def run(process, *args, **inputs): if isinstance(process, processes.Process): runner = process.runner else: - runner = manager.AiiDAManager.get_runner() + runner = manager.get_manager().get_runner() return runner.run(process, *args, **inputs) @@ -48,7 +48,7 @@ def run_get_node(process, *args, **inputs): if isinstance(process, processes.Process): runner = process.runner else: - runner = manager.AiiDAManager.get_runner() + runner = manager.get_manager().get_runner() return runner.run_get_node(process, *args, **inputs) @@ -65,7 +65,7 @@ def run_get_pid(process, *args, **inputs): if isinstance(process, processes.Process): runner = process.runner else: - runner = manager.AiiDAManager.get_runner() + runner = manager.get_manager().get_runner() return runner.run_get_pid(process, *args, **inputs) @@ -81,8 +81,8 @@ def submit(process, **inputs): """ assert not utils.is_process_function(process), 'Cannot submit a process function' - runner = manager.AiiDAManager.get_runner() - controller = manager.AiiDAManager.get_process_controller() + runner = manager.get_manager().get_runner() + controller = manager.get_manager().get_process_controller() process = processes.instantiate_process(runner, process, **inputs) runner.persister.save_checkpoint(process) diff --git a/aiida/work/process_function.py b/aiida/work/process_function.py index 7ef62b952a..e0859f99ff 100644 --- a/aiida/work/process_function.py +++ b/aiida/work/process_function.py @@ -11,8 +11,7 @@ from six.moves import zip # pylint: disable=unused-import from aiida.common.lang import override - -from . import manager +from aiida.manage import get_manager from . import processes __all__ = ('FunctionProcess', 'process_function', 'calcfunction', 'workfunction') @@ -102,7 +101,7 @@ def run_get_node(*args, **kwargs): :param kwargs: input keyword arguments to construct the FunctionProcess :return: tuple of the outputs of the process and the calculation node """ - runner = manager.AiiDAManager.create_runner(with_persistence=False) + runner = get_manager().create_runner(with_persistence=False) inputs = process_class.create_inputs(*args, **kwargs) # Remove all the known inputs from the kwargs diff --git a/aiida/work/processes.py b/aiida/work/processes.py index f0755bc524..41c07a3511 100644 --- a/aiida/work/processes.py +++ b/aiida/work/processes.py @@ -127,7 +127,7 @@ def get_or_create_db_record(cls): def __init__(self, inputs=None, logger=None, runner=None, parent_pid=None, enable_persistence=True): from aiida.manage import manager - self._runner = runner if runner is not None else manager.AiiDAManager.get_runner() + self._runner = runner if runner is not None else manager.get_manager().get_runner() super(Process, self).__init__( inputs=self.spec().inputs.serialize(inputs), @@ -192,7 +192,7 @@ def load_instance_state(self, saved_state, load_context): if 'runner' in load_context: self._runner = load_context.runner else: - self._runner = manager.AiiDAManager.get_runner() + self._runner = manager.get_manager().get_runner() load_context = load_context.copyextend(loop=self._runner.loop, communicator=self._runner.communicator) super(Process, self).load_instance_state(saved_state, load_context) diff --git a/aiida/work/workfunctions.py b/aiida/work/workfunctions.py index c5823438fb..29a6df142a 100644 --- a/aiida/work/workfunctions.py +++ b/aiida/work/workfunctions.py @@ -17,7 +17,6 @@ from aiida.common.warnings import AiidaDeprecationWarning as DeprecationWarning # pylint: disable=redefined-builtin from .process_function import workfunction -warnings.warn('this module has been deprecated, import directly from `aiida.work` instead', - DeprecationWarning) # pylint: disable=no-member +warnings.warn('this module has been deprecated, import directly from `aiida.work` instead', DeprecationWarning) # pylint: disable=no-member __all__ = ('workfunction',) diff --git a/aiida/workflows/test.py b/aiida/workflows/test.py index c2e913d14c..07aa437b42 100644 --- a/aiida/workflows/test.py +++ b/aiida/workflows/test.py @@ -123,7 +123,7 @@ def generate_calc(workflow): CustomCalc = CalculationFactory('templatereplacer') - computer = orm.Computer.objects(workflow._backend).get(name='localhost') + computer = orm.Computer.objects.get(name='localhost') calc = CustomCalc(computer=computer, withmpi=True) calc.set_option('resources', diff --git a/docs/source/developer_guide/cookbook.rst b/docs/source/developer_guide/cookbook.rst index 71b1c67e89..338772a9c1 100644 --- a/docs/source/developer_guide/cookbook.rst +++ b/docs/source/developer_guide/cookbook.rst @@ -24,11 +24,10 @@ you can use a modification of the following script:: feature is supported by the scheduler plugin). Otherwise, if False show all jobs. """ - from aiida.backends.utils import get_automatic_user + from aiida import orm computer = Computer.get('deneb') - dbauthinfo = computer.get_dbauthinfo(get_automatic_user()) - transport = dbauthinfo.get_transport() + transport = computer.get_transport() scheduler = computer.get_scheduler() scheduler.set_transport(transport) diff --git a/docs/source/nitpick-exceptions b/docs/source/nitpick-exceptions index 122f932e8c..1dc2712005 100644 --- a/docs/source/nitpick-exceptions +++ b/docs/source/nitpick-exceptions @@ -232,3 +232,6 @@ py:class uuid.UUID # typing py:class typing.Generic py:class typing.TypeVar + +# Python 3 complains about this because of orm.Entity.Collection inner class (no idea why) +py:class Collection