Add finish status support to JobCalculations

The JobCalculation already has a sort of finish status, the terminal
states of the DbCalcState table, i.e.:

	* SUBMISSIONFAILED
	* RETRIEVALFAILED
	* PARSINGFAILED
	* FAILED

The idea is to map these calc states to a fixed finish status integer
and have the JobProcess layer set it according to the calculation state,
which remains leading. The FAILED status was set when the whole calc
execution was successful, from submission, through retrieval, up until
successful parsing, but the result of the calculation itself was not
considered to be successful, which will vary per calculation. We extend
this paradigm to allow the parsing function to return any non-zero
positive integer to mean a specific calculation failure mode, which can
then be defined on a per calculation plugin basis and used as error
codes. Support for extending the basic enumeration for JobCalculations
the JobCalculationFinishStatus is not yet implemented, and therefore
until that is done, any non standard integers cannot be mapped onto
a error label but instead will show UNKNOWN.

aiida/daemon/execmanager.py

@@ -801,22 +801,36 @@ def retrieve_all(job, transport, retrieved_temporary_folder, logger_extra=None):
 
 
 def parse_results(job, retrieved_temporary_folder=None, logger_extra=None):
+    """
+    Parse the results for a given JobCalculation (job)
+
+    :returns: integer exit code, where 0 indicates success and non-zero failure
+    """
+    from aiida.orm.calculation.job import JobCalculationFinishStatus
+
     job._set_state(calc_states.PARSING)
 
     Parser = job.get_parserclass()
 
-    # If no parser is set, the calculation is successful
-    successful = True
     if Parser is not None:
+
         parser = Parser(job)
-        successful, new_nodes_tuple = parser.parse_from_calc(retrieved_temporary_folder)
+        exit_code, new_nodes_tuple = parser.parse_from_calc(retrieved_temporary_folder)
+
+        # Some implementations of parse_from_calc may still return a boolean for the exit_code
+        # If we get True we convert to 0, for false we simply use the generic value that
+        # maps to the calculation state FAILED
+        if isinstance(exit_code, bool) and exit_code is True:
+            exit_code = 0
+        elif isinstance(exit_code, bool) and exit_code is False:
+            exit_code = JobCalculationFinishStatus[calc_states.FAILED]
 
         for label, n in new_nodes_tuple:
             n.add_link_from(job, label=label, link_type=LinkType.CREATE)
             n.store()
 
     try:
-        if successful:
+        if exit_code == 0:
             job._set_state(calc_states.FINISHED)
         else:
             job._set_state(calc_states.FAILED)
@@ -825,14 +839,14 @@ def parse_results(job, retrieved_temporary_folder=None, logger_extra=None):
         # in order to avoid useless error messages, I just ignore
         pass
 
-    if not successful:
+    if exit_code is not 0:
         execlogger.error("[parsing of calc {}] "
                          "The parser returned an error, but it should have "
                          "created an output node with some partial results "
                          "and warnings. Check there for more information on "
                          "the problem".format(job.pk), extra=logger_extra)
 
-    return successful
+    return exit_code
 
 
 def _update_job_calc(calc, scheduler, job_info):

aiida/orm/calculation/job/__init__.py

@@ -8,4 +8,4 @@
 # For further information please visit http://www.aiida.net               #
 ###########################################################################
 from aiida.orm.calculation import Calculation
-from aiida.orm.implementation.calculation import JobCalculation, _input_subfolder
+from aiida.orm.implementation.calculation import JobCalculation, _input_subfolder, JobCalculationFinishStatus

aiida/orm/implementation/calculation.py

@@ -16,6 +16,7 @@
 
 from aiida.common.old_pluginloader import from_type_to_pluginclassname
 from aiida.orm.implementation.general.calculation.job import _input_subfolder
+from aiida.orm.implementation.general.calculation.job import JobCalculationFinishStatus
 
 
 if BACKEND == BACKEND_SQLA:

aiida/orm/implementation/django/calculation/job/__init__.py

@@ -42,6 +42,7 @@ def _set_state(self, state):
           from ``aiida.common.datastructures.calc_states``.
         :raise: ModificationNotAllowed if the given state was already set.
         """
+        super(JobCalculation, self)._set_state(state)
 
         from aiida.common.datastructures import sort_states
         from aiida.backends.djsite.db.models import DbCalcState

aiida/orm/implementation/general/calculation/__init__.py

@@ -9,6 +9,7 @@
 ###########################################################################
 
 import collections
+import enum
 import logging
 
 from plumpy import ProcessState
@@ -259,6 +260,9 @@ def _set_finish_status(self, status):
         if status is None:
             status = 0
 
+        if isinstance(status, enum.Enum):
+            status = status.value
+
         if not isinstance(status, int):
             raise ValueError('finish status has to be an integer, got {}'.format(status))
 

aiida/orm/implementation/general/calculation/job/__init__.py

@@ -10,6 +10,7 @@
 import abc
 import copy
 import datetime
+import enum
 
 from aiida.backends.utils import get_automatic_user
 from aiida.common.datastructures import calc_states
@@ -33,12 +34,48 @@
 _input_subfolder = 'raw_input'
 
 
+class JobCalculationFinishStatus(enum.Enum):
+    """
+    This enumeration maps specific calculation states to an integer. This integer can
+    then be used to set the finish status of a JobCalculation node. The values defined
+    here map directly on the failed calculation states, but the idea is that sub classes
+    of AbstractJobCalculation can extend this enum with additional error codes
+    """
+    FINISHED = 0
+    SUBMISSIONFAILED = 100
+    RETRIEVALFAILED = 200
+    PARSINGFAILED = 300
+    FAILED = 400
+    I_AM_A_TEAPOT = 418
+
+
 class AbstractJobCalculation(AbstractCalculation):
     """
     This class provides the definition of an AiiDA calculation that is run
     remotely on a job scheduler.
     """
 
+    @classproperty
+    def finish_status_enum(cls):
+        return JobCalculationFinishStatus
+
+    @property
+    def finish_status_label(self):
+        """
+        Return the label belonging to the finish status of the Calculation
+
+        :returns: the finish status, an integer exit code or None
+        """
+        finish_status = self.finish_status
+
+        try:
+            finish_status_enum = self.finish_status_enum(finish_status)
+            finish_status_label = finish_status_enum.name
+        except ValueError:
+            finish_status_label = 'UNKNOWN'
+
+        return finish_status_label
+
     _cacheable = True
 
     @classproperty

aiida/orm/implementation/sqlalchemy/calculation/job/__init__.py

@@ -51,6 +51,7 @@ def _set_state(self, state):
           from ``aiida.common.datastructures.calc_states``.
         :raise: ModificationNotAllowed if the given state was already set.
         """
+        super(JobCalculation, self)._set_state(state)
 
         if not self.is_stored:
             raise ModificationNotAllowed("Cannot set the calculation state "

aiida/parsers/parser.py

@@ -11,6 +11,9 @@
 This module implements a generic output plugin, that is general enough
 to allow the reading of the outputs of a calculation.
 """
+import logging
+from aiida.common.exceptions import NotExistent
+from aiida.common.log import aiidalogger, get_dblogger_extra
 
 
 class Parser(object):
@@ -22,12 +25,11 @@ class Parser(object):
     Looks for the attached parser_opts or input_settings nodes attached to the calculation.
     Get the child Folderdata, parse it and store the parsed data.
     """
+
     _linkname_outparams = 'output_parameters'
     _retrieved_temporary_folder_key = 'retrieved_temporary_folder'
 
     def __init__(self, calc):
-        from aiida.common import aiidalogger
-
         self._logger = aiidalogger.getChild('parser').getChild( self.__class__.__name__)
         self._calc = calc
 
@@ -37,9 +39,6 @@ def logger(self):
         Return the logger, also with automatic extras of the associated
         extras of the calculation
         """
-        import logging
-        from aiida.common.log import get_dblogger_extra
-
         return logging.LoggerAdapter(logger=self._logger, extra=get_dblogger_extra(self._calc))
 
     @property
@@ -52,24 +51,33 @@ def retrieved_temporary_folder_key(self):
 
     def parse_with_retrieved(self, retrieved):
         """
-        Receives in input a dictionary of retrieved nodes.
-        Implement all the logic in this function of the subclass.
+        This function should be implemented in the Parser subclass and should parse the desired
+        output from the retrieved nodes in the 'retrieved' input dictionary. It should return a
+        tuple of an integer and a list of tuples. The integer serves as an exit code to indicate
+        the successfulness of the parsing, where 0 means success and any non-zero integer indicates
+        a failure. These integer codes can be chosen by the plugin developer. The list of tuples
+        are the parsed nodes that need to be stored as ouput nodes of the calculation. The first key
+        should be the link name and the second key the output node itself.
 
         :param retrieved: dictionary of retrieved nodes
+        :returns: exit code, list of tuples ('link_name', output_node)
+        :rtype: int, [(basestring, Data)]
         """
         raise NotImplementedError
 
     def parse_from_calc(self, retrieved_temporary_folder=None):
         """
-        Parses the datafolder, stores results.
-        Main functionality of the class. If you only have one retrieved node,
-        you do not need to reimplement this. Implement only the
-        parse_with_retrieved
+        Parse the contents of the retrieved folder data node and return a tuple of to be stored
+        output data nodes. If you only have one retrieved node, the default folder data node, this
+        function does not have to be reimplemented in a plugin, only the parse_with_retrieved method.
+
+        :param retrieved_temporary_folder: optional absolute path to directory with temporary retrieved files
+        :returns: exit code, list of tuples ('link_name', output_node)
+        :rtype: int, [(basestring, Data)]
         """
-        # select the folder object
         out_folder = self._calc.get_retrieved_node()
         if out_folder is None:
-            self.logger.error("No retrieved folder found")
+            self.logger.error('No retrieved folder found')
             return False, ()
 
         retrieved = {self._calc._get_linkname_retrieved(): out_folder}
@@ -92,11 +100,9 @@ def get_result_dict(self):
         Return a dictionary with all results (faster than doing multiple queries)
 
         :note: the function returns an empty dictionary if no output params node
-          can be found (either because the parser did not create it, or because
-          the calculation has not been parsed yet).
+            can be found (either because the parser did not create it, or because
+            the calculation has not been parsed yet).
         """
-        from aiida.common.exceptions import NotExistent
-
         try:
             resnode = self.get_result_parameterdata_node()
         except NotExistent:
@@ -112,61 +118,54 @@ def get_result_parameterdata_node(self):
         :raise NotExistent: if the node does not exist
         """
         from aiida.orm.data.parameter import ParameterData
-        from aiida.common.exceptions import NotExistent
 
         out_parameters = self._calc.get_outputs(type=ParameterData, also_labels=True)
-        out_parameterdata = [i[1] for i in out_parameters
-                             if i[0] == self.get_linkname_outparams()]
+        out_parameter_data = [i[1] for i in out_parameters if i[0] == self.get_linkname_outparams()]
 
-        if not out_parameterdata:
-            raise NotExistent("No output .res ParameterData node found")
-        elif len(out_parameterdata) > 1:
+        if not out_parameter_data:
+            raise NotExistent('No output .res ParameterData node found')
+
+        elif len(out_parameter_data) > 1:
             from aiida.common.exceptions import UniquenessError
 
-            raise UniquenessError("Output ParameterData should be found once, "
-                                  "found it instead {} times"
-                                  .format(len(out_parameterdata)))
+            raise UniquenessError(
+                'Output ParameterData should be found once, found it instead {} times'
+                .format(len(out_parameter_data)))
 
-        return out_parameterdata[0]
+        return out_parameter_data[0]
 
     def get_result_keys(self):
         """
         Return an iterator of list of strings of valid result keys,
         that can be then passed to the get_result() method.
 
         :note: the function returns an empty list if no output params node
-          can be found (either because the parser did not create it, or because
-          the calculation has not been parsed yet).
+            can be found (either because the parser did not create it, or because
+            the calculation has not been parsed yet).
 
         :raise UniquenessError: if more than one output node with the name
-          self._get_linkname_outparams() is found.
+            self._get_linkname_outparams() is found.
         """
-        from aiida.common.exceptions import NotExistent
-
         try:
-            resnode = self.get_result_parameterdata_node()
+            node = self.get_result_parameterdata_node()
         except NotExistent:
             return iter([])
 
-        return resnode.keys()
+        return node.keys()
 
     def get_result(self, key_name):
         """
         Access the parameters of the output.
         The following method will should work for a generic parser,
         provided it has to query only one ParameterData object.
         """
-        resnode = self.get_result_parameterdata_node()
+        node = self.get_result_parameterdata_node()
 
         try:
-            value = resnode.get_attr(key_name)
+            value = node.get_attr(key_name)
         except KeyError:
             from aiida.common.exceptions import ContentNotExistent
 
             raise ContentNotExistent("Key {} not found in results".format(key_name))
 
-        # TODO: eventually, here insert further operations
-        # (ex: key_name = energy_float_rydberg could return only the last element of a list,
-        # and convert in the right units)
-
         return value