Add export_parameters to export archive metadata.json (#3386)

The new dictionary contains the parameters used in creating the export file that can be useful at later stages to debug in the case of archives that seem inconsistent. Especially the `graph_traversal_rules` that detail how the originally selected nodes of the `entity_starting_set` were extended through the graph traversal algorithm.
aiidateam · Oct 11, 2019 · 28a7dc9 · 28a7dc9
1 parent fff3cad
commit 28a7dc9
Show file tree

Hide file tree

Showing 3 changed files with 62 additions and 19 deletions.
diff --git a/aiida/tools/importexport/dbexport/__init__.py b/aiida/tools/importexport/dbexport/__init__.py
@@ -32,7 +32,7 @@
 )
 from aiida.tools.importexport.common.utils import export_shard_uuid
 from aiida.tools.importexport.dbexport.utils import (
-    check_licences, fill_in_query, serialize_dict, check_process_nodes_sealed, retrieve_linked_nodes
+    check_licenses, fill_in_query, serialize_dict, check_process_nodes_sealed, retrieve_linked_nodes
 )
 
 from .zip import ZipFolder
@@ -140,11 +140,15 @@ def export_tree(
         exporting.
     :raises `~aiida.common.exceptions.LicensingException`: if any node is licensed under forbidden license.
     """
+    from collections import defaultdict
+
     if not silent:
         print('STARTING EXPORT...')
 
     all_fields_info, unique_identifiers = get_all_fields_info()
 
+    entities_starting_set = defaultdict(set)
+
     # The set that contains the nodes ids of the nodes that should be exported
     given_data_entry_ids = set()
     given_calculation_entry_ids = set()
@@ -162,14 +166,17 @@ def export_tree(
         # Now a load the backend-independent name into entry_entity_name, e.g. Node!
         # entry_entity_name = schema_to_entity_names(entry_class_string)
         if issubclass(entry.__class__, orm.Group):
+            entities_starting_set[GROUP_ENTITY_NAME].add(entry.uuid)
             given_group_entry_ids.add(entry.id)
             given_groups.add(entry)
         elif issubclass(entry.__class__, orm.Node):
+            entities_starting_set[NODE_ENTITY_NAME].add(entry.uuid)
             if issubclass(entry.__class__, orm.Data):
                 given_data_entry_ids.add(entry.pk)
             elif issubclass(entry.__class__, orm.ProcessNode):
                 given_calculation_entry_ids.add(entry.pk)
         elif issubclass(entry.__class__, orm.Computer):
+            entities_starting_set[COMPUTER_ENTITY_NAME].add(entry.uuid)
             given_computer_entry_ids.add(entry.pk)
         else:
             raise exceptions.ArchiveExportError(
@@ -179,19 +186,25 @@ def export_tree(
     # Add all the nodes contained within the specified groups
     for group in given_groups:
         for entry in group.nodes:
+            entities_starting_set[NODE_ENTITY_NAME].add(entry.uuid)
             if issubclass(entry.__class__, orm.Data):
                 given_data_entry_ids.add(entry.pk)
             elif issubclass(entry.__class__, orm.ProcessNode):
                 given_calculation_entry_ids.add(entry.pk)
 
+    for entity, entity_set in entities_starting_set.items():
+        entities_starting_set[entity] = list(entity_set)
+
     # We will iteratively explore the AiiDA graph to find further nodes that
     # should also be exported.
     # At the same time, we will create the links_uuid list of dicts to be exported
 
     if not silent:
         print('RETRIEVING LINKED NODES AND STORING LINKS...')
 
-    to_be_exported, links_uuid = retrieve_linked_nodes(given_calculation_entry_ids, given_data_entry_ids, **kwargs)
+    to_be_exported, links_uuid, graph_traversal_rules = retrieve_linked_nodes(
+        given_calculation_entry_ids, given_data_entry_ids, **kwargs
+    )
 
     ## Universal "entities" attributed to all types of nodes
     # Logs
@@ -271,7 +284,7 @@ def export_tree(
         builder.append(orm.Node, project=['id', 'attributes.source.license'], filters={'id': {'in': to_be_exported}})
         # Skip those nodes where the license is not set (this is the standard behavior with Django)
         node_licenses = list((a, b) for [a, b] in builder.all() if b is not None)
-        check_licences(node_licenses, allowed_licenses, forbidden_licenses)
+        check_licenses(node_licenses, allowed_licenses, forbidden_licenses)
 
     ############################################################
     ##### Start automatic recursive export data generation #####
@@ -395,10 +408,10 @@ def export_tree(
         'node_extras': node_extras,
         'export_data': export_data,
         'links_uuid': links_uuid,
-        'groups_uuid': groups_uuid,
+        'groups_uuid': groups_uuid
     }
 
-    # N.B. We're really calling zipfolder.open
+    # N.B. We're really calling zipfolder.open (if exporting a zipfile)
     with folder.open('data.json', mode='w') as fhandle:
         # fhandle.write(json.dumps(data, cls=UUIDEncoder))
         fhandle.write(json.dumps(data))
@@ -411,6 +424,12 @@ def export_tree(
         'export_version': EXPORT_VERSION,
         'all_fields_info': all_fields_info,
         'unique_identifiers': unique_identifiers,
+        'export_parameters': {
+            'graph_traversal_rules': graph_traversal_rules,
+            'entities_starting_set': entities_starting_set,
+            'include_comments': include_comments,
+            'include_logs': include_logs
+        }
     }
 
     with folder.open('metadata.json', 'w') as fhandle:

diff --git a/aiida/tools/importexport/dbexport/utils.py b/aiida/tools/importexport/dbexport/utils.py
@@ -102,7 +102,7 @@ def fill_in_query(partial_query, originating_entity_str, current_entity_str, tag
         fill_in_query(partial_query, current_entity_str, ref_model_name, new_tag_suffixes)
 
 
-def check_licences(node_licenses, allowed_licenses, forbidden_licenses):
+def check_licenses(node_licenses, allowed_licenses, forbidden_licenses):
     """Check licenses"""
     from aiida.common.exceptions import LicensingException
     from inspect import isfunction
@@ -384,7 +384,7 @@ def retrieve_linked_nodes(process_nodes, data_nodes, **kwargs):  # pylint: disab
     :param call_calc_backward: Follow CALL_CALC links in the backward direction (recursively).
     :param call_work_backward: Follow CALL_WORK links in the backward direction (recursively).
 
-    :return: Set of retrieved Nodes and list of links information.
+    :return: Set of retrieved Nodes, list of links information, and updated dict of LINK_FLAGS.
 
     :raises `~aiida.tools.importexport.common.exceptions.ExportValidationError`: if wrong or too many kwargs are given.
     """
@@ -573,4 +573,4 @@ def retrieve_linked_nodes(process_nodes, data_nodes, **kwargs):  # pylint: disab
                 process_nodes.update(found_nodes - retrieved_nodes)
                 links_uuid_dict.update(links_uuids)
 
-    return retrieved_nodes, list(links_uuid_dict.values())
+    return retrieved_nodes, list(links_uuid_dict.values()), traversal_rules
diff --git a/docs/source/import_export/main.rst b/docs/source/import_export/main.rst
@@ -60,8 +60,29 @@ Let's have a look at the contents of ``metadata.json``:
 .. code-block:: json
 
     {
-      "export_version": "0.6",
+      "export_version": "0.7",
       "aiida_version": "1.0.0",
+      "export_parameters": {
+        "graph_traversal_rules": {
+          "input_calc_forward": false,
+          "input_calc_backward": true,
+          "create_forward": true,
+          "create_backward": true,
+          "return_forward": true,
+          "return_backward": false,
+          "input_work_forward": false,
+          "input_work_backward": true,
+          "call_calc_forward": true,
+          "call_calc_backward": false,
+          "call_work_forward": true,
+          "call_work_backward": false
+        },
+        "entities_starting_set": {
+          "Node": ["1024e35e-166b-4104-95f6-c1706df4ce15"]
+        },
+        "include_comments": true,
+        "include_logs": false
+      },
       "unique_identifiers": {
         "Computer": "uuid",
         "Group": "uuid",
@@ -186,10 +207,10 @@ A sample of the ``data.json`` file follows:
     {
       "links_uuid": [
         {
-          "output": "c208c9da-23b4-4c32-8f99-f9141ab28363",
-          "label": "parent_calc_folder",
-          "input": "eaaa114d-3d5b-42eb-a269-cf0e7a3a935d",
-          "type": "inputlink"
+          "output": "1024e35e-166b-4104-95f6-c1706df4ce15",
+          "label": "parameters",
+          "input": "628ba258-ccc1-47bf-bab7-8aee64b563ea",
+          "type": "input_calc"
         }
       ],
       "export_data": {
@@ -248,7 +269,7 @@ A sample of the ``data.json`` file follows:
             "ctime": "2016-08-21T11:56:05.501162",
             "mtime": "2016-08-21T11:56:05.501697",
             "content": "vc-relax calculation with cold smearing",
-            "dbnode": 5921143,
+            "dbnode": 20063,
             "user": 2
           }
         }
@@ -295,13 +316,16 @@ A sample of the ``data.json`` file follows:
             "default_mpiprocs_per_machine": 8
           },
           "remote_workdir": "/scratch/aiida/aiida_run/10/24/e35e-166b-4104-95f6-c1706df4ce15",
-          "state": "FINISHED",
+          "process_state": "finished",
           "max_wallclock_seconds": 900,
           "retrieve_singlefile_list": [],
-          "scheduler_lastchecktime": "2015-10-02T20:30:36.481951",
-          "job_id": "13489"
-        },
-        "6480111": {}
+          "scheduler_lastchecktime": "2015-10-02T20:30:36.481951+00:00",
+          "job_id": "13489",
+          "exit_status": 0,
+          "process_status": null,
+          "process_label": "vc-relax",
+          "sealed": true
+        }
       },
       "node_extras": {
         "5921143": {},