diff --git a/aiida/tools/importexport/dbexport/__init__.py b/aiida/tools/importexport/dbexport/__init__.py index 9eab9a83c0..d27b1be318 100644 --- a/aiida/tools/importexport/dbexport/__init__.py +++ b/aiida/tools/importexport/dbexport/__init__.py @@ -32,7 +32,7 @@ ) from aiida.tools.importexport.common.utils import export_shard_uuid from aiida.tools.importexport.dbexport.utils import ( - check_licences, fill_in_query, serialize_dict, check_process_nodes_sealed, retrieve_linked_nodes + check_licenses, fill_in_query, serialize_dict, check_process_nodes_sealed, retrieve_linked_nodes ) from .zip import ZipFolder @@ -140,11 +140,15 @@ def export_tree( exporting. :raises `~aiida.common.exceptions.LicensingException`: if any node is licensed under forbidden license. """ + from collections import defaultdict + if not silent: print('STARTING EXPORT...') all_fields_info, unique_identifiers = get_all_fields_info() + entities_starting_set = defaultdict(set) + # The set that contains the nodes ids of the nodes that should be exported given_data_entry_ids = set() given_calculation_entry_ids = set() @@ -162,14 +166,17 @@ def export_tree( # Now a load the backend-independent name into entry_entity_name, e.g. Node! # entry_entity_name = schema_to_entity_names(entry_class_string) if issubclass(entry.__class__, orm.Group): + entities_starting_set[GROUP_ENTITY_NAME].add(entry.uuid) given_group_entry_ids.add(entry.id) given_groups.add(entry) elif issubclass(entry.__class__, orm.Node): + entities_starting_set[NODE_ENTITY_NAME].add(entry.uuid) if issubclass(entry.__class__, orm.Data): given_data_entry_ids.add(entry.pk) elif issubclass(entry.__class__, orm.ProcessNode): given_calculation_entry_ids.add(entry.pk) elif issubclass(entry.__class__, orm.Computer): + entities_starting_set[COMPUTER_ENTITY_NAME].add(entry.uuid) given_computer_entry_ids.add(entry.pk) else: raise exceptions.ArchiveExportError( @@ -179,11 +186,15 @@ def export_tree( # Add all the nodes contained within the specified groups for group in given_groups: for entry in group.nodes: + entities_starting_set[NODE_ENTITY_NAME].add(entry.uuid) if issubclass(entry.__class__, orm.Data): given_data_entry_ids.add(entry.pk) elif issubclass(entry.__class__, orm.ProcessNode): given_calculation_entry_ids.add(entry.pk) + for entity, entity_set in entities_starting_set.items(): + entities_starting_set[entity] = list(entity_set) + # We will iteratively explore the AiiDA graph to find further nodes that # should also be exported. # At the same time, we will create the links_uuid list of dicts to be exported @@ -191,7 +202,9 @@ def export_tree( if not silent: print('RETRIEVING LINKED NODES AND STORING LINKS...') - to_be_exported, links_uuid = retrieve_linked_nodes(given_calculation_entry_ids, given_data_entry_ids, **kwargs) + to_be_exported, links_uuid, graph_traversal_rules = retrieve_linked_nodes( + given_calculation_entry_ids, given_data_entry_ids, **kwargs + ) ## Universal "entities" attributed to all types of nodes # Logs @@ -271,7 +284,7 @@ def export_tree( builder.append(orm.Node, project=['id', 'attributes.source.license'], filters={'id': {'in': to_be_exported}}) # Skip those nodes where the license is not set (this is the standard behavior with Django) node_licenses = list((a, b) for [a, b] in builder.all() if b is not None) - check_licences(node_licenses, allowed_licenses, forbidden_licenses) + check_licenses(node_licenses, allowed_licenses, forbidden_licenses) ############################################################ ##### Start automatic recursive export data generation ##### @@ -395,10 +408,10 @@ def export_tree( 'node_extras': node_extras, 'export_data': export_data, 'links_uuid': links_uuid, - 'groups_uuid': groups_uuid, + 'groups_uuid': groups_uuid } - # N.B. We're really calling zipfolder.open + # N.B. We're really calling zipfolder.open (if exporting a zipfile) with folder.open('data.json', mode='w') as fhandle: # fhandle.write(json.dumps(data, cls=UUIDEncoder)) fhandle.write(json.dumps(data)) @@ -411,6 +424,12 @@ def export_tree( 'export_version': EXPORT_VERSION, 'all_fields_info': all_fields_info, 'unique_identifiers': unique_identifiers, + 'export_parameters': { + 'graph_traversal_rules': graph_traversal_rules, + 'entities_starting_set': entities_starting_set, + 'include_comments': include_comments, + 'include_logs': include_logs + } } with folder.open('metadata.json', 'w') as fhandle: diff --git a/aiida/tools/importexport/dbexport/utils.py b/aiida/tools/importexport/dbexport/utils.py index 28a2e20543..52e6e033f5 100644 --- a/aiida/tools/importexport/dbexport/utils.py +++ b/aiida/tools/importexport/dbexport/utils.py @@ -102,7 +102,7 @@ def fill_in_query(partial_query, originating_entity_str, current_entity_str, tag fill_in_query(partial_query, current_entity_str, ref_model_name, new_tag_suffixes) -def check_licences(node_licenses, allowed_licenses, forbidden_licenses): +def check_licenses(node_licenses, allowed_licenses, forbidden_licenses): """Check licenses""" from aiida.common.exceptions import LicensingException from inspect import isfunction @@ -384,7 +384,7 @@ def retrieve_linked_nodes(process_nodes, data_nodes, **kwargs): # pylint: disab :param call_calc_backward: Follow CALL_CALC links in the backward direction (recursively). :param call_work_backward: Follow CALL_WORK links in the backward direction (recursively). - :return: Set of retrieved Nodes and list of links information. + :return: Set of retrieved Nodes, list of links information, and updated dict of LINK_FLAGS. :raises `~aiida.tools.importexport.common.exceptions.ExportValidationError`: if wrong or too many kwargs are given. """ @@ -573,4 +573,4 @@ def retrieve_linked_nodes(process_nodes, data_nodes, **kwargs): # pylint: disab process_nodes.update(found_nodes - retrieved_nodes) links_uuid_dict.update(links_uuids) - return retrieved_nodes, list(links_uuid_dict.values()) + return retrieved_nodes, list(links_uuid_dict.values()), traversal_rules diff --git a/docs/source/import_export/main.rst b/docs/source/import_export/main.rst index 311ab1c61d..f202db6294 100644 --- a/docs/source/import_export/main.rst +++ b/docs/source/import_export/main.rst @@ -60,8 +60,29 @@ Let's have a look at the contents of ``metadata.json``: .. code-block:: json { - "export_version": "0.6", + "export_version": "0.7", "aiida_version": "1.0.0", + "export_parameters": { + "graph_traversal_rules": { + "input_calc_forward": false, + "input_calc_backward": true, + "create_forward": true, + "create_backward": true, + "return_forward": true, + "return_backward": false, + "input_work_forward": false, + "input_work_backward": true, + "call_calc_forward": true, + "call_calc_backward": false, + "call_work_forward": true, + "call_work_backward": false + }, + "entities_starting_set": { + "Node": ["1024e35e-166b-4104-95f6-c1706df4ce15"] + }, + "include_comments": true, + "include_logs": false + }, "unique_identifiers": { "Computer": "uuid", "Group": "uuid", @@ -186,10 +207,10 @@ A sample of the ``data.json`` file follows: { "links_uuid": [ { - "output": "c208c9da-23b4-4c32-8f99-f9141ab28363", - "label": "parent_calc_folder", - "input": "eaaa114d-3d5b-42eb-a269-cf0e7a3a935d", - "type": "inputlink" + "output": "1024e35e-166b-4104-95f6-c1706df4ce15", + "label": "parameters", + "input": "628ba258-ccc1-47bf-bab7-8aee64b563ea", + "type": "input_calc" } ], "export_data": { @@ -248,7 +269,7 @@ A sample of the ``data.json`` file follows: "ctime": "2016-08-21T11:56:05.501162", "mtime": "2016-08-21T11:56:05.501697", "content": "vc-relax calculation with cold smearing", - "dbnode": 5921143, + "dbnode": 20063, "user": 2 } } @@ -295,13 +316,16 @@ A sample of the ``data.json`` file follows: "default_mpiprocs_per_machine": 8 }, "remote_workdir": "/scratch/aiida/aiida_run/10/24/e35e-166b-4104-95f6-c1706df4ce15", - "state": "FINISHED", + "process_state": "finished", "max_wallclock_seconds": 900, "retrieve_singlefile_list": [], - "scheduler_lastchecktime": "2015-10-02T20:30:36.481951", - "job_id": "13489" - }, - "6480111": {} + "scheduler_lastchecktime": "2015-10-02T20:30:36.481951+00:00", + "job_id": "13489", + "exit_status": 0, + "process_status": null, + "process_label": "vc-relax", + "sealed": true + } }, "node_extras": { "5921143": {},