continue keras core integration

training_core/results/MD17Dataset/Schnet_EnergyForceModel/Schnet_MD17Dataset_score_aspirin_ccsd.yaml

@@ -0,0 +1,62 @@
+OS: nt_win32
+backend: tensorflow
+cuda_available: 'False'
+data_unit: ''
+date_time: '2023-09-21 20:02:54'
+device_id: '[LogicalDevice(name=''/device:CPU:0'', device_type=''CPU'')]'
+device_memory: '[]'
+device_name: '[{}]'
+energy_scaled_mean_absolute_error:
+- 0.5632363557815552
+epochs:
+- 1000
+execute_folds: null
+force_scaled_mean_absolute_error:
+- 0.9529726505279541
+kgcnn_version: 4.0.0
+learning_rate:
+- 6.721010777255287e-06
+loss:
+- 7.189131259918213
+max_energy_scaled_mean_absolute_error:
+- 50.77715301513672
+max_force_scaled_mean_absolute_error:
+- 23.126117706298828
+max_learning_rate:
+- 0.0010000000474974513
+max_loss:
+- 175.1174774169922
+max_val_energy_scaled_mean_absolute_error:
+- 114.7723388671875
+max_val_force_scaled_mean_absolute_error:
+- 22.369768142700195
+max_val_loss:
+- 174.42893981933594
+min_energy_scaled_mean_absolute_error:
+- 0.5554268956184387
+min_force_scaled_mean_absolute_error:
+- 0.9529726505279541
+min_learning_rate:
+- 6.721010777255287e-06
+min_loss:
+- 7.189131259918213
+min_val_energy_scaled_mean_absolute_error:
+- 0.6133600473403931
+min_val_force_scaled_mean_absolute_error:
+- 1.1866310834884644
+min_val_loss:
+- 9.02548885345459
+model_class: EnergyForceModel
+model_name: Schnet
+model_version: ''
+multi_target_indices: null
+number_histories: 1
+seed: 42
+time_list: null
+trajectory_name: aspirin_ccsd
+val_energy_scaled_mean_absolute_error:
+- 0.6667003631591797
+val_force_scaled_mean_absolute_error:
+- 1.1869312524795532
+val_loss:
+- 9.035613059997559

training_core/results/MD17Dataset/Schnet_EnergyForceModel/Schnet_hyper_aspirin_ccsd.json

@@ -0,0 +1 @@
+{"model": {"class_name": "EnergyForceModel", "module_name": "kgcnn.models_core.force", "config": {"name": "Schnet", "coordinate_input": 1, "inputs": [{"shape": [null], "name": "atomic_number", "dtype": "int32"}, {"shape": [null, 3], "name": "node_coordinates", "dtype": "float32"}, {"shape": [null, 2], "name": "range_indices", "dtype": "int64"}, {"shape": [], "name": "total_nodes", "dtype": "int64"}, {"shape": [], "name": "total_ranges", "dtype": "int64"}], "nested_model_config": true, "output_to_tensor": false, "output_squeeze_states": true, "model_energy": {"class_name": "make_model", "module_name": "kgcnn.literature_core.Schnet", "config": {"name": "Schnet", "inputs": [{"shape": [null], "name": "atomic_number", "dtype": "int32"}, {"shape": [null, 3], "name": "node_coordinates", "dtype": "float32"}, {"shape": [null, 2], "name": "range_indices", "dtype": "int64"}, {"shape": [], "name": "total_nodes", "dtype": "int64"}, {"shape": [], "name": "total_ranges", "dtype": "int64"}], "cast_disjoint_kwargs": {"padded_disjoint": false}, "input_node_embedding": {"input_dim": 95, "output_dim": 64}, "last_mlp": {"use_bias": [true, true, true], "units": [128, 64, 1], "activation": ["kgcnn>shifted_softplus", "kgcnn>shifted_softplus", "linear"]}, "interaction_args": {"units": 128, "use_bias": true, "activation": "kgcnn>shifted_softplus", "cfconv_pool": "scatter_sum"}, "node_pooling_args": {"pooling_method": "scatter_sum"}, "depth": 4, "gauss_args": {"bins": 20, "distance": 4, "offset": 0.0, "sigma": 0.4}, "verbose": 10, "output_embedding": "graph", "use_output_mlp": false, "output_mlp": null}}}}, "training": {"fit": {"batch_size": 32, "epochs": 1000, "validation_freq": 1, "verbose": 2, "callbacks": [{"class_name": "kgcnn>LinearWarmupExponentialLRScheduler", "config": {"lr_start": 0.001, "gamma": 0.995, "epo_warmup": 1, "verbose": 1, "steps_per_epoch": 32}}]}, "compile": {"optimizer": {"class_name": "Adam", "config": {"learning_rate": 0.001}}, "loss_weights": {"energy": 1.0, "force": 49.0}}, "scaler": {"class_name": "EnergyForceExtensiveLabelScaler", "config": {"standardize_scale": true}}}, "data": {}, "info": {"postfix": "", "postfix_file": "_aspirin_ccsd", "kgcnn_version": "3.1.0"}, "dataset": {"class_name": "MD17Dataset", "module_name": "kgcnn.data.datasets.MD17Dataset", "config": {"trajectory_name": "aspirin_ccsd"}, "methods": [{"rename_property_on_graphs": {"old_property_name": "E", "new_property_name": "energy"}}, {"rename_property_on_graphs": {"old_property_name": "F", "new_property_name": "force"}}, {"rename_property_on_graphs": {"old_property_name": "z", "new_property_name": "atomic_number"}}, {"rename_property_on_graphs": {"old_property_name": "R", "new_property_name": "node_coordinates"}}, {"map_list": {"method": "set_range", "max_distance": 5, "max_neighbours": 10000, "node_coordinates": "node_coordinates"}}, {"map_list": {"method": "count_nodes_and_edges", "total_edges": "total_ranges", "count_edges": "range_indices", "count_nodes": "atomic_number", "total_nodes": "total_nodes"}}]}}

training_core/train_force.py

@@ -6,10 +6,11 @@
 from datetime import timedelta
 import kgcnn.training_core.schedule
 import kgcnn.training_core.scheduler
+from kgcnn.data.utils import save_pickle_file
 from kgcnn.data.transform.scaler.serial import deserialize as deserialize_scaler
 from kgcnn.utils_core.devices import check_device
-from kgcnn.training_core.history import save_history_score
-from kgcnn.utils.plots import plot_train_test_loss, plot_predict_true
+from kgcnn.training_core.history import save_history_score, load_history_list, load_time_list
+from kgcnn.utils_core.plots import plot_train_test_loss, plot_predict_true
 from kgcnn.models_core.serial import deserialize as deserialize_model
 from kgcnn.data.serial import deserialize as deserialize_dataset
 from kgcnn.training_core.hyper import HyperParameter
@@ -77,10 +78,9 @@
 # Training on splits. Since training on Force datasets can be expensive, there is a 'execute_splits' parameter to not
 # train on all splits for testing. Can be set via command line or hyperparameter.
 execute_folds = args["fold"] if "execute_folds" not in hyper["training"] else hyper["training"]["execute_folds"]
-splits_done = 0
-history_list, test_indices_list = [], []
+splits_done, current_split = 0, None
 train_indices_all, test_indices_all = [], []
-model, hist, x_test, scaler = None, None, None, None
+model = None
 for current_split, (train_index, test_index) in enumerate(dataset.get_train_test_indices(train="train", test="test")):
 
     # Keep list of train/test indices.
@@ -162,35 +162,42 @@
     stop = time.time()
     print("Print Time for training: ", str(timedelta(seconds=stop - start)))
 
-    # Get loss from history
-    history_list.append(hist)
-    test_indices_list.append([train_index, test_index])
-    splits_done = splits_done + 1
+    # Save history for this fold.
+    save_pickle_file(hist.history, os.path.join(filepath, f"history{postfix_file}_fold_{current_split}.pickle"))
+    save_pickle_file(str(timedelta(seconds=stop - start)),
+                     os.path.join(filepath, f"time{postfix_file}_fold_{current_split}.pickle"))
 
     # Plot prediction
     predicted_y = model.predict(x_test, verbose=0)
     true_y = y_test
 
-    plot_predict_true(np.array(predicted_y[0]), np.array(true_y["energy"]),
+    plot_predict_true(np.array(predicted_y["energy"]), np.array(true_y["energy"]),
                       filepath=filepath, data_unit=label_units,
                       model_name=hyper.model_name, dataset_name=hyper.dataset_class, target_names=label_names,
                       file_name=f"predict_energy{postfix_file}_fold_{splits_done}.png")
 
-    plot_predict_true(np.concatenate([np.array(f) for f in predicted_y[1]], axis=0),
+    plot_predict_true(np.concatenate([np.array(f) for f in predicted_y["force"]], axis=0),
                       np.concatenate([np.array(f) for f in true_y["force"]], axis=0),
                       filepath=filepath, data_unit=label_units,
                       model_name=hyper.model_name, dataset_name=hyper.dataset_class, target_names=label_names,
                       file_name=f"predict_force{postfix_file}_fold_{splits_done}.png")
 
-    # Save keras-model to output-folder.
-    model.save(os.path.join(filepath, f"model{postfix_file}_fold_{splits_done}"))
+    # Save last keras-model to output-folder.
+    model.save(os.path.join(filepath, f"model{postfix_file}_fold_{current_split}.keras"))
+
+    # Save last keras-model to output-folder.
+    model.save_weights(os.path.join(filepath, f"model{postfix_file}_fold_{current_split}.weights.h5"))
+
+    # Get loss from history
+    splits_done = splits_done + 1
 
 # Save original data indices of the splits.
 np.savez(os.path.join(filepath, f"{hyper.model_name}_test_indices_{postfix_file}.npz"), *test_indices_all)
 np.savez(os.path.join(filepath, f"{hyper.model_name}_train_indices_{postfix_file}.npz"), *train_indices_all)
 
 # Plot training- and test-loss vs epochs for all splits.
 data_unit = hyper["data"]["data_unit"] if "data_unit" in hyper["data"] else ""
+history_list = load_history_list(os.path.join(filepath, f"history{postfix_file}_fold_(i).pickle"), current_split + 1)
 plot_train_test_loss(history_list, loss_name=None, val_loss_name=None,
                      model_name=hyper.model_name, data_unit=data_unit, dataset_name=hyper.dataset_class,
                      filepath=filepath, file_name=f"loss{postfix_file}.png")
@@ -199,6 +206,7 @@
 hyper.save(os.path.join(filepath, f"{hyper.model_name}_hyper{postfix_file}.json"))
 
 # Save score of fit result for as text file.
+time_list = load_time_list(os.path.join(filepath, f"time{postfix_file}_fold_(i).pickle"), current_split + 1)
 save_history_score(
     history_list, loss_name=None, val_loss_name=None,
     model_name=hyper.model_name, data_unit=data_unit, dataset_name=hyper.dataset_class,
@@ -207,5 +215,6 @@
         "training"] else None,
     execute_folds=execute_folds, seed=args["seed"],
     filepath=filepath, file_name=f"score{postfix_file}.yaml",
-    trajectory_name=(dataset.trajectory_name if hasattr(dataset, "trajectory_name") else None)
+    trajectory_name=(dataset.trajectory_name if hasattr(dataset, "trajectory_name") else None),
+    time_list=time_list
 )

training_core/train_graph.py

@@ -192,6 +192,7 @@
     model_class=hyper.model_class,
     multi_target_indices=hyper["training"]["multi_target_indices"] if "multi_target_indices" in hyper[
         "training"] else None,
+    execute_folds=execute_folds,
     model_version=model.__kgcnn_model_version__ if hasattr(model, "__kgcnn_model_version__") else "",
     filepath=filepath, file_name=f"score{postfix_file}.yaml", time_list=time_list,
     seed=args["seed"]