diff --git a/training_core/results/MD17Dataset/Schnet_EnergyForceModel/Schnet_MD17Dataset_score_aspirin_ccsd.yaml b/training_core/results/MD17Dataset/Schnet_EnergyForceModel/Schnet_MD17Dataset_score_aspirin_ccsd.yaml new file mode 100644 index 00000000..fa7711b1 --- /dev/null +++ b/training_core/results/MD17Dataset/Schnet_EnergyForceModel/Schnet_MD17Dataset_score_aspirin_ccsd.yaml @@ -0,0 +1,62 @@ +OS: nt_win32 +backend: tensorflow +cuda_available: 'False' +data_unit: '' +date_time: '2023-09-21 20:02:54' +device_id: '[LogicalDevice(name=''/device:CPU:0'', device_type=''CPU'')]' +device_memory: '[]' +device_name: '[{}]' +energy_scaled_mean_absolute_error: +- 0.5632363557815552 +epochs: +- 1000 +execute_folds: null +force_scaled_mean_absolute_error: +- 0.9529726505279541 +kgcnn_version: 4.0.0 +learning_rate: +- 6.721010777255287e-06 +loss: +- 7.189131259918213 +max_energy_scaled_mean_absolute_error: +- 50.77715301513672 +max_force_scaled_mean_absolute_error: +- 23.126117706298828 +max_learning_rate: +- 0.0010000000474974513 +max_loss: +- 175.1174774169922 +max_val_energy_scaled_mean_absolute_error: +- 114.7723388671875 +max_val_force_scaled_mean_absolute_error: +- 22.369768142700195 +max_val_loss: +- 174.42893981933594 +min_energy_scaled_mean_absolute_error: +- 0.5554268956184387 +min_force_scaled_mean_absolute_error: +- 0.9529726505279541 +min_learning_rate: +- 6.721010777255287e-06 +min_loss: +- 7.189131259918213 +min_val_energy_scaled_mean_absolute_error: +- 0.6133600473403931 +min_val_force_scaled_mean_absolute_error: +- 1.1866310834884644 +min_val_loss: +- 9.02548885345459 +model_class: EnergyForceModel +model_name: Schnet +model_version: '' +multi_target_indices: null +number_histories: 1 +seed: 42 +time_list: null +trajectory_name: aspirin_ccsd +val_energy_scaled_mean_absolute_error: +- 0.6667003631591797 +val_force_scaled_mean_absolute_error: +- 1.1869312524795532 +val_loss: +- 9.035613059997559 diff --git a/training_core/results/MD17Dataset/Schnet_EnergyForceModel/Schnet_hyper_aspirin_ccsd.json b/training_core/results/MD17Dataset/Schnet_EnergyForceModel/Schnet_hyper_aspirin_ccsd.json new file mode 100644 index 00000000..ad491559 --- /dev/null +++ b/training_core/results/MD17Dataset/Schnet_EnergyForceModel/Schnet_hyper_aspirin_ccsd.json @@ -0,0 +1 @@ +{"model": {"class_name": "EnergyForceModel", "module_name": "kgcnn.models_core.force", "config": {"name": "Schnet", "coordinate_input": 1, "inputs": [{"shape": [null], "name": "atomic_number", "dtype": "int32"}, {"shape": [null, 3], "name": "node_coordinates", "dtype": "float32"}, {"shape": [null, 2], "name": "range_indices", "dtype": "int64"}, {"shape": [], "name": "total_nodes", "dtype": "int64"}, {"shape": [], "name": "total_ranges", "dtype": "int64"}], "nested_model_config": true, "output_to_tensor": false, "output_squeeze_states": true, "model_energy": {"class_name": "make_model", "module_name": "kgcnn.literature_core.Schnet", "config": {"name": "Schnet", "inputs": [{"shape": [null], "name": "atomic_number", "dtype": "int32"}, {"shape": [null, 3], "name": "node_coordinates", "dtype": "float32"}, {"shape": [null, 2], "name": "range_indices", "dtype": "int64"}, {"shape": [], "name": "total_nodes", "dtype": "int64"}, {"shape": [], "name": "total_ranges", "dtype": "int64"}], "cast_disjoint_kwargs": {"padded_disjoint": false}, "input_node_embedding": {"input_dim": 95, "output_dim": 64}, "last_mlp": {"use_bias": [true, true, true], "units": [128, 64, 1], "activation": ["kgcnn>shifted_softplus", "kgcnn>shifted_softplus", "linear"]}, "interaction_args": {"units": 128, "use_bias": true, "activation": "kgcnn>shifted_softplus", "cfconv_pool": "scatter_sum"}, "node_pooling_args": {"pooling_method": "scatter_sum"}, "depth": 4, "gauss_args": {"bins": 20, "distance": 4, "offset": 0.0, "sigma": 0.4}, "verbose": 10, "output_embedding": "graph", "use_output_mlp": false, "output_mlp": null}}}}, "training": {"fit": {"batch_size": 32, "epochs": 1000, "validation_freq": 1, "verbose": 2, "callbacks": [{"class_name": "kgcnn>LinearWarmupExponentialLRScheduler", "config": {"lr_start": 0.001, "gamma": 0.995, "epo_warmup": 1, "verbose": 1, "steps_per_epoch": 32}}]}, "compile": {"optimizer": {"class_name": "Adam", "config": {"learning_rate": 0.001}}, "loss_weights": {"energy": 1.0, "force": 49.0}}, "scaler": {"class_name": "EnergyForceExtensiveLabelScaler", "config": {"standardize_scale": true}}}, "data": {}, "info": {"postfix": "", "postfix_file": "_aspirin_ccsd", "kgcnn_version": "3.1.0"}, "dataset": {"class_name": "MD17Dataset", "module_name": "kgcnn.data.datasets.MD17Dataset", "config": {"trajectory_name": "aspirin_ccsd"}, "methods": [{"rename_property_on_graphs": {"old_property_name": "E", "new_property_name": "energy"}}, {"rename_property_on_graphs": {"old_property_name": "F", "new_property_name": "force"}}, {"rename_property_on_graphs": {"old_property_name": "z", "new_property_name": "atomic_number"}}, {"rename_property_on_graphs": {"old_property_name": "R", "new_property_name": "node_coordinates"}}, {"map_list": {"method": "set_range", "max_distance": 5, "max_neighbours": 10000, "node_coordinates": "node_coordinates"}}, {"map_list": {"method": "count_nodes_and_edges", "total_edges": "total_ranges", "count_edges": "range_indices", "count_nodes": "atomic_number", "total_nodes": "total_nodes"}}]}} \ No newline at end of file diff --git a/training_core/train_force.py b/training_core/train_force.py index 1e4ecfe4..e861990b 100644 --- a/training_core/train_force.py +++ b/training_core/train_force.py @@ -6,10 +6,11 @@ from datetime import timedelta import kgcnn.training_core.schedule import kgcnn.training_core.scheduler +from kgcnn.data.utils import save_pickle_file from kgcnn.data.transform.scaler.serial import deserialize as deserialize_scaler from kgcnn.utils_core.devices import check_device -from kgcnn.training_core.history import save_history_score -from kgcnn.utils.plots import plot_train_test_loss, plot_predict_true +from kgcnn.training_core.history import save_history_score, load_history_list, load_time_list +from kgcnn.utils_core.plots import plot_train_test_loss, plot_predict_true from kgcnn.models_core.serial import deserialize as deserialize_model from kgcnn.data.serial import deserialize as deserialize_dataset from kgcnn.training_core.hyper import HyperParameter @@ -77,10 +78,9 @@ # Training on splits. Since training on Force datasets can be expensive, there is a 'execute_splits' parameter to not # train on all splits for testing. Can be set via command line or hyperparameter. execute_folds = args["fold"] if "execute_folds" not in hyper["training"] else hyper["training"]["execute_folds"] -splits_done = 0 -history_list, test_indices_list = [], [] +splits_done, current_split = 0, None train_indices_all, test_indices_all = [], [] -model, hist, x_test, scaler = None, None, None, None +model = None for current_split, (train_index, test_index) in enumerate(dataset.get_train_test_indices(train="train", test="test")): # Keep list of train/test indices. @@ -162,28 +162,34 @@ stop = time.time() print("Print Time for training: ", str(timedelta(seconds=stop - start))) - # Get loss from history - history_list.append(hist) - test_indices_list.append([train_index, test_index]) - splits_done = splits_done + 1 + # Save history for this fold. + save_pickle_file(hist.history, os.path.join(filepath, f"history{postfix_file}_fold_{current_split}.pickle")) + save_pickle_file(str(timedelta(seconds=stop - start)), + os.path.join(filepath, f"time{postfix_file}_fold_{current_split}.pickle")) # Plot prediction predicted_y = model.predict(x_test, verbose=0) true_y = y_test - plot_predict_true(np.array(predicted_y[0]), np.array(true_y["energy"]), + plot_predict_true(np.array(predicted_y["energy"]), np.array(true_y["energy"]), filepath=filepath, data_unit=label_units, model_name=hyper.model_name, dataset_name=hyper.dataset_class, target_names=label_names, file_name=f"predict_energy{postfix_file}_fold_{splits_done}.png") - plot_predict_true(np.concatenate([np.array(f) for f in predicted_y[1]], axis=0), + plot_predict_true(np.concatenate([np.array(f) for f in predicted_y["force"]], axis=0), np.concatenate([np.array(f) for f in true_y["force"]], axis=0), filepath=filepath, data_unit=label_units, model_name=hyper.model_name, dataset_name=hyper.dataset_class, target_names=label_names, file_name=f"predict_force{postfix_file}_fold_{splits_done}.png") - # Save keras-model to output-folder. - model.save(os.path.join(filepath, f"model{postfix_file}_fold_{splits_done}")) + # Save last keras-model to output-folder. + model.save(os.path.join(filepath, f"model{postfix_file}_fold_{current_split}.keras")) + + # Save last keras-model to output-folder. + model.save_weights(os.path.join(filepath, f"model{postfix_file}_fold_{current_split}.weights.h5")) + + # Get loss from history + splits_done = splits_done + 1 # Save original data indices of the splits. np.savez(os.path.join(filepath, f"{hyper.model_name}_test_indices_{postfix_file}.npz"), *test_indices_all) @@ -191,6 +197,7 @@ # Plot training- and test-loss vs epochs for all splits. data_unit = hyper["data"]["data_unit"] if "data_unit" in hyper["data"] else "" +history_list = load_history_list(os.path.join(filepath, f"history{postfix_file}_fold_(i).pickle"), current_split + 1) plot_train_test_loss(history_list, loss_name=None, val_loss_name=None, model_name=hyper.model_name, data_unit=data_unit, dataset_name=hyper.dataset_class, filepath=filepath, file_name=f"loss{postfix_file}.png") @@ -199,6 +206,7 @@ hyper.save(os.path.join(filepath, f"{hyper.model_name}_hyper{postfix_file}.json")) # Save score of fit result for as text file. +time_list = load_time_list(os.path.join(filepath, f"time{postfix_file}_fold_(i).pickle"), current_split + 1) save_history_score( history_list, loss_name=None, val_loss_name=None, model_name=hyper.model_name, data_unit=data_unit, dataset_name=hyper.dataset_class, @@ -207,5 +215,6 @@ "training"] else None, execute_folds=execute_folds, seed=args["seed"], filepath=filepath, file_name=f"score{postfix_file}.yaml", - trajectory_name=(dataset.trajectory_name if hasattr(dataset, "trajectory_name") else None) + trajectory_name=(dataset.trajectory_name if hasattr(dataset, "trajectory_name") else None), + time_list=time_list ) diff --git a/training_core/train_graph.py b/training_core/train_graph.py index 23dff66b..e0eb03b5 100644 --- a/training_core/train_graph.py +++ b/training_core/train_graph.py @@ -192,6 +192,7 @@ model_class=hyper.model_class, multi_target_indices=hyper["training"]["multi_target_indices"] if "multi_target_indices" in hyper[ "training"] else None, + execute_folds=execute_folds, model_version=model.__kgcnn_model_version__ if hasattr(model, "__kgcnn_model_version__") else "", filepath=filepath, file_name=f"score{postfix_file}.yaml", time_list=time_list, seed=args["seed"]