adaoted hyperparamter for change in activation serialization.

aimat-lab · Feb 18, 2024 · 61617b0 · 61617b0
1 parent c647684
commit 61617b0
Show file tree

Hide file tree

Showing 26 changed files with 413 additions and 178 deletions.
diff --git a/kgcnn/literature/MEGAN/_make.py b/kgcnn/literature/MEGAN/_make.py
@@ -4,7 +4,7 @@
 from keras.backend import backend as backend_to_use
 from kgcnn.models.casting import (template_cast_output, template_cast_list_input,
                                   template_cast_list_input_docs, template_cast_output_docs)
-from kgcnn.ops.activ import *
+import kgcnn.ops.activ
 
 # Keep track of model version from commit date in literature.
 # To be updated if model is changed in a significant way.

diff --git a/kgcnn/literature/MEGAN/_model.py b/kgcnn/literature/MEGAN/_model.py
@@ -9,7 +9,7 @@
 from kgcnn.layers.aggr import AggregateLocalEdges
 from kgcnn.layers.modules import Embedding
 from ._layers import ExplanationSparsityRegularization
-from kgcnn.ops.activ import leaky_relu
+import kgcnn.ops.activ
 
 
 def shifted_sigmoid(x: KerasTensor, multiplier: float = 1.0, shift: float = 10) -> float:
@@ -30,7 +30,7 @@ class MEGAN(ks.models.Model):
     def __init__(self,
                  # convolutional network related arguments
                  units: List[int],
-                 activation: str = "kgcnn>leaky_relu2",
+                 activation: Union[str, dict] = "kgcnn>leaky_relu2",
                  use_bias: bool = True,
                  dropout_rate: float = 0.0,
                  use_edge_features: bool = True,

diff --git a/training/hyper/hyper_clintox.py b/training/hyper/hyper_clintox.py
@@ -143,7 +143,7 @@
                 "input_node_embedding": {"input_dim": 95, "output_dim": 64},
                 "input_edge_embedding": {"input_dim": 8, "output_dim": 64},
                 "attention_args": {"units": 64, "use_bias": True, "use_edge_features": True,
-                                    "activation": "kgcnn>leaky_relu",
+                                   "activation": {"class_name": "function", "config": "kgcnn>leaky_relu2"},
                                    "use_final_activation": False, "has_self_loops": True},
                 "pooling_nodes_args": {"pooling_method": "scatter_sum"},
                 "depth": 1, "attention_heads_num": 10,
@@ -209,7 +209,7 @@
                 "input_node_embedding": {"input_dim": 95, "output_dim": 64},
                 "input_edge_embedding": {"input_dim": 8, "output_dim": 64},
                 "attention_args": {"units": 64, "use_bias": True, "use_edge_features": True,
-                                   "activation": "kgcnn>leaky_relu",
+                                   "activation": {"class_name": "function", "config": "kgcnn>leaky_relu2"},
                                    "use_final_activation": False, "has_self_loops": True},
                 "pooling_nodes_args": {"pooling_method": "scatter_sum"},
                 "depth": 4, "attention_heads_num": 10,
@@ -275,11 +275,16 @@
                 "input_node_embedding": {"input_dim": 95, "output_dim": 64},
                 "output_embedding": "graph",
                 'output_mlp': {"use_bias": [True, True], "units": [64, 1],
-                               "activation": ['kgcnn>shifted_softplus', "sigmoid"]},
+                               "activation": [
+                                   {"class_name": "function", "config": "kgcnn>shifted_softplus"},
+                                   "sigmoid"
+                               ]},
                 'last_mlp': {"use_bias": [True, True], "units": [128, 64],
-                             "activation": ['kgcnn>shifted_softplus', 'kgcnn>shifted_softplus']},
+                             "activation": [{"class_name": "function", "config": "kgcnn>shifted_softplus"},
+                                            {"class_name": "function", "config": "kgcnn>shifted_softplus"}]},
                 "interaction_args": {
-                    "units": 128, "use_bias": True, "activation": "kgcnn>shifted_softplus",
+                    "units": 128, "use_bias": True,
+                    "activation": {"class_name": "function", "config": "kgcnn>shifted_softplus"},
                     "cfconv_pool": "scatter_sum"
                 },
                 "node_pooling_args": {"pooling_method": "scatter_sum"},

diff --git a/training/hyper/hyper_cora.py b/training/hyper/hyper_cora.py
@@ -83,7 +83,7 @@
                 "input_node_embedding": {"input_dim": 95, "output_dim": 64},
                 "input_edge_embedding": {"input_dim": 25, "output_dim": 64},
                 "attention_args": {"units": 140, "use_bias": True, "use_edge_features": True,
-                                    "activation": "kgcnn>leaky_relu",
+                                   "activation": {"class_name": "function", "config": "kgcnn>leaky_relu2"},
                                    "use_final_activation": False, "has_self_loops": True},
                 "pooling_nodes_args": {"pooling_method": "scatter_mean"},
                 "depth": 1, "attention_heads_num": 10,

diff --git a/training/hyper/hyper_esol.py b/training/hyper/hyper_esol.py
@@ -160,7 +160,7 @@
                 "input_node_embedding": {"input_dim": 95, "output_dim": 64},
                 "input_edge_embedding": {"input_dim": 8, "output_dim": 64},
                 "attention_args": {"units": 64, "use_bias": True, "use_edge_features": True,
-                                   "activation": "kgcnn>leaky_relu",
+                                   "activation": {"class_name": "function", "config": 'kgcnn>leaky_relu2'},
                                    "use_final_activation": False, "has_self_loops": True},
                 "pooling_nodes_args": {"pooling_method": "scatter_sum"},
                 "depth": 1, "attention_heads_num": 10,
@@ -230,7 +230,7 @@
                 "input_node_embedding": {"input_dim": 95, "output_dim": 64},
                 "input_edge_embedding": {"input_dim": 8, "output_dim": 64},
                 "attention_args": {"units": 64, "use_bias": True, "use_edge_features": True,
-                                   "activation": "kgcnn>leaky_relu",
+                                   "activation": {"class_name": "function", "config": 'kgcnn>leaky_relu2'},
                                    "use_final_activation": False, "has_self_loops": True},
                 "pooling_nodes_args": {"pooling_method": "scatter_sum"},
                 "depth": 4, "attention_heads_num": 10,
@@ -555,7 +555,7 @@
                 "verbose": 10,
                 "output_embedding": "graph",
                 "output_mlp": {"use_bias": [True, True], "units": [200, 1],
-                               "activation": [{"class_name": "function", "config": "kgcnn>leaky_relu"}, "linear"]}
+                               "activation": [{"class_name": "function", "config": "kgcnn>leaky_relu2"}, "linear"]}
             }
         },
         "training": {
@@ -1051,17 +1051,24 @@
                 "input_graph_embedding": {"input_dim": 100, "output_dim": 64},
                 "gauss_args": {"bins": 20, "distance": 4, "offset": 0.0, "sigma": 0.4},
                 "meg_block_args": {"node_embed": [64, 32, 32], "edge_embed": [64, 32, 32],
-                                   "env_embed": [64, 32, 32], "activation": "kgcnn>softplus2"},
+                                   "env_embed": [64, 32, 32],
+                                   "activation": {"class_name": "function", "config": "kgcnn>softplus2"}
+                                   },
                 "set2set_args": {"channels": 16, "T": 3, "pooling_method": "sum", "init_qstar": "0"},
-                "node_ff_args": {"units": [64, 32], "activation": "kgcnn>softplus2"},
-                "edge_ff_args": {"units": [64, 32], "activation": "kgcnn>softplus2"},
-                "state_ff_args": {"units": [64, 32], "activation": "kgcnn>softplus2"},
+                "node_ff_args": {"units": [64, 32],
+                                 "activation": {"class_name": "function", "config": "kgcnn>softplus2"}},
+                "edge_ff_args": {"units": [64, 32],
+                                 "activation": {"class_name": "function", "config": "kgcnn>softplus2"}},
+                "state_ff_args": {"units": [64, 32],
+                                  "activation": {"class_name": "function", "config": "kgcnn>softplus2"}},
                 "nblocks": 3, "has_ff": True, "dropout": None, "use_set2set": True,
                 "make_distance": True, "expand_distance": True,
                 "verbose": 10,
                 "output_embedding": "graph",
                 "output_mlp": {"use_bias": [True, True, True], "units": [32, 16, 1],
-                               "activation": ["kgcnn>softplus2", "kgcnn>softplus2", "linear"]}
+                               "activation": [{"class_name": "function", "config": "kgcnn>softplus2"},
+                                              {"class_name": "function", "config": "kgcnn>softplus2"},
+                                              "linear"]}
             }
         },
         "training": {

diff --git a/training/hyper/hyper_freesolv.py b/training/hyper/hyper_freesolv.py
@@ -28,15 +28,15 @@
             "fit": {"batch_size": 32, "epochs": 300, "validation_freq": 1, "verbose": 2, "callbacks": []},
             "compile": {
                 "optimizer": {"class_name": "Adam",
-                    "config": {"learning_rate": {
-                        "module": "keras.optimizers.schedules",
-                        "class_name": "ExponentialDecay",
-                        "config": {"initial_learning_rate": 0.001,
-                                   "decay_steps": 5800,
-                                   "decay_rate": 0.5, "staircase":  False}
-                        }
-                    }
-                },
+                              "config": {"learning_rate": {
+                                  "module": "keras.optimizers.schedules",
+                                  "class_name": "ExponentialDecay",
+                                  "config": {"initial_learning_rate": 0.001,
+                                             "decay_steps": 5800,
+                                             "decay_rate": 0.5, "staircase": False}
+                              }
+                              }
+                              },
                 "loss": "mean_absolute_error"
             },
             "scaler": {"class_name": "StandardLabelScaler",
@@ -142,7 +142,7 @@
                 "input_node_embedding": {"input_dim": 95, "output_dim": 64},
                 "input_edge_embedding": {"input_dim": 8, "output_dim": 64},
                 "attention_args": {"units": 64, "use_bias": True, "use_edge_features": True,
-                                   "activation": "kgcnn>leaky_relu",
+                                   "activation": {"class_name": "function", "config": "kgcnn>leaky_relu2"},
                                    "use_final_activation": False, "has_self_loops": True},
                 "pooling_nodes_args": {"pooling_method": "scatter_sum"},
                 "depth": 4, "attention_heads_num": 10,
@@ -206,11 +206,17 @@
                 "input_node_embedding": {"input_dim": 95, "output_dim": 64},
                 "output_embedding": "graph",
                 'output_mlp': {"use_bias": [True, True], "units": [64, 1],
-                               "activation": ['kgcnn>shifted_softplus', "linear"]},
+                               "activation": [
+                                   {"class_name": "function", "config": "kgcnn>shifted_softplus"},
+                                   "linear"]
+                               },
                 'last_mlp': {"use_bias": [True, True], "units": [128, 64],
-                             "activation": ['kgcnn>shifted_softplus', 'kgcnn>shifted_softplus']},
+                             "activation": [{"class_name": "function", "config": "kgcnn>shifted_softplus"},
+                                            {"class_name": "function", "config": "kgcnn>shifted_softplus"}]},
                 "interaction_args": {
-                    "units": 128, "use_bias": True, "activation": "kgcnn>shifted_softplus", "cfconv_pool": "scatter_sum"
+                    "units": 128, "use_bias": True,
+                    "activation": {"class_name": "function", "config": "kgcnn>shifted_softplus"},
+                    "cfconv_pool": "scatter_sum"
                 },
                 "node_pooling_args": {"pooling_method": "scatter_sum"},
                 "depth": 4,
@@ -545,7 +551,7 @@
                 "input_embedding": None,
                 "input_node_embedding": {"input_dim": 95, "output_dim": 64},
                 "input_edge_embedding": {"input_dim": 5, "output_dim": 64},
-                "gin_mlp": {"units": [64,64], "use_bias": True, "activation": ["relu","linear"],
+                "gin_mlp": {"units": [64, 64], "use_bias": True, "activation": ["relu", "linear"],
                             "use_normalization": True, "normalization_technique": "graph_layer"},
                 "gin_args": {},
                 "pooling_args": {"pooling_method": "sum"},
@@ -554,7 +560,7 @@
                 "edge_dense": {"units": 100, "use_bias": True, "activation": "linear"},
                 "edge_activation": {"activation": "relu"},
                 "node_dense": {"units": 100, "use_bias": True, "activation": "relu"},
-                "verbose": 10, "depthDMPNN": 5,"depthGIN": 5,
+                "verbose": 10, "depthDMPNN": 5, "depthGIN": 5,
                 "dropoutDMPNN": {"rate": 0.05},
                 "dropoutGIN": {"rate": 0.05},
                 "output_embedding": "graph", "output_to_tensor": True,
@@ -859,7 +865,7 @@
                 'output_embedding': 'graph',
                 'output_mlp': {"use_bias": [True, False], "units": [200, 1],
                                "activation": ['relu', 'linear'],
-                               "use_dropout": [True,  False],
+                               "use_dropout": [True, False],
                                "rate": [0.5, 0.0]}
             }
         },
@@ -915,17 +921,25 @@
                 "input_graph_embedding": {"input_dim": 100, "output_dim": 64},
                 "gauss_args": {"bins": 20, "distance": 4, "offset": 0.0, "sigma": 0.4},
                 "meg_block_args": {"node_embed": [64, 32, 32], "edge_embed": [64, 32, 32],
-                                   "env_embed": [64, 32, 32], "activation": "kgcnn>softplus2"},
+                                   "env_embed": [64, 32, 32],
+                                   "activation": {"class_name": "function", "config": "kgcnn>softplus2"}
+                                   },
                 "set2set_args": {"channels": 16, "T": 3, "pooling_method": "sum", "init_qstar": "0"},
-                "node_ff_args": {"units": [64, 32], "activation": "kgcnn>softplus2"},
-                "edge_ff_args": {"units": [64, 32], "activation": "kgcnn>softplus2"},
-                "state_ff_args": {"units": [64, 32], "activation": "kgcnn>softplus2"},
+                "node_ff_args": {"units": [64, 32],
+                                 "activation": {"class_name": "function", "config": "kgcnn>softplus2"}},
+                "edge_ff_args": {"units": [64, 32],
+                                 "activation": {"class_name": "function", "config": "kgcnn>softplus2"}},
+                "state_ff_args": {"units": [64, 32],
+                                  "activation": {"class_name": "function", "config": "kgcnn>softplus2"}},
                 "nblocks": 3, "has_ff": True, "dropout": None, "use_set2set": True,
                 "make_distance": True, "expand_distance": True,
                 "verbose": 10,
                 "output_embedding": "graph",
                 "output_mlp": {"use_bias": [True, True, True], "units": [32, 16, 1],
-                               "activation": ["kgcnn>softplus2", "kgcnn>softplus2", "linear"]}
+                               "activation": [
+                                   {"class_name": "function", "config": "kgcnn>softplus2"},
+                                   {"class_name": "function", "config": "kgcnn>softplus2"},
+                                   "linear"]}
             }
         },
         "training": {
@@ -1122,7 +1136,7 @@
                 "input_edge_embedding": {"input_dim": 5, "output_dim": 64},
                 "attention_args": {"units": 100},
                 "depthato": 2, "depthmol": 2,
-                "pooling_gat_nodes_args":  {'pooling_method': 'mean'},
+                "pooling_gat_nodes_args": {'pooling_method': 'mean'},
                 "dropout": 0.2,
                 "verbose": 10,
                 "output_embedding": "graph",
@@ -1239,7 +1253,7 @@
                 'name': "MEGAN",
                 'inputs': [
                     {'shape': (None, 41), 'name': "node_attributes", 'dtype': 'float32'},
-                    {'shape': (None, ), 'name': "edge_number", 'dtype': 'float32'},
+                    {'shape': (None,), 'name': "edge_number", 'dtype': 'float32'},
                     {'shape': (None, 2), 'name': "edge_indices", 'dtype': 'int64'},
                     {"shape": [2], "name": "graph_attributes", "dtype": "float32"},
                     {"shape": (), "name": "total_nodes", "dtype": "int64"},
@@ -1439,7 +1453,7 @@
                                   "angle_attributes": "angle_attributes_2"}},
                     {"map_list": {"method": "count_nodes_and_edges"}},
                     {"map_list": {"method": "count_nodes_and_edges", "total_edges": "total_ranges",
-                                   "count_edges": "range_indices"}},
+                                  "count_edges": "range_indices"}},
                     {"map_list": {"method": "count_nodes_and_edges", "total_edges": "total_angles_1",
                                   "count_edges": "angle_indices_1"}},
                     {"map_list": {"method": "count_nodes_and_edges", "total_edges": "total_angles_2",
@@ -1535,4 +1549,4 @@
             "kgcnn_version": "4.0.0"
         }
     },
-}
+}
diff --git a/training/hyper/hyper_iso17.py b/training/hyper/hyper_iso17.py
@@ -31,9 +31,13 @@
                         "cast_disjoint_kwargs": {"padded_disjoint": False},
                         "input_node_embedding": {"input_dim": 95, "output_dim": 128},
                         "last_mlp": {"use_bias": [True, True, True], "units": [128, 64, 1],
-                                     "activation": ['kgcnn>shifted_softplus', 'kgcnn>shifted_softplus', 'linear']},
+                                     "activation": [
+                                         {"class_name": "function", "config": "kgcnn>shifted_softplus"},
+                                         {"class_name": "function", "config": "kgcnn>shifted_softplus"},
+                                         'linear']},
                         "interaction_args": {
-                            "units": 128, "use_bias": True, "activation": "kgcnn>shifted_softplus",
+                            "units": 128, "use_bias": True,
+                            "activation": {"class_name": "function", "config": "kgcnn>shifted_softplus"},
                             "cfconv_pool": "scatter_sum"
                         },
                         "node_pooling_args": {"pooling_method": "scatter_sum"},