Do not directly pass an optimizer to CmpNet

An optimizer is a tensorflow object, which (at least in graph mode in
tf1) is not deepcopy-able. Even if we were able to deeocopy it, we
probably wouldn't want to since it contains state. Scikit-learn needs to
be able to deepcopy an estimators arguments so that it can create copies
and derivatives of it.

Instead we pass a function that constructs the optimizer on-demand. This
function is deepcopy-able.

HISTORY.rst

@@ -9,6 +9,8 @@ Unreleased
   particular, the parameters nesterov, momentum and lr are now set to the
   default values set by keras.
 
+* TODO make an entry here
+
 1.2.1 (2020-06-08)
 ------------------
 

csrank/choicefunction/cmpnet_choice.py

@@ -19,7 +19,7 @@ def __init__(
         kernel_regularizer=l2(1e-4),
         kernel_initializer="lecun_normal",
         activation="relu",
-        optimizer=SGD(),
+        optimizer=SGD,
         metrics=["binary_accuracy"],
         batch_size=256,
         random_state=None,

csrank/choicefunction/fate_choice.py

@@ -21,7 +21,7 @@ def __init__(
         activation="selu",
         kernel_initializer="lecun_normal",
         kernel_regularizer=l2(0.01),
-        optimizer=SGD(),
+        optimizer=SGD,
         batch_size=256,
         metrics=None,
         random_state=None,

csrank/choicefunction/feta_choice.py

@@ -34,7 +34,7 @@ def __init__(
         kernel_regularizer=l2(1e-4),
         kernel_initializer="lecun_normal",
         activation="selu",
-        optimizer=SGD(),
+        optimizer=SGD,
         metrics=["binary_accuracy"],
         batch_size=256,
         random_state=None,
@@ -218,7 +218,7 @@ def create_input_lambda(i):
         model = Model(inputs=self.input_layer, outputs=scores)
         self.logger.debug("Compiling complete model...")
         model.compile(
-            loss=self.loss_function, optimizer=self.optimizer, metrics=self.metrics
+            loss=self.loss_function, optimizer=self.optimizer_, metrics=self.metrics
         )
         return model
 

csrank/choicefunction/ranknet_choice.py

@@ -19,7 +19,7 @@ def __init__(
         kernel_regularizer=l2(1e-4),
         kernel_initializer="lecun_normal",
         activation="relu",
-        optimizer=SGD(),
+        optimizer=SGD,
         metrics=["binary_accuracy"],
         batch_size=256,
         random_state=None,

csrank/core/cmpnet_core.py

@@ -4,7 +4,6 @@
 from keras import backend as K
 from keras import Input
 from keras import Model
-from keras import optimizers
 from keras.layers import concatenate
 from keras.layers import Dense
 from keras.optimizers import SGD
@@ -29,7 +28,7 @@ def __init__(
         kernel_regularizer=l2(1e-4),
         kernel_initializer="lecun_normal",
         activation="relu",
-        optimizer=SGD(),
+        optimizer=SGD,
         metrics=["binary_accuracy"],
         batch_size=256,
         random_state=None,
@@ -47,8 +46,7 @@ def __init__(
         self.kernel_initializer = kernel_initializer
         self.loss_function = loss_function
 
-        self.optimizer = optimizers.get(optimizer)
-        self._optimizer_config = self.optimizer.get_config()
+        self.optimizer = optimizer
 
         self.n_hidden = n_hidden
         self.n_units = n_units
@@ -97,6 +95,7 @@ def construct_model(self):
             model: keras :class:`Model`
                 Neural network to learn the CmpNet utility score
         """
+        self.optimizer_ = self.optimizer()  # instantiate the optimizer
         x1x2 = concatenate([self.x1, self.x2])
         x2x1 = concatenate([self.x2, self.x1])
         self.logger.debug("Creating the model")
@@ -110,7 +109,7 @@ def construct_model(self):
         merged_output = concatenate([N_g, N_l])
         model = Model(inputs=[self.x1, self.x2], outputs=merged_output)
         model.compile(
-            loss=self.loss_function, optimizer=self.optimizer, metrics=self.metrics
+            loss=self.loss_function, optimizer=self.optimizer_, metrics=self.metrics
         )
         return model
 
@@ -212,7 +211,7 @@ def clear_memory(self, **kwargs):
             sess = tf.Session()
             K.set_session(sess)
 
-            self.optimizer = self.optimizer.from_config(self._optimizer_config)
+            self._initialize_optimizer()
             self._construct_layers(
                 kernel_regularizer=self.kernel_regularizer,
                 kernel_initializer=self.kernel_initializer,
@@ -255,8 +254,8 @@ def set_tunable_parameters(
         self.n_units = n_units
         self.kernel_regularizer = l2(reg_strength)
         self.batch_size = batch_size
-        self.optimizer = self.optimizer.from_config(self._optimizer_config)
-        K.set_value(self.optimizer.lr, learning_rate)
+        self._initialize_optimizer()
+        K.set_value(self.optimizer_.lr, learning_rate)
         self._construct_layers(
             kernel_regularizer=self.kernel_regularizer,
             kernel_initializer=self.kernel_initializer,

csrank/core/fate_network.py

@@ -1,6 +1,5 @@
 import logging
 
-from keras import optimizers
 import keras.backend as K
 from keras.layers import Dense
 from keras.layers import Input
@@ -29,7 +28,7 @@ def __init__(
         activation="selu",
         kernel_initializer="lecun_normal",
         kernel_regularizer=l2(0.01),
-        optimizer=SGD(),
+        optimizer=SGD,
         batch_size=256,
         random_state=None,
         **kwargs,
@@ -69,15 +68,15 @@ def __init__(
         self.kernel_initializer = kernel_initializer
         self.kernel_regularizer = kernel_regularizer
         self.batch_size = batch_size
-        self.optimizer = optimizers.get(optimizer)
-        self._optimizer_config = self.optimizer.get_config()
+        self.optimizer = optimizer
         self.joint_layers = None
         self.scorer = None
         keys = list(kwargs.keys())
         for key in keys:
             if key not in allowed_dense_kwargs:
                 del kwargs[key]
         self.kwargs = kwargs
+        self._initialize_optimizer()
         self._construct_layers(
             activation=self.activation,
             kernel_initializer=self.kernel_initializer,
@@ -167,8 +166,8 @@ def set_tunable_parameters(
         self.kernel_regularizer = l2(reg_strength)
         self.batch_size = batch_size
         # Hack to fix memory leak:
-        self.optimizer = self.optimizer.from_config(self._optimizer_config)
-        K.set_value(self.optimizer.lr, learning_rate)
+        self._initialize_optimizer()
+        K.set_value(self.optimizer_.lr, learning_rate)
 
         self._construct_layers(
             activation=self.activation,
@@ -474,7 +473,7 @@ def construct_model(self, n_features, n_objects):
         model = Model(inputs=input_layer, outputs=scores)
 
         model.compile(
-            loss=self.loss_function, optimizer=self.optimizer, metrics=self.metrics
+            loss=self.loss_function, optimizer=self.optimizer_, metrics=self.metrics
         )
         return model
 
@@ -536,6 +535,7 @@ def fit(
         """
         self.random_state_ = check_random_state(self.random_state)
         _n_instances, self.n_objects_fit_, self.n_object_features_fit_ = X.shape
+        self._initialize_optimizer()
         self._fit(
             X=X,
             Y=Y,
@@ -703,7 +703,7 @@ def clear_memory(self, n_objects=5, **kwargs):
             K.clear_session()
             sess = tf.Session()
             K.set_session(sess)
-            self.optimizer = self.optimizer.from_config(self._optimizer_config)
+            self._initialize_optimizer()
             self._construct_layers(
                 activation=self.activation,
                 kernel_initializer=self.kernel_initializer,

csrank/core/feta_network.py

@@ -5,7 +5,6 @@
 from keras import backend as K
 from keras import Input
 from keras import Model
-from keras import optimizers
 from keras.layers import add
 from keras.layers import concatenate
 from keras.layers import Dense
@@ -36,7 +35,7 @@ def __init__(
         kernel_regularizer=l2(1e-4),
         kernel_initializer="lecun_normal",
         activation="selu",
-        optimizer=SGD(),
+        optimizer=SGD,
         metrics=None,
         batch_size=256,
         random_state=None,
@@ -54,8 +53,7 @@ def __init__(
         self.num_subsample = num_subsample
         self.batch_size = batch_size
         self.hash_file = None
-        self.optimizer = optimizers.get(optimizer)
-        self._optimizer_config = self.optimizer.get_config()
+        self.optimizer = optimizer
         self._use_zeroth_model = add_zeroth_order_model
         self.n_hidden = n_hidden
         self.n_units = n_units
@@ -251,7 +249,7 @@ def create_input_lambda(i):
         model = Model(inputs=self.input_layer, outputs=scores)
         self.logger.debug("Compiling complete model...")
         model.compile(
-            loss=self.loss_function, optimizer=self.optimizer, metrics=self.metrics
+            loss=self.loss_function, optimizer=self.optimizer_, metrics=self.metrics
         )
         return model
 
@@ -282,6 +280,7 @@ def fit(
                 Keyword arguments for the fit function
         """
         _n_instances, self.n_objects_fit_, self.n_object_features_fit_ = X.shape
+        self._initialize_optimizer()
         self._construct_layers(
             kernel_regularizer=self.kernel_regularizer,
             kernel_initializer=self.kernel_initializer,
@@ -369,8 +368,8 @@ def set_tunable_parameters(
         self.n_units = n_units
         self.kernel_regularizer = l2(reg_strength)
         self.batch_size = batch_size
-        self.optimizer = self.optimizer.from_config(self._optimizer_config)
-        K.set_value(self.optimizer.lr, learning_rate)
+        self._initialize_optimizer()
+        K.set_value(self.optimizer_.lr, learning_rate)
         self._pairwise_model = None
         self._zero_order_model = None
         self._construct_layers(
@@ -402,7 +401,7 @@ def clear_memory(self, **kwargs):
 
             self._pairwise_model = None
             self._zero_order_model = None
-            self.optimizer = self.optimizer.from_config(self._optimizer_config)
+            self._initialize_optimizer()
             self._construct_layers(
                 kernel_regularizer=self.kernel_regularizer,
                 kernel_initializer=self.kernel_initializer,

csrank/core/ranknet_core.py

@@ -3,7 +3,6 @@
 from keras import backend as K
 from keras import Input
 from keras import Model
-from keras import optimizers
 from keras.layers import add
 from keras.layers import Dense
 from keras.layers import Lambda
@@ -28,7 +27,7 @@ def __init__(
         kernel_regularizer=l2(1e-4),
         kernel_initializer="lecun_normal",
         activation="relu",
-        optimizer=SGD(),
+        optimizer=SGD,
         metrics=["binary_accuracy"],
         batch_size=256,
         random_state=None,
@@ -41,8 +40,7 @@ def __init__(
         self.kernel_regularizer = kernel_regularizer
         self.kernel_initializer = kernel_initializer
         self.loss_function = loss_function
-        self.optimizer = optimizers.get(optimizer)
-        self._optimizer_config = self.optimizer.get_config()
+        self.optimizer = optimizer
         self.n_hidden = n_hidden
         self.n_units = n_units
         keys = list(kwargs.keys())
@@ -101,7 +99,7 @@ def construct_model(self):
         output = self.output_node(merged_inputs)
         model = Model(inputs=[self.x1, self.x2], outputs=output)
         model.compile(
-            loss=self.loss_function, optimizer=self.optimizer, metrics=self.metrics
+            loss=self.loss_function, optimizer=self.optimizer_, metrics=self.metrics
         )
         return model
 
@@ -147,6 +145,7 @@ def fit(
         self.logger.debug("Instances created {}".format(X1.shape[0]))
         self.logger.debug("Creating the model")
 
+        self._initialize_optimizer()
         self._construct_layers(
             kernel_regularizer=self.kernel_regularizer,
             kernel_initializer=self.kernel_initializer,
@@ -217,7 +216,7 @@ def clear_memory(self, **kwargs):
             K.set_session(sess)
 
             self._scoring_model = None
-            self.optimizer = self.optimizer.from_config(self._optimizer_config)
+            self._initialize_optimizer()
             self._construct_layers(
                 kernel_regularizer=self.kernel_regularizer,
                 kernel_initializer=self.kernel_initializer,
@@ -260,8 +259,8 @@ def set_tunable_parameters(
         self.n_units = n_units
         self.kernel_regularizer = l2(reg_strength)
         self.batch_size = batch_size
-        self.optimizer = self.optimizer.from_config(self._optimizer_config)
-        K.set_value(self.optimizer.lr, learning_rate)
+        self._initialize_optimizer()
+        K.set_value(self.optimizer_.lr, learning_rate)
         self._scoring_model = None
         self._construct_layers(
             kernel_regularizer=self.kernel_regularizer,

csrank/discretechoice/cmpnet_discrete_choice.py

@@ -18,7 +18,7 @@ def __init__(
         kernel_regularizer=l2(1e-4),
         kernel_initializer="lecun_normal",
         activation="relu",
-        optimizer=SGD(),
+        optimizer=SGD,
         metrics=["binary_accuracy"],
         batch_size=256,
         random_state=None,

csrank/discretechoice/fate_discrete_choice.py

@@ -20,7 +20,7 @@ def __init__(
         activation="selu",
         kernel_initializer="lecun_normal",
         kernel_regularizer=l2(0.01),
-        optimizer=SGD(),
+        optimizer=SGD,
         batch_size=256,
         random_state=None,
         **kwargs,

csrank/discretechoice/feta_discrete_choice.py

@@ -32,7 +32,7 @@ def __init__(
         kernel_regularizer=l2(1e-4),
         kernel_initializer="lecun_normal",
         activation="selu",
-        optimizer=SGD(),
+        optimizer=SGD,
         metrics=["categorical_accuracy"],
         batch_size=256,
         random_state=None,
@@ -262,7 +262,7 @@ def get_score_object(i):
         model = Model(inputs=self.input_layer, outputs=scores)
         self.logger.debug("Compiling complete model...")
         model.compile(
-            loss=self.loss_function, optimizer=self.optimizer, metrics=self.metrics
+            loss=self.loss_function, optimizer=self.optimizer_, metrics=self.metrics
         )
         return model
 

csrank/discretechoice/ranknet_discrete_choice.py

@@ -18,7 +18,7 @@ def __init__(
         kernel_regularizer=l2(1e-4),
         kernel_initializer="lecun_normal",
         activation="relu",
-        optimizer=SGD(),
+        optimizer=SGD,
         metrics=["binary_accuracy"],
         batch_size=256,
         random_state=None,

csrank/learner.py

@@ -4,7 +4,21 @@
 from csrank.tunable import Tunable
 
 
+def filter_dict_by_prefix(source, prefix):
+    result = dict()
+    for key in source.keys():
+        if key.startswith(prefix):
+            key_stripped = key[len(prefix) :]
+            result[key_stripped] = source[key]
+    return result
+
+
 class Learner(Tunable, metaclass=ABCMeta):
+    def _initialize_optimizer(self):
+        optimizer_params = filter_dict_by_prefix(self.__dict__, "optimizer__")
+        optimizer_params.update(filter_dict_by_prefix(self.kwargs, "optimizer__"))
+        self.optimizer_ = self.optimizer(**optimizer_params)
+
     @abstractmethod
     def fit(self, X, Y, **kwargs):
         """