From 46c582066aa0f98ae191273b76893ac38b1f434e Mon Sep 17 00:00:00 2001 From: SuperSashka Date: Fri, 28 Jul 2023 17:19:17 +0300 Subject: [PATCH 1/3] normalized_loss_stop switch At some point we return to the stop criterion, so far it is user-defined to use normalized loss (with all lambdas =1) or not for stop criterion. --- examples/example_1d_navier_stokes.py | 2 +- examples/example_SODtest.py | 6 +++--- tedeous/cache.py | 23 ++++++++++++++--------- tedeous/solver.py | 10 +++++++--- 4 files changed, 25 insertions(+), 16 deletions(-) diff --git a/examples/example_1d_navier_stokes.py b/examples/example_1d_navier_stokes.py index 3259cdc6..9d542157 100644 --- a/examples/example_1d_navier_stokes.py +++ b/examples/example_1d_navier_stokes.py @@ -7,7 +7,7 @@ os.environ['KMP_DUPLICATE_LIB_OK'] = 'TRUE' -sys.path.pop() +#sys.path.pop() sys.path.append(os.path.abspath(os.path.join(os.path.dirname( __file__ ), '..'))) from tedeous.input_preprocessing import Equation diff --git a/examples/example_SODtest.py b/examples/example_SODtest.py index 38a7fc7f..ab35f769 100644 --- a/examples/example_SODtest.py +++ b/examples/example_SODtest.py @@ -14,9 +14,9 @@ os.environ['KMP_DUPLICATE_LIB_OK'] = 'TRUE' -sys.path.append('../') +#sys.path.append('../') -sys.path.pop() +#sys.path.pop() sys.path.append(os.path.abspath(os.path.join(os.path.dirname( __file__ ), '..'))) from tedeous.input_preprocessing import Equation @@ -203,7 +203,7 @@ def u0(x,x0): model = Solver(grid, equation, model, 'NN').solve( lambda_bound=1000, verbose=True, learning_rate=1e-2, eps=1e-6, tmin=1000, tmax=1e5,use_cache=False,cache_dir='../cache/',cache_verbose=False, - save_always=False,no_improvement_patience=500,print_every=1000,step_plot_print=False,step_plot_save=True,image_save_dir=img_dir) + save_always=False,no_improvement_patience=500,print_every=1000,step_plot_print=False,step_plot_save=True,image_save_dir=img_dir,normalized_loss_stop=True) end = time.time() diff --git a/tedeous/cache.py b/tedeous/cache.py index 3fd3fd25..5960d2bf 100644 --- a/tedeous/cache.py +++ b/tedeous/cache.py @@ -132,7 +132,7 @@ def model_reform(init_model, model): def cache_lookup(self, lambda_operator: float = 1., lambda_bound: float = 0.001, - nmodels: Union[int, None] = None, save_graph: bool = False, cache_verbose: bool = False) -> Tuple[dict, torch.Tensor]: + nmodels: Union[int, None] = None, save_graph: bool = False, cache_verbose: bool = False, return_normalized_loss: bool = False) -> Tuple[dict, torch.Tensor]: """ Looking for a saved cache. Args: @@ -193,7 +193,9 @@ def cache_lookup(self, lambda_operator: float = 1., lambda_bound: float = 0.001, if best_checkpoint == {}: best_checkpoint = None min_loss = np.inf - return best_checkpoint, min_norm_loss + if return_normalized_loss: + min_loss=min_norm_loss + return best_checkpoint, min_loss def save_model(self, prep_model: Any, state: dict, optimizer_state: dict, name: Union[str, None] = None): """ @@ -326,7 +328,7 @@ def cache_retrain(self, cache_checkpoint, cache_verbose: bool = False) -> Union[ def cache_nn(self, cache_dir: str, nmodels: Union[int, None], lambda_operator: float, lambda_bound: float, cache_verbose: bool,model_randomize_parameter: Union[float, None], - cache_model: torch.nn.Sequential, ): + cache_model: torch.nn.Sequential, return_normalized_loss: bool = False): """ Restores the model from the cache and uses it for retraining. Args: @@ -345,7 +347,8 @@ def cache_nn(self, cache_dir: str, nmodels: Union[int, None], lambda_operator: f cache_checkpoint, min_loss = self.cache_lookup(nmodels=nmodels, cache_verbose=cache_verbose, lambda_operator= lambda_operator, - lambda_bound=lambda_bound) + lambda_bound=lambda_bound, + return_normalized_loss = return_normalized_loss) self.model, optimizer_state = self.cache_retrain(cache_checkpoint, cache_verbose=cache_verbose) @@ -356,7 +359,7 @@ def cache_nn(self, cache_dir: str, nmodels: Union[int, None], lambda_operator: f def cache_mat(self, nmodels: Union[int, None],lambda_operator: float, lambda_bound: float, cache_verbose: bool,model_randomize_parameter: Union[float, None], - cache_model: torch.nn.Sequential): + cache_model: torch.nn.Sequential, return_normalized_loss: bool = False): """ Restores the model from the cache and uses it for retraining. Args: @@ -389,7 +392,8 @@ def cache_mat(self, nmodels: Union[int, None],lambda_operator: float, lambda_bou cache_dir=self.cache_dir, nmodels=nmodels, cache_verbose=cache_verbose, - lambda_bound=lambda_bound) + lambda_bound=lambda_bound, + return_normalized_loss=return_normalized_loss) prepared_model, optimizer_state = model_cls.cache_retrain( cache_checkpoint, cache_verbose=cache_verbose) @@ -411,7 +415,8 @@ def cache_mat(self, nmodels: Union[int, None],lambda_operator: float, lambda_bou def cache(self, nmodels: Union[int, None],lambda_operator, lambda_bound: float, cache_verbose: bool,model_randomize_parameter: Union[float, None], - cache_model: torch.nn.Sequential, ): + cache_model: torch.nn.Sequential, + return_normalized_loss: bool = False): """ Restores the model from the cache and uses it for retraining. Args: @@ -430,9 +435,9 @@ def cache(self, nmodels: Union[int, None],lambda_operator, lambda_bound: float, if self.mode != 'mat': return self.cache_nn(self.cache_dir, nmodels,lambda_operator, lambda_bound, cache_verbose, model_randomize_parameter, - cache_model) + cache_model,return_normalized_loss=return_normalized_loss) elif self.mode == 'mat': return self.cache_mat(self.cache_dir, nmodels, lambda_operator, lambda_bound, cache_verbose, model_randomize_parameter, - cache_model) + cache_model,return_normalized_loss=return_normalized_loss) diff --git a/tedeous/solver.py b/tedeous/solver.py index 9e0ea57d..dfac65dd 100644 --- a/tedeous/solver.py +++ b/tedeous/solver.py @@ -222,7 +222,8 @@ def solve(self,lambda_operator: Union[float, list] = 1,lambda_bound: Union[float patience: int = 5, loss_oscillation_window: int = 100, no_improvement_patience: int = 1000, model_randomize_parameter: Union[int, float] = 0, optimizer_mode: str = 'Adam', step_plot_print: Union[bool, int] = False, - step_plot_save: Union[bool, int] = False, image_save_dir: Union[str, None] = None, tol: float = 0,clear_cache: bool =False) -> Any: + step_plot_save: Union[bool, int] = False, image_save_dir: Union[str, None] = None, tol: float = 0, + clear_cache: bool =False, normalized_loss_stop: bool = False) -> Any: """ High-level interface for solving equations. @@ -277,7 +278,7 @@ def solve(self,lambda_operator: Union[float, list] = 1,lambda_bound: Union[float cache_verbose, model_randomize_parameter, cache_model, - ) + return_normalized_loss=normalized_loss_stop) Solution_class = Solution(self.grid, self.equal_cls, self.model, self.mode, self.weak_form, @@ -315,7 +316,10 @@ def closure(): tol=tol) loss.backward() - cur_loss = loss_normalized.item() + if normalized_loss_stop: + cur_loss = loss_normalized.item() + else: + cur_loss = loss.item() return loss stop_dings = 0 From f9810d35f6a4ebbaba28526ba956bb0093b6df4f Mon Sep 17 00:00:00 2001 From: SuperSashka Date: Fri, 28 Jul 2023 17:22:14 +0300 Subject: [PATCH 2/3] Example status quo --- examples/example_SODtest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/example_SODtest.py b/examples/example_SODtest.py index ab35f769..1290c768 100644 --- a/examples/example_SODtest.py +++ b/examples/example_SODtest.py @@ -203,7 +203,7 @@ def u0(x,x0): model = Solver(grid, equation, model, 'NN').solve( lambda_bound=1000, verbose=True, learning_rate=1e-2, eps=1e-6, tmin=1000, tmax=1e5,use_cache=False,cache_dir='../cache/',cache_verbose=False, - save_always=False,no_improvement_patience=500,print_every=1000,step_plot_print=False,step_plot_save=True,image_save_dir=img_dir,normalized_loss_stop=True) + save_always=False,no_improvement_patience=500,print_every=1000,step_plot_print=False,step_plot_save=True,image_save_dir=img_dir) end = time.time() From 5f1538f85c53e812ae46b2551b1c35205013b5fa Mon Sep 17 00:00:00 2001 From: SuperSashka Date: Fri, 28 Jul 2023 17:23:22 +0300 Subject: [PATCH 3/3] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index db6a66ac..de29d079 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ def get_requirements(): setup( name = 'tedeous', - version= '0.3.0' , + version= '0.3.1' , description = 'TEDEouS - Torch Exhaustive Differential Equations Solver. Differential equation solver, based on pytorch library', long_description = 'Combine power of pytorch, numerical methods and math overall to conquer and solve ALL {O,P}DEs. There are some examples to provide a little insight to an operator form', author = 'Alexander Hvatov',