From ae8592521433ba734376018422b3b32fd13ca531 Mon Sep 17 00:00:00 2001 From: Chris Van Pelt Date: Sat, 14 Mar 2020 15:30:45 -0700 Subject: [PATCH 1/4] Allow reinits in sub procs --- pytorch_lightning/loggers/wandb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/loggers/wandb.py b/pytorch_lightning/loggers/wandb.py index 6f22050b2a96a..ad1ce56c8ac29 100644 --- a/pytorch_lightning/loggers/wandb.py +++ b/pytorch_lightning/loggers/wandb.py @@ -84,7 +84,7 @@ def experiment(self) -> Run: os.environ['WANDB_MODE'] = 'dryrun' self._experiment = wandb.init( name=self._name, dir=self._save_dir, project=self._project, anonymous=self._anonymous, - id=self._id, resume='allow', tags=self._tags, entity=self._entity) + reinit=True, id=self._id, resume='allow', tags=self._tags, entity=self._entity) return self._experiment def watch(self, model: nn.Module, log: str = 'gradients', log_freq: int = 100): From b6e8eb5ae39726ba083d867078f4483e39268a5d Mon Sep 17 00:00:00 2001 From: Chris Van Pelt Date: Sat, 14 Mar 2020 16:28:18 -0700 Subject: [PATCH 2/4] Dont create an experiment on pickle, name, or project --- pytorch_lightning/loggers/wandb.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/pytorch_lightning/loggers/wandb.py b/pytorch_lightning/loggers/wandb.py index ad1ce56c8ac29..1b434df1b19bc 100644 --- a/pytorch_lightning/loggers/wandb.py +++ b/pytorch_lightning/loggers/wandb.py @@ -62,10 +62,14 @@ def __init__(self, name: Optional[str] = None, save_dir: Optional[str] = None, def __getstate__(self): state = self.__dict__.copy() + # args needed to reload correct experiment + if self._experiment is not None: + state['_id'] = self._experiment.id + else: + state['_id'] = None + # cannot be pickled state['_experiment'] = None - # args needed to reload correct experiment - state['_id'] = self.experiment.id return state @property @@ -111,8 +115,14 @@ def finalize(self, status: str = 'success') -> None: @property def name(self) -> str: - return self.experiment.project_name() + if self._experiment: + return self.experiment.project_name() + else: + return None @property def version(self) -> str: - return self.experiment.id + if self._experiment: + return self.experiment.id + else: + return None From b77f3c871166a4940e90d32f065d9834f288d959 Mon Sep 17 00:00:00 2001 From: Chris Van Pelt Date: Sat, 14 Mar 2020 16:30:33 -0700 Subject: [PATCH 3/4] Comments consistency --- pytorch_lightning/loggers/wandb.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pytorch_lightning/loggers/wandb.py b/pytorch_lightning/loggers/wandb.py index 1b434df1b19bc..c68f9e389f302 100644 --- a/pytorch_lightning/loggers/wandb.py +++ b/pytorch_lightning/loggers/wandb.py @@ -115,14 +115,16 @@ def finalize(self, status: str = 'success') -> None: @property def name(self) -> str: + # don't create an experiment if we don't have one if self._experiment: - return self.experiment.project_name() + return self._experiment.project_name() else: return None @property def version(self) -> str: + # don't create an experiment if we don't have one if self._experiment: - return self.experiment.id + return self._experiment.id else: return None From 4241d84dba7eec43b4334bc923b07ca932e04b2d Mon Sep 17 00:00:00 2001 From: Chris Van Pelt Date: Sat, 14 Mar 2020 18:50:26 -0700 Subject: [PATCH 4/4] Fix test --- tests/loggers/test_wandb.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/loggers/test_wandb.py b/tests/loggers/test_wandb.py index abb49544b1ade..34076b17b825a 100644 --- a/tests/loggers/test_wandb.py +++ b/tests/loggers/test_wandb.py @@ -64,6 +64,8 @@ class Experiment: trainer_options = dict(max_epochs=1, logger=logger) trainer = Trainer(**trainer_options) + # Access the experiment to ensure it's created + trainer.logger.experiment pkl_bytes = pickle.dumps(trainer) trainer2 = pickle.loads(pkl_bytes)