diff --git a/tests/base/deterministic_model.py b/tests/base/deterministic_model.py index d8847d592e1de..6f6b5f858ff17 100644 --- a/tests/base/deterministic_model.py +++ b/tests/base/deterministic_model.py @@ -15,7 +15,6 @@ from torch import nn from torch.utils.data import Dataset, DataLoader -from pytorch_lightning.core.step_result import TrainResult, EvalResult from pytorch_lightning.core.lightning import LightningModule @@ -111,235 +110,6 @@ def training_epoch_end_scalar(self, outputs): assert batch_out.grad_fn is None assert isinstance(batch_out, torch.Tensor) - def training_step_no_default_callbacks_for_train_loop(self, batch, batch_idx): - """ - Early stop and checkpoint only on these values - """ - acc = self.step(batch, batch_idx) - result = TrainResult(minimize=acc) - assert 'early_step_on' not in result - assert 'checkpoint_on' in result - return result - - def training_step_no_callbacks_result_obj(self, batch, batch_idx): - """ - Early stop and checkpoint only on these values - """ - acc = self.step(batch, batch_idx) - result = TrainResult(minimize=acc, checkpoint_on=False) - assert 'early_step_on' not in result - assert 'checkpoint_on' not in result - return result - - def training_step_result_log_epoch_and_step_for_callbacks(self, batch, batch_idx): - """ - Early stop and checkpoint only on these values - """ - acc = self.step(batch, batch_idx) - - self.assert_backward = False - losses = [20, 19, 18, 10, 15, 14, 9, 11, 11, 20] - idx = self.current_epoch - loss = acc + losses[idx] - result = TrainResult(minimize=loss, early_stop_on=loss, checkpoint_on=loss) - return result - - def training_step_result_log_step_only(self, batch, batch_idx): - acc = self.step(batch, batch_idx) - result = TrainResult(minimize=acc) - - # step only metrics - result.log(f'step_log_and_pbar_acc1_b{batch_idx}', torch.tensor(11).type_as(acc), prog_bar=True) - result.log(f'step_log_acc2_b{batch_idx}', torch.tensor(12).type_as(acc)) - result.log(f'step_pbar_acc3_b{batch_idx}', torch.tensor(13).type_as(acc), logger=False, prog_bar=True) - - self.training_step_called = True - return result - - def training_step_result_log_epoch_only(self, batch, batch_idx): - acc = self.step(batch, batch_idx) - result = TrainResult(minimize=acc) - - result.log(f'epoch_log_and_pbar_acc1_e{self.current_epoch}', torch.tensor(14).type_as(acc), - on_epoch=True, prog_bar=True, on_step=False) - result.log(f'epoch_log_acc2_e{self.current_epoch}', torch.tensor(15).type_as(acc), - on_epoch=True, on_step=False) - result.log(f'epoch_pbar_acc3_e{self.current_epoch}', torch.tensor(16).type_as(acc), - on_epoch=True, logger=False, prog_bar=True, on_step=False) - - self.training_step_called = True - return result - - def training_step_result_log_epoch_and_step(self, batch, batch_idx): - acc = self.step(batch, batch_idx) - result = TrainResult(minimize=acc) - - val_1 = (5 + batch_idx) * (self.current_epoch + 1) - val_2 = (6 + batch_idx) * (self.current_epoch + 1) - val_3 = (7 + batch_idx) * (self.current_epoch + 1) - result.log('step_epoch_log_and_pbar_acc1', torch.tensor(val_1).type_as(acc), - on_epoch=True, prog_bar=True) - result.log('step_epoch_log_acc2', torch.tensor(val_2).type_as(acc), - on_epoch=True) - result.log('step_epoch_pbar_acc3', torch.tensor(val_3).type_as(acc), - on_epoch=True, logger=False, prog_bar=True) - - self.training_step_called = True - return result - - def training_epoch_end_return_for_log_epoch_and_step(self, result): - """ - There should be an array of scalars without graphs that are all 171 (4 of them) - """ - self.training_epoch_end_called = True - - if self.use_dp or self.use_ddp2: - pass - else: - # only saw 4 batches - assert isinstance(result, TrainResult) - - result.step_epoch_log_acc2 = result.step_epoch_log_acc2_step.prod() - result.step_epoch_pbar_acc3 = result.step_epoch_pbar_acc3_step.prod() - result.step_epoch_log_and_pbar_acc1 = result.step_epoch_log_and_pbar_acc1_step.prod() - result.minimize = result.minimize.mean() - result.checkpoint_on = result.checkpoint_on.mean() - - result.step_epoch_log_and_pbar_acc1_step = result.step_epoch_log_and_pbar_acc1_step.prod() - result.step_epoch_log_and_pbar_acc1_epoch = result.step_epoch_log_and_pbar_acc1_epoch.prod() - result.step_epoch_log_acc2_step = result.step_epoch_log_acc2_step.prod() - result.step_epoch_log_acc2_epoch = result.step_epoch_log_acc2_epoch.prod() - result.step_epoch_pbar_acc3_step = result.step_epoch_pbar_acc3_step.prod() - result.step_epoch_pbar_acc3_epoch = result.step_epoch_pbar_acc3_epoch.prod() - result.log('epoch_end_log_acc', torch.tensor(1212).type_as(result.step_epoch_log_acc2_epoch), - logger=True, on_epoch=True) - result.log('epoch_end_pbar_acc', torch.tensor(1213).type_as(result.step_epoch_log_acc2_epoch), - logger=False, prog_bar=True, on_epoch=True) - result.log('epoch_end_log_pbar_acc', torch.tensor(1214).type_as(result.step_epoch_log_acc2_epoch), - logger=True, prog_bar=True, on_epoch=True) - return result - - # -------------------------- - # EvalResults - # -------------------------- - def validation_step_result_callbacks(self, batch, batch_idx): - acc = self.step(batch, batch_idx) - - self.assert_backward = False - losses = [20, 19, 20, 21, 22, 23] - idx = self.current_epoch - loss = acc + losses[idx] - result = EvalResult(early_stop_on=loss, checkpoint_on=loss) - - self.validation_step_called = True - return result - - def validation_step_result_no_callbacks(self, batch, batch_idx): - acc = self.step(batch, batch_idx) - - self.assert_backward = False - losses = [20, 19, 20, 21, 22, 23, 50, 50, 50, 50, 50, 50] - idx = self.current_epoch - loss = acc + losses[idx] - - result = EvalResult(checkpoint_on=loss) - - self.validation_step_called = True - return result - - def validation_step_result_only_epoch_metrics(self, batch, batch_idx): - """ - Only track epoch level metrics - """ - acc = self.step(batch, batch_idx) - result = EvalResult(checkpoint_on=acc, early_stop_on=acc) - - # step only metrics - result.log('no_val_no_pbar', torch.tensor(11 + batch_idx).type_as(acc), prog_bar=False, logger=False) - result.log('val_step_log_acc', torch.tensor(11 + batch_idx).type_as(acc), prog_bar=False, logger=True) - result.log('val_step_log_pbar_acc', torch.tensor(12 + batch_idx).type_as(acc), prog_bar=True, logger=True) - result.log('val_step_pbar_acc', torch.tensor(13 + batch_idx).type_as(acc), prog_bar=True, logger=False) - - self.validation_step_called = True - return result - - def validation_step_result_only_step_metrics(self, batch, batch_idx): - """ - Only track epoch level metrics - """ - acc = self.step(batch, batch_idx) - result = EvalResult(checkpoint_on=acc, early_stop_on=acc) - - # step only metrics - result.log('no_val_no_pbar', torch.tensor(11 + batch_idx).type_as(acc), - prog_bar=False, logger=False, on_epoch=False, on_step=True) - result.log('val_step_log_acc', torch.tensor(11 + batch_idx).type_as(acc), - prog_bar=False, logger=True, on_epoch=False, on_step=True) - result.log('val_step_log_pbar_acc', torch.tensor(12 + batch_idx).type_as(acc), - prog_bar=True, logger=True, on_epoch=False, on_step=True) - result.log('val_step_pbar_acc', torch.tensor(13 + batch_idx).type_as(acc), - prog_bar=True, logger=False, on_epoch=False, on_step=True) - result.log('val_step_batch_idx', torch.tensor(batch_idx).type_as(acc), - prog_bar=True, logger=True, on_epoch=False, on_step=True) - - self.validation_step_called = True - return result - - def validation_step_result_epoch_step_metrics(self, batch, batch_idx): - """ - Only track epoch level metrics - """ - acc = self.step(batch, batch_idx) - result = EvalResult(checkpoint_on=acc, early_stop_on=acc) - - # step only metrics - result.log('no_val_no_pbar', torch.tensor(11 + batch_idx).type_as(acc), - prog_bar=False, logger=False, on_epoch=True, on_step=True) - result.log('val_step_log_acc', torch.tensor(11 + batch_idx).type_as(acc), - prog_bar=False, logger=True, on_epoch=True, on_step=True) - result.log('val_step_log_pbar_acc', torch.tensor(12 + batch_idx).type_as(acc), - prog_bar=True, logger=True, on_epoch=True, on_step=True) - result.log('val_step_pbar_acc', torch.tensor(13 + batch_idx).type_as(acc), - prog_bar=True, logger=False, on_epoch=True, on_step=True) - result.log('val_step_batch_idx', torch.tensor(batch_idx).type_as(acc), - prog_bar=True, logger=True, on_epoch=True, on_step=True) - - self.validation_step_called = True - return result - - def validation_step_for_epoch_end_result(self, batch, batch_idx): - """ - EvalResult flows to epoch end (without step_end) - """ - acc = self.step(batch, batch_idx) - result = EvalResult(checkpoint_on=acc, early_stop_on=acc) - - # step only metrics - result.log('val_step_metric', torch.tensor(batch_idx).type_as(acc), - prog_bar=True, logger=True, on_epoch=True, on_step=False) - result.log('batch_idx', torch.tensor(batch_idx).type_as(acc), - prog_bar=True, logger=True, on_epoch=True, on_step=False) - - self.validation_step_called = True - return result - - def validation_epoch_end_result(self, result): - self.validation_epoch_end_called = True - - if self.trainer.running_sanity_check: - assert len(result.batch_idx) == 2 - else: - assert len(result.batch_idx) == self.trainer.limit_val_batches - - expected_val = result.val_step_metric.sum() / len(result.batch_idx) - result.val_step_metric = result.val_step_metric.mean() - result.batch_idx = result.batch_idx.mean() - assert result.val_step_metric == expected_val - - result.log('val_epoch_end_metric', torch.tensor(189).type_as(result.val_step_metric), prog_bar=True) - - return result - # -------------------------- # dictionary returns # -------------------------- diff --git a/tests/base/model_test_steps.py b/tests/base/model_test_steps.py index 0010dcdf14a19..440ec4c4c35b4 100644 --- a/tests/base/model_test_steps.py +++ b/tests/base/model_test_steps.py @@ -59,38 +59,6 @@ def test_step(self, batch, batch_idx, *args, **kwargs): 'test_dic': {'test_loss_a': loss_test}}) return output - def test_step_result_obj(self, batch, batch_idx, *args, **kwargs): - """ - Default, baseline test_step - :param batch: - :return: - """ - x, y = batch - x = x.view(x.size(0), -1) - y_hat = self(x) - - loss_test = self.loss(y, y_hat) - - # acc - labels_hat = torch.argmax(y_hat, dim=1) - test_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) - test_acc = torch.tensor(test_acc) - - test_acc = test_acc.type_as(x) - - result = EvalResult() - # alternate possible outputs to test - if batch_idx % 1 == 0: - result.log_dict({'test_loss': loss_test, 'test_acc': test_acc}) - return result - if batch_idx % 2 == 0: - return test_acc - - if batch_idx % 3 == 0: - result.log_dict({'test_loss': loss_test, 'test_acc': test_acc}) - result.test_dic = {'test_loss_a': loss_test} - return result - def test_step__multiple_dataloaders(self, batch, batch_idx, dataloader_idx, **kwargs): """ Default, baseline test_step diff --git a/tests/base/model_train_steps.py b/tests/base/model_train_steps.py index caec6db9aaa10..0590f5b7b5ccc 100644 --- a/tests/base/model_train_steps.py +++ b/tests/base/model_train_steps.py @@ -53,25 +53,6 @@ def training_step(self, batch, batch_idx, optimizer_idx=None): ) return output - def training_step_result_obj(self, batch, batch_idx, optimizer_idx=None): - # forward pass - x, y = batch - x = x.view(x.size(0), -1) - y_hat = self(x) - - # calculate loss - loss_val = self.loss(y, y_hat) - log_val = loss_val - - # alternate between tensors and scalars for "log" and "progress_bar" - if batch_idx % 2 == 0: - log_val = log_val.item() - - result = TrainResult(loss_val) - result.log('some_val', log_val * log_val, prog_bar=True, logger=False) - result.log('train_some_val', log_val * log_val) - return result - def training_step__inf_loss(self, batch, batch_idx, optimizer_idx=None): output = self.training_step(batch, batch_idx, optimizer_idx) if batch_idx == self.test_step_inf_loss: @@ -81,19 +62,6 @@ def training_step__inf_loss(self, batch, batch_idx, optimizer_idx=None): output /= 0 return output - def training_step_full_loop_result_obj_dp(self, batch, batch_idx, optimizer_idx=None): - """ - Full loop flow train step (result obj + dp) - """ - x, y = batch - x = x.view(x.size(0), -1) - y_hat = self(x.to(self.device)) - loss_val = y_hat.sum() - result = TrainResult(minimize=loss_val) - result.log('train_step_metric', loss_val + 1) - self.training_step_called = True - return result - def training_step_result_obj_dp(self, batch, batch_idx, optimizer_idx=None): # forward pass x, y = batch @@ -136,23 +104,6 @@ def training_epoch_end_full_loop_result_obj_dp(self, result): return result - def eval_step_full_loop_result_obj_dp(self, batch, batch_idx, optimizer_idx=None): - """ - Full loop flow train step (result obj + dp) - """ - x, y = batch - x = x.view(x.size(0), -1) - y_hat = self(x.to(self.device)) - loss_val = y_hat.sum() - result = EvalResult(checkpoint_on=loss_val, early_stop_on=loss_val) - - eval_name = 'validation' if not self.trainer.testing else 'test' - result.log(f'{eval_name}_step_metric', loss_val + 1, on_step=True) - - setattr(self, f'{eval_name}_step_called', True) - - return result - def eval_step_end_full_loop_result_obj_dp(self, result): """ Full loop flow train step (result obj + dp) @@ -198,20 +149,3 @@ def eval_epoch_end_full_loop_result_obj_dp(self, result): setattr(result, f'{eval_name}_step_metric', reduced) return result - - def training_step__using_metrics(self, batch, batch_idx, optimizer_idx=None): - """Lightning calls this inside the training loop""" - # forward pass - x, y = batch - x = x.view(x.size(0), -1) - y_hat = self(x) - - # calculate loss - loss_val = self.loss(y, y_hat) - - # call metric - val = self.metric(x, y) - - result = TrainResult(minimize=loss_val) - result.log('metric_val', val) - return result diff --git a/tests/base/model_valid_steps.py b/tests/base/model_valid_steps.py index e23e62dccdaba..a008a6cecf110 100644 --- a/tests/base/model_valid_steps.py +++ b/tests/base/model_valid_steps.py @@ -71,25 +71,6 @@ def validation_step_no_monitor(self, batch, batch_idx, *args, **kwargs): }) return output - def validation_step_result_obj(self, batch, batch_idx, *args, **kwargs): - x, y = batch - x = x.view(x.size(0), -1) - y_hat = self(x) - - loss_val = self.loss(y, y_hat) - - # acc - labels_hat = torch.argmax(y_hat, dim=1) - val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) - val_acc = torch.tensor(val_acc).type_as(x) - - result = EvalResult(checkpoint_on=loss_val, early_stop_on=loss_val) - result.log_dict({ - 'val_loss': loss_val, - 'val_acc': val_acc, - }) - return result - def validation_step_result_obj_dp(self, batch, batch_idx, *args, **kwargs): x, y = batch x = x.view(x.size(0), -1) diff --git a/tests/core/test_results.py b/tests/core/test_results.py index f4486ce6ae419..797004b7f21ff 100644 --- a/tests/core/test_results.py +++ b/tests/core/test_results.py @@ -18,7 +18,7 @@ import torch import torch.distributed as dist import torch.multiprocessing as mp -from pytorch_lightning import Trainer, seed_everything +from pytorch_lightning import Trainer from pytorch_lightning.core.step_result import Result, TrainResult, EvalResult import tests.base.develop_utils as tutils diff --git a/tests/models/test_tpu.py b/tests/models/test_tpu.py index e838dc60d81b3..37ab774bc8342 100644 --- a/tests/models/test_tpu.py +++ b/tests/models/test_tpu.py @@ -244,39 +244,6 @@ def test_distributed_backend_set_when_using_tpu(tmpdir, tpu_cores): assert Trainer(tpu_cores=tpu_cores).distributed_backend == "tpu" -@mock.patch.dict(os.environ, {"PL_DEV_DEBUG": "1"}) -@pytest.mark.skipif(not TPU_AVAILABLE, reason="test requires TPU machine") -@pl_multi_process_test -def test_result_obj_on_tpu(tmpdir): - seed_everything(1234) - - batches = 5 - epochs = 2 - - model = EvalModelTemplate() - model.training_step = model.training_step_result_obj - model.training_step_end = None - model.training_epoch_end = None - model.validation_step = model.validation_step_result_obj - model.validation_step_end = None - model.validation_epoch_end = None - model.test_step = model.test_step_result_obj - model.test_step_end = None - model.test_epoch_end = None - - trainer_options = dict( - default_root_dir=tmpdir, - max_epochs=epochs, - callbacks=[EarlyStopping()], - log_every_n_steps=2, - limit_train_batches=batches, - weights_summary=None, - tpu_cores=8 - ) - - tpipes.run_model_test(trainer_options, model, on_gpu=False, with_hpc=False) - - @pytest.mark.skipif(not TPU_AVAILABLE, reason="test requires TPU machine") @pl_multi_process_test def test_broadcast_on_tpu():