[WIP] ddp testing

tests/models/data/ddp/train_test_variations.py

@@ -0,0 +1,54 @@
+"""
+Runs several combinations of `.fit()` and `.test()` on a single node across multiple gpus.
+"""
+from argparse import ArgumentParser
+
+from pytorch_lightning import Trainer, seed_everything
+from tests.base import EvalModelTemplate
+
+
+def variation_fit_test(trainer, model):
+    trainer.fit(model)
+    trainer.test(model)
+
+
+def variation_test_fit(trainer, model):
+    trainer.test(model)
+    trainer.fit(model)
+
+
+def variation_test_test(trainer, model):
+    trainer.test(model)
+    trainer.test(model)
+
+
+def variation_test_fit_test(trainer, model):
+    trainer.test(model)
+    trainer.fit(model)
+    trainer.test(model)
+
+
+def get_variations():
+    variations = [v for v in globals() if v.startswith("variation")]
+    return variations
+
+
+def main():
+    seed_everything(1234)
+    parser = ArgumentParser(add_help=False)
+    parser = Trainer.add_argparse_args(parser)
+    parser.add_argument('--variation', default=variation_fit_test.__name__)
+    parser.set_defaults(gpus=2)
+    parser.set_defaults(distributed_backend="ddp")
+    args = parser.parse_args()
+
+    model = EvalModelTemplate()
+    trainer = Trainer.from_argparse_args(args)
+
+    # run the chosen variation
+    run_variation = globals()[args.variation]
+    run_variation(trainer, model)
+
+
+if __name__ == '__main__':
+    main()

tests/models/test_gpu.py

@@ -1,4 +1,8 @@
+import subprocess
+import sys
 from collections import namedtuple
+from pathlib import Path
+from unittest import mock
 
 import pytest
 import torch
@@ -11,6 +15,7 @@
 from pytorch_lightning.trainer.distrib_parts import _parse_gpu_ids, determine_root_gpu_device
 from pytorch_lightning.utilities.exceptions import MisconfigurationException
 from tests.base import EvalModelTemplate
+from tests.models.data.ddp import train_test_variations
 
 PRETEND_N_OF_GPUS = 16
 
@@ -93,6 +98,35 @@ def test_multi_gpu_model_dp(tmpdir):
     memory.get_memory_profile('min_max')
 
 
+@pytest.mark.parametrize('cli_args', [
+    pytest.param('--max_epochs 1 --gpus 2 --distributed_backend ddp'),
+])
+@pytest.mark.parametrize('variation', train_test_variations.get_variations())
+@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine")
+def test_multi_gpu_model_ddp(tmpdir, cli_args, variation):
+    file = Path(train_test_variations.__file__).absolute()
+    cli_args = cli_args.split(' ') if cli_args else []
+    cli_args += ['--default_root_dir', str(tmpdir)]
+    command = [sys.executable, file, '--variation', variation] + cli_args
+    p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    p.communicate()
+    # assert p.returncode == 0
+    std, err = p.communicate(timeout=60)
+    std = std.decode('utf-8').strip()
+    err = err.decode('utf-8').strip()
+    # assert std and not err
+    if p.returncode:
+        print(std)
+        print(err)
+        print(command)
+        pytest.fail(err)
+
+    # cli_args += ['--variation', variation]
+    # from tests.models.data.ddp.train_test_variations import main
+    # with mock.patch("argparse._sys.argv", ["any.py"] + cli_args):
+    #     main()
+
+
 @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine")
 def test_multi_gpu_model_ddp_spawn(tmpdir):
     tutils.set_random_master_port()