Windows tests

.github/workflows/windows-tests.yml

@@ -0,0 +1,38 @@
+name: Tests Windows
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        python-version: [3.8]  # Test only one Python Version for Windows
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install python dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install numpy scipy Cython
+        pip install flake8 pytest pytest-cov codecov
+        pip install --editable .[all]
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax error or undefined names
+        #flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        #flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Test with pytest
+      run: |
+        pytest -v "tests/" "padertorch/"

padertorch/configurable.py

@@ -1131,7 +1131,8 @@ class instead of the corresponding string.
     >>> cfg = {'inplace': False, 'negative_slope': 0.01, 'partial': torch.nn.LeakyReLU}
     >>> recursive_class_to_str(cfg, sort=True)
     {'partial': 'torch.nn.modules.activation.LeakyReLU', 'negative_slope': 0.01, 'inplace': False}
-    >>> recursive_class_to_str(Path('/pathlib/Path/object'), sort=True)
+    >>> import os
+    >>> recursive_class_to_str(Path('/pathlib/Path/object'), sort=True).replace(os.sep, '/')
     '/pathlib/Path/object'
 
     """
@@ -1664,7 +1665,7 @@ def normalize(cls, dictionary):
           'partial': 'torch.nn.Linear'},
          'storage_dir': 'abc'}
         """
-        if isinstance(dictionary, collections.Mapping):
+        if isinstance(dictionary, collections.abc.Mapping):
             special_key = _get_special_key(dictionary)
             if special_key:
                 dictionary[special_key] = cls._force_factory_type(

padertorch/io.py

@@ -71,10 +71,12 @@ def get_new_storage_dir(
     >>> import tempfile
     >>> with tempfile.TemporaryDirectory() as tmp_dir:
     ...     os.environ['STORAGE_ROOT'] = tmp_dir  # simulate enviroment variable for doctest
-    ...     print(get_new_storage_dir('fance_nn_experiment').relative_to(tmp_dir))
-    ...     print(get_new_storage_dir('fance_nn_experiment').relative_to(tmp_dir))
-    fance_nn_experiment/1
-    fance_nn_experiment/2
+    ...     # on Windows the temporary directory may be shortened, resulting in an error in relative_to
+    ...     tmp_dir_path = Path(tmp_dir).resolve()
+    ...     print(get_new_storage_dir('fancy_nn_experiment').relative_to(tmp_dir_path).as_posix())
+    ...     print(get_new_storage_dir('fancy_nn_experiment').relative_to(tmp_dir_path).as_posix())
+    fancy_nn_experiment/1
+    fancy_nn_experiment/2
 
     """
     basedir = Path(os.environ['STORAGE_ROOT']) / experiment_name

padertorch/modules/dual_path_rnn.py

@@ -146,7 +146,7 @@ def segment(
     if sequence_lengths is not None:
         sequence_lengths = sequence_lengths + 2 * padding
         sequence_lengths = (sequence_lengths - padding)
-        sequence_lengths = (sequence_lengths - 1) // hop_size + 1
+        sequence_lengths = torch.div(sequence_lengths - 1, hop_size, rounding_mode='floor')  + 1
     return segmented, sequence_lengths
 
 

padertorch/ops/losses/source_separation.py

@@ -185,8 +185,8 @@ def compute_pairwise_losses(
         >>> T, K, F = 4, 2, 5
         >>> estimate = torch.stack([torch.ones(F, T), torch.zeros(F, T)])
         >>> target = estimate[(1, 0), :, :]
-        >>> pit_loss_from_loss_matrix(compute_pairwise_losses(estimate, target, axis=0), return_permutation=True)
-        (tensor(0.), array([1, 0]))
+        >>> pit_loss_from_loss_matrix(compute_pairwise_losses(estimate, target, axis=0), return_permutation=True) # doctest: +ELLIPSIS
+        (tensor(0.), array([1, 0]...))
 
         >>> K = 5
         >>> estimate, target = torch.ones(K), torch.zeros(K)

padertorch/testing/windows.py

@@ -0,0 +1,11 @@
+import sys
+import pytest
+
+def skip_on_windows():
+    """Skip test if the current platform is Windows.
+
+    This is necessary in the tests for the training.
+    """
+    if sys.platform.startswith('win'):
+        pytest.skip('this doctest does not work on Windows, '
+                    'training is not possible on Windows because file symlinks are unavailable for non-admin users')

padertorch/utils.py

@@ -47,8 +47,8 @@ def to_list(x, length=None):
     >>> to_list('ab')
     ['ab']
     >>> from pathlib import Path
-    >>> to_list(Path('/foo/bar'))
-    [PosixPath('/foo/bar')]
+    >>> to_list(Path('/foo/bar')) # doctest: +ELLIPSIS
+    [...Path('/foo/bar')]
     """
     # Important cases (change type):
     #  - generator -> list

tests/test_ops/test_losses.py

@@ -163,7 +163,7 @@ def test_against_multivariate_multivariate(self):
         q = MultivariateNormal(
             loc=torch.randn((B, 1, D)),
             scale_tril=torch.Tensor(
-                np.broadcast_to(np.diag(np.random.rand(D)), (B, 1, D, D))
+                np.broadcast_to(np.diag(np.random.rand(D)), (B, 1, D, D)).copy()
             )
         )
         q_ = Normal(loc=q.loc[:, 0], scale=pt.ops.losses.kl_divergence._batch_diag(q.scale_tril[:, 0]))

tests/test_train/test_hooks.py

@@ -1,3 +1,4 @@
+import sys
 import types
 import tempfile
 from pathlib import Path
@@ -13,6 +14,8 @@
 import padertorch as pt
 import paderbox as pb
 
+from padertorch.testing.windows import skip_on_windows
+
 
 class ProgresbarHookTest(unittest.TestCase):
     num_epochs = 2
@@ -86,6 +89,7 @@ def train_loop_epoch(self, length, max_it_len):
 
 
 def test_summary_hook():
+    skip_on_windows()
 
     with tempfile.TemporaryDirectory() as tmp_dir:
         tmp_dir = Path(tmp_dir)
@@ -180,6 +184,8 @@ class PytorchOptimizer:
 
 
 def test_summary_hook_fail_duplicate_key():
+    skip_on_windows()
+
     hook = pt.train.hooks.SummaryHook((1, 'iteration'))
 
     hook.update_summary({
@@ -244,6 +250,7 @@ def review(self, inputs, outputs):
 
 
 def test_summary_hook_create_snapshot_flag():
+    skip_on_windows()
     class Model(DummyModel):
 
         def __init__(self, validation_losses, exp_dir, optimizer):
@@ -267,6 +274,7 @@ def review(self, example, output):
 
 
 def test_validation_hook_create_snapshot_flag():
+    skip_on_windows()
     class Model(DummyModel):
 
         def __init__(self, validation_losses, exp_dir, optimizer):
@@ -298,6 +306,8 @@ def review(self, example, output):
 
 
 def test_validation_hook_modify_summary_training_flag():
+    skip_on_windows()
+
     class Model(DummyModel):
         def review(self, example, output):
             summary = super().review(example, output)
@@ -328,6 +338,8 @@ def modify_summary(self, summary):
 
 
 def test_backoff():
+    skip_on_windows()
+
     ds = [0]
     with tempfile.TemporaryDirectory() as tmp_dir:
         optimizer = pt.optimizer.Adam()
@@ -356,6 +368,7 @@ def test_backoff():
 
 
 def test_loss_weight_annealing_hook():
+    skip_on_windows()
     class DummyTrainer:
         epoch = 0
         iteration = 0
@@ -375,6 +388,7 @@ class DummyTrainer:
 
 
 def test_model_attribute_annealing_hook():
+    skip_on_windows()
 
     class DummyTrainer:
         epoch = 0
@@ -398,6 +412,7 @@ class DummyModel:
 
 
 def test_lr_annealing_hook():
+    skip_on_windows()
 
     class DummyTrainer:
         epoch = 0
@@ -423,6 +438,7 @@ class PytorchOptimizer:
 
 
 def test_LRSchedulerHook():
+    skip_on_windows()
     class DummyLRScheduler:
         def __init__(self):
             self.calls_iteration = []

tests/test_train/test_runtime_tests.py

@@ -1,3 +1,5 @@
+import sys
+
 import tempfile
 from pathlib import Path
 import contextlib
@@ -52,6 +54,9 @@ def get_datasets():
 
 
 def test_single_model():
+    if sys.platform.startswith('win'):
+        pytest.skip('this doctest does not work on Windows, '
+                    'training is not possible on Windows due to symlinks being unavailable')
     tr_dataset, dt_dataset = get_datasets()
 
     config = pt.Trainer.get_config(
@@ -87,6 +92,9 @@ def assert_dir_unchanged_after_context(tmp_dir):
 
 
 def test_single_model_dir_unchanged():
+    if sys.platform.startswith('win'):
+        pytest.skip('this doctest does not work on Windows, '
+                    'training is not possible on Windows due to symlinks being unavailable')
     tr_dataset, dt_dataset = get_datasets()
     model = Model()
 
@@ -101,6 +109,9 @@ def test_single_model_dir_unchanged():
 
 
 def test_single_model_with_back_off_validation():
+    if sys.platform.startswith('win'):
+        pytest.skip('this doctest does not work on Windows, '
+                    'training is not possible on Windows due to symlinks being unavailable')
     tr_dataset, dt_dataset = get_datasets()
     model = Model()
 
@@ -170,6 +181,9 @@ def review(self, inputs, output):
 
 
 def test_single_grad_check():
+    if sys.platform.startswith('win'):
+        pytest.skip('this doctest does not work on Windows, '
+                    'training is not possible on Windows due to symlinks being unavailable')
     tr_dataset, dt_dataset = get_datasets()
 
     with tempfile.TemporaryDirectory() as tmp_dir:
@@ -193,6 +207,9 @@ def test_single_grad_check():
 
 
 def test_single_virtual_minibatch():
+    if sys.platform.startswith('win'):
+        pytest.skip('this doctest does not work on Windows, '
+                    'training is not possible on Windows due to symlinks being unavailable')
     tr_dataset, dt_dataset = get_datasets()
     model = Model()
 
@@ -238,6 +255,9 @@ def review(self, inputs, outputs):
 
 
 def test_multiple_optimizers():
+    if sys.platform.startswith('win'):
+        pytest.skip('this doctest does not work on Windows, '
+                    'training is not possible on Windows due to symlinks being unavailable')
     tr_dataset, dataset_dt = get_datasets()
 
     model = AE()

tests/test_train/test_trainer.py

@@ -1,4 +1,5 @@
 import os
+import sys
 import tempfile
 from pathlib import Path
 import inspect
@@ -24,6 +25,8 @@
 import padertorch as pt
 import paderbox as pb
 
+from padertorch.testing.windows import skip_on_windows
+
 
 class Model(pt.Model):
 
@@ -122,6 +125,8 @@ def set_last(self, iteration, epoch):
 
 
 def test_single_model():
+    skip_on_windows()
+
     tr_dataset, dt_dataset = get_dataset()
     tr_dataset = tr_dataset[:2]
     dt_dataset = dt_dataset[:2]
@@ -476,6 +481,7 @@ def test_single_model():
 
 
 def test_virtual_minibatch_few__examples():
+    skip_on_windows()
     test_virtual_minibatch(3, 1)
     test_virtual_minibatch(4, 1)
 
@@ -499,6 +505,8 @@ def test_virtual_minibatch(
     optimizer step, so the parameters are changed.
     """
 
+    skip_on_windows()
+
     it_tr, it_dt = get_dataset()
     it_tr = it_tr[:no_of_examples]
     it_dt = it_dt[:no_of_examples]
@@ -553,6 +561,8 @@ def test_virtual_minibatch(
 
 
 def test_released_tensors():
+    skip_on_windows()
+
     import gc
     gc.collect()
 
@@ -684,6 +694,8 @@ def post_step(
 
 
 def test_log_error_state():
+    skip_on_windows()
+
     with tempfile.TemporaryDirectory() as tmp_dir:
         t = pt.Trainer(
             Model(),