Skip core tests when not core testing (#1330)

.github/workflows/ci-testing.yml

@@ -52,6 +52,12 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
+    - name: Set Swap Space
+      if: runner.os == 'Linux'
+      uses: pierotofy/set-swap-space@master
+      with:
+        swap-size-gb: 10
+
     # Github Actions: Run step on specific OS: https://stackoverflow.com/a/57948488/4521646
     - name: Setup macOS
       if: runner.os == 'macOS'

flash/core/data/data_module.py

@@ -37,9 +37,14 @@
 from flash.core.data.splits import SplitDataset
 from flash.core.data.utils import _STAGES_PREFIX
 from flash.core.registry import FlashRegistry
+from flash.core.utilities.imports import _CORE_TESTING
 from flash.core.utilities.stages import RunningStage
 from flash.core.utilities.types import INPUT_TRANSFORM_TYPE
 
+# Skip doctests if requirements aren't available
+if not _CORE_TESTING:
+    __doctest_skip__ = ["DataModule"]
+
 
 class DatasetInput(Input):
     """The ``DatasetInput`` implements default behaviours for data sources which expect the input to

flash/core/data/utilities/classification.py

@@ -19,6 +19,11 @@
 import torch
 
 from flash.core.data.utilities.sort import sorted_alphanumeric
+from flash.core.utilities.imports import _CORE_TESTING
+
+# Skip doctests if requirements aren't available
+if not _CORE_TESTING:
+    __doctest_skip__ = ["*"]
 
 
 def _is_list_like(x: Any) -> bool:

flash/core/data/utils.py

@@ -23,9 +23,13 @@
 from pytorch_lightning.utilities.apply_func import apply_to_collection
 from tqdm.auto import tqdm as tq
 
-from flash.core.utilities.imports import _PIL_AVAILABLE, _TORCHVISION_AVAILABLE
+from flash.core.utilities.imports import _CORE_TESTING, _PIL_AVAILABLE, _TORCHVISION_AVAILABLE
 from flash.core.utilities.stages import RunningStage
 
+# Skip doctests if requirements aren't available
+if not _CORE_TESTING:
+    __doctest_skip__ = ["download_data"]
+
 if _PIL_AVAILABLE:
     from PIL.Image import Image
 else:

flash/core/model.py

@@ -49,7 +49,7 @@
 from flash.core.registry import FlashRegistry
 from flash.core.serve.composition import Composition
 from flash.core.utilities.apply_func import get_callable_dict
-from flash.core.utilities.imports import _PL_GREATER_EQUAL_1_5_0, requires
+from flash.core.utilities.imports import _CORE_TESTING, _PL_GREATER_EQUAL_1_5_0, requires
 from flash.core.utilities.providers import _HUGGINGFACE
 from flash.core.utilities.stages import RunningStage
 from flash.core.utilities.types import (
@@ -62,6 +62,10 @@
     OUTPUT_TRANSFORM_TYPE,
 )
 
+# Skip doctests if requirements aren't available
+if not _CORE_TESTING:
+    __doctest_skip__ = ["Task", "Task.*"]
+
 
 class ModuleWrapperBase:
     """The ``ModuleWrapperBase`` is a base for classes which wrap a ``LightningModule`` or an instance of

flash/core/optimizers/lamb.py

@@ -25,6 +25,12 @@
 from torch import nn
 from torch.optim.optimizer import Optimizer
 
+from flash.core.utilities.imports import _CORE_TESTING
+
+# Skip doctests if requirements aren't available
+if not _CORE_TESTING:
+    __doctest_skip__ = ["LAMB"]
+
 
 class LAMB(Optimizer):
     r"""Extends ADAM in pytorch to incorporate LAMB algorithm from the paper:

flash/core/optimizers/lars.py

@@ -22,6 +22,12 @@
 from torch import nn
 from torch.optim.optimizer import Optimizer, required
 
+from flash.core.utilities.imports import _CORE_TESTING
+
+# Skip doctests if requirements aren't available
+if not _CORE_TESTING:
+    __doctest_skip__ = ["LARS"]
+
 
 class LARS(Optimizer):
     r"""Extends SGD in PyTorch with LARS scaling from the paper

flash/core/optimizers/lr_scheduler.py

@@ -23,6 +23,12 @@
 from torch.optim import Adam, Optimizer
 from torch.optim.lr_scheduler import _LRScheduler
 
+from flash.core.utilities.imports import _CORE_TESTING
+
+# Skip doctests if requirements aren't available
+if not _CORE_TESTING:
+    __doctest_skip__ = ["LinearWarmupCosineAnnealingLR"]
+
 
 class LinearWarmupCosineAnnealingLR(_LRScheduler):
     """Sets the learning rate of each parameter group to follow a linear warmup schedule between warmup_start_lr

flash/core/serve/dag/optimization.py

@@ -5,6 +5,11 @@
 from flash.core.serve.dag.task import flatten, get, get_dependencies, ishashable, istask, reverse_dict, subs, toposort
 from flash.core.serve.dag.utils import key_split
 from flash.core.serve.dag.utils_test import add, inc, mul
+from flash.core.utilities.imports import _SERVE_TESTING
+
+# Skip doctests if requirements aren't available
+if not _SERVE_TESTING:
+    __doctest_skip__ = ["*"]
 
 
 def cull(dsk, keys):

flash/core/serve/dag/order.py

@@ -81,6 +81,11 @@
 
 from flash.core.serve.dag.task import get_dependencies, get_deps, getcycle, reverse_dict
 from flash.core.serve.dag.utils_test import add, inc
+from flash.core.utilities.imports import _SERVE_TESTING
+
+# Skip doctests if requirements aren't available
+if not _SERVE_TESTING:
+    __doctest_skip__ = ["*"]
 
 
 def order(dsk, dependencies=None):

flash/core/serve/dag/rewrite.py

@@ -1,6 +1,11 @@
 from collections import deque
 
 from flash.core.serve.dag.task import istask, subs
+from flash.core.utilities.imports import _SERVE_TESTING
+
+# Skip doctests if requirements aren't available
+if not _SERVE_TESTING:
+    __doctest_skip__ = ["*"]
 
 
 def head(task):

flash/core/serve/dag/task.py

@@ -2,6 +2,11 @@
 from typing import List, Sequence
 
 from flash.core.serve.dag.utils_test import add, inc
+from flash.core.utilities.imports import _SERVE_TESTING
+
+# Skip doctests if requirements aren't available
+if not _SERVE_TESTING:
+    __doctest_skip__ = ["*"]
 
 no_default = "__no_default__"
 

flash/core/serve/dag/utils.py

@@ -6,6 +6,12 @@
 import re
 from operator import methodcaller
 
+from flash.core.utilities.imports import _SERVE_TESTING
+
+# Skip doctests if requirements aren't available
+if not _SERVE_TESTING:
+    __doctest_skip__ = ["*"]
+
 
 def funcname(func):
     """Get the name of a function."""

flash/core/serve/decorators.py

@@ -8,7 +8,11 @@
 from flash.core.serve.core import Connection, make_param_dict, make_parameter_container, ParameterContainer, Servable
 from flash.core.serve.types.base import BaseType
 from flash.core.serve.utils import fn_outputs_to_keyed_map
-from flash.core.utilities.imports import _CYTOOLZ_AVAILABLE
+from flash.core.utilities.imports import _CYTOOLZ_AVAILABLE, _SERVE_TESTING
+
+# Skip doctests if requirements aren't available
+if not _SERVE_TESTING:
+    __doctest_skip__ = ["*"]
 
 if _CYTOOLZ_AVAILABLE:
     from cytoolz import compose

flash/core/serve/interfaces/models.py

@@ -3,7 +3,11 @@
 from flash.core.serve.component import ModelComponent
 from flash.core.serve.core import Endpoint
 from flash.core.serve.types import Repeated
-from flash.core.utilities.imports import _PYDANTIC_AVAILABLE
+from flash.core.utilities.imports import _PYDANTIC_AVAILABLE, _SERVE_TESTING
+
+# Skip doctests if requirements aren't available
+if not _SERVE_TESTING:
+    __doctest_skip__ = ["EndpointProtocol.*"]
 
 if _PYDANTIC_AVAILABLE:
     from pydantic import BaseModel, create_model

flash/core/serve/utils.py

@@ -1,4 +1,3 @@
-from importlib.util import find_spec
 from pathlib import Path
 from typing import Any, Dict, Optional
 
@@ -46,21 +45,3 @@ def download_file(url: str, *, download_path: Optional[Path] = None) -> str:
                 f.write(chunk)
 
     return fpath
-
-
-def _module_available(module_path: str) -> bool:
-    """Check if a path is available in your environment.
-
-    >>> _module_available('os')
-    True
-    >>> _module_available('bla.bla')
-    False
-    """
-    try:
-        return find_spec(module_path) is not None
-    except AttributeError:
-        # Python 3.6
-        return False
-    except ModuleNotFoundError:
-        # Python 3.7+
-        return False

flash/core/utilities/imports.py

@@ -275,6 +275,7 @@ def _import_module(self):
 
 
 # Global variables used for testing purposes (e.g. to only run doctests in the correct CI job)
+_CORE_TESTING = True
 _IMAGE_TESTING = _IMAGE_AVAILABLE
 _IMAGE_EXTRAS_TESTING = False  # Not for normal use
 _VIDEO_TESTING = _VIDEO_AVAILABLE
@@ -288,6 +289,7 @@ def _import_module(self):
 
 if "FLASH_TEST_TOPIC" in os.environ:
     topic = os.environ["FLASH_TEST_TOPIC"]
+    _CORE_TESTING = topic == "core"
     _IMAGE_TESTING = topic == "image"
     _IMAGE_EXTRAS_TESTING = topic == "image,image_extras" or topic == "icevision" or topic == "vissl"
     _VIDEO_TESTING = topic == "video"

flash/core/utilities/stability.py

@@ -17,6 +17,12 @@
 
 from pytorch_lightning.utilities import rank_zero_warn
 
+from flash.core.utilities.imports import _CORE_TESTING
+
+# Skip doctests if requirements aren't available
+if not _CORE_TESTING:
+    __doctest_skip__ = ["beta"]
+
 
 @functools.lru_cache()  # Trick to only warn once for each message
 def _raise_beta_warning(message: str, stacklevel: int = 6):

flash_examples/question_answering.py

@@ -13,22 +13,15 @@
 # limitations under the License.
 from flash import Trainer
 from flash.core.data.utils import download_data
-from flash.core.utilities.imports import example_requires
 from flash.text import QuestionAnsweringData, QuestionAnsweringTask
 
-example_requires("text")
-
-import nltk  # noqa: E402
-
-nltk.download("punkt")
-
 # 1. Create the DataModule
 download_data("https://pl-flash-data.s3.amazonaws.com/squad_tiny.zip", "./data/")
 
 datamodule = QuestionAnsweringData.from_squad_v2(
     train_file="./data/squad_tiny/train.json",
     val_file="./data/squad_tiny/val.json",
-    batch_size=4,
+    batch_size=1,
 )
 
 # 2. Build the task

tests/core/data/io/test_input.py

@@ -15,9 +15,11 @@
 from pytorch_lightning.utilities.exceptions import MisconfigurationException
 
 from flash.core.data.io.input import Input, IterableInput, ServeInput
+from flash.core.utilities.imports import _CORE_TESTING
 from flash.core.utilities.stages import RunningStage
 
 
+@pytest.mark.skipif(not _CORE_TESTING, reason="Not testing core.")
 def test_input_validation():
     with pytest.raises(RuntimeError, match="Use `IterableInput` instead."):
 
@@ -38,6 +40,7 @@ def __init__(self, *args, **kwargs):
     ValidInput(RunningStage.TRAINING)
 
 
+@pytest.mark.skipif(not _CORE_TESTING, reason="Not testing core.")
 def test_iterable_input_validation():
     with pytest.raises(RuntimeError, match="Use `Input` instead."):
 
@@ -58,6 +61,7 @@ def __init__(self, *args, **kwargs):
     ValidIterableInput(RunningStage.TRAINING)
 
 
+@pytest.mark.skipif(not _CORE_TESTING, reason="Not testing core.")
 def test_serve_input():
 
     server_input = ServeInput()

tests/core/data/io/test_output.py

@@ -13,9 +13,13 @@
 # limitations under the License.
 from unittest.mock import Mock
 
+import pytest
+
 from flash.core.data.io.output import Output
+from flash.core.utilities.imports import _CORE_TESTING
 
 
+@pytest.mark.skipif(not _CORE_TESTING, reason="Not testing core.")
 def test_output():
     """Tests basic ``Output`` methods."""
     my_output = Output()

tests/core/data/io/test_output_transform.py

@@ -11,11 +11,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import pytest
 import torch
 
 from flash.core.data.io.output_transform import OutputTransform
+from flash.core.utilities.imports import _CORE_TESTING
 
 
+@pytest.mark.skipif(not _CORE_TESTING, reason="Not testing core.")
 def test_output_transform():
     class CustomOutputTransform(OutputTransform):
         @staticmethod

tests/core/data/test_batch.py

@@ -17,6 +17,7 @@
 import torch
 
 from flash.core.data.batch import default_uncollate
+from flash.core.utilities.imports import _CORE_TESTING
 
 Case = namedtuple("Case", ["collated_batch", "uncollated_batch"])
 
@@ -46,6 +47,7 @@
 ]
 
 
+@pytest.mark.skipif(not _CORE_TESTING, reason="Not testing core.")
 @pytest.mark.parametrize("case", cases)
 def test_default_uncollate(case):
     assert default_uncollate(case.collated_batch) == case.uncollated_batch
@@ -60,6 +62,7 @@ def test_default_uncollate(case):
 ]
 
 
+@pytest.mark.skipif(not _CORE_TESTING, reason="Not testing core.")
 @pytest.mark.parametrize("error_case", error_cases)
 def test_default_uncollate_raises(error_case):
     with pytest.raises(ValueError, match=error_case.match):

tests/core/data/test_callback.py

@@ -13,16 +13,19 @@
 # limitations under the License.
 from unittest import mock
 
+import pytest
 import torch
 
 from flash import DataKeys
 from flash.core.data.data_module import DataModule, DatasetInput
 from flash.core.data.io.input_transform import InputTransform
 from flash.core.model import Task
 from flash.core.trainer import Trainer
+from flash.core.utilities.imports import _CORE_TESTING
 from flash.core.utilities.stages import RunningStage
 
 
+@pytest.mark.skipif(not _CORE_TESTING, reason="Not testing core.")
 @mock.patch("pickle.dumps")  # need to mock pickle or we get pickle error
 @mock.patch("torch.save")  # need to mock torch.save, or we get pickle error
 def test_flash_callback(_, __, tmpdir):