Merge pull request #1006 from openvinotoolkit/ik/move_cli_tests

Ik/move cli tests
openvinotoolkit · Mar 25, 2022 · 0d4ce2c · 0d4ce2c
2 parents 83e96b2 + 7301d0d
commit 0d4ce2c
Show file tree

Hide file tree

Showing 20 changed files with 723 additions and 524 deletions.
diff --git a/QUICK_START_GUIDE.md b/QUICK_START_GUIDE.md
@@ -14,17 +14,13 @@
     git checkout -b develop origin/develop
     git submodule update --init --recursive
     ```
-2. Export `OTE_SDK_PATH` environment variable to use it inside our scripts:
-   ```
-   export OTE_SDK_PATH=`pwd`/ote_sdk
-   ```
 
-3. Install prerequisites by running the following:
+2. Install prerequisites by running the following:
    ```
    sudo apt-get install python3-pip python3-venv
    ```
 
-4. Search for available scripts that create python virtual environments for different task types:
+3. Search for available scripts that create python virtual environments for different task types:
    ```bash
    find external/ -name init_venv.sh
    ```
@@ -38,15 +34,15 @@
    Each line in the output gives an `init_venv.sh` script that creates a virtual environment
    for the corresponding task type.
 
-5. Let's choose a task type.
+4. Let's choose a task type.
    Let it be `external/mmdetection` for Object Detection task.
    ```bash
    TASK_ALGO_DIR=./external/mmdetection/
    ```
    Note that we will not use the variable `TASK_ALGO_DIR` inside our scripts, we set it just to
    simplify this guide.
 
-6. Let's create, activate virtual environment for the chosen task, and install `ote_cli`.
+5. Let's create, activate virtual environment for the chosen task, and install `ote_cli`.
    Note that the virtual environment folder may be created in any place in your system,
    but we will create it in the folder `./cur_task_venv` for convenience.
    ```bash
@@ -62,7 +58,7 @@
    from the chosen task folder is used to avoid breaking constraints
    for the OTE task.
 
-7. As soon as `ote_cli` is installed in the virtual environment, you can use
+6. As soon as `ote_cli` is installed in the virtual environment, you can use
    `ote` command line interface described below to run
    train/eval/export/other action for templates related to the chosen task type.
 

diff --git a/external/anomaly/init_venv.sh b/external/anomaly/init_venv.sh
@@ -26,11 +26,6 @@ if [[ $PYTHON_VERSION != "3.8" && $PYTHON_VERSION != "3.9" ]]; then
   exit 1
 fi
 
-if [[ -z $OTE_SDK_PATH ]]; then
-  echo "The environment variable OTE_SDK_PATH is not set -- it is required for creating virtual environment"
-  exit 1
-fi
-
 cd ${work_dir}
 
 if [[ -e ${venv_dir} ]]; then
@@ -109,10 +104,11 @@ else
   echo torchvision==${TORCHVISION_VERSION}+cu${CUDA_VERSION_CODE} >> ${CONSTRAINTS_FILE}
 fi
 
-pip install -r requirements.txt
-pip install -e .
+pip install -r requirements.txt || exit 1
+pip install -e . || exit 1
 
-pip install -e $OTE_SDK_PATH  || exit 1
+# Install OTE SDK
+pip install -e ../../ote_sdk/ || exit 1
 
 deactivate
 

diff --git a/...t_ote_cli_tools_anomaly_classification.py → ...ts/ote_cli/test_anomaly_classification.py b/...t_ote_cli_tools_anomaly_classification.py → ...ts/ote_cli/test_anomaly_classification.py
@@ -20,7 +20,7 @@
 from ote_sdk.test_suite.e2e_test_system import e2e_pytest_component
 
 from ote_cli.registry import Registry
-from common import (
+from ote_cli.utils.tests import (
     create_venv,
     get_some_vars,
     ote_demo_deployment_testing,
@@ -42,28 +42,28 @@
 
 
 args = {
-    '--train-ann-file': 'data/anomaly/classification/train.json',
-    '--train-data-roots': 'data/anomaly/shapes',
-    '--val-ann-file': 'data/anomaly/classification/val.json',
-    '--val-data-roots': 'data/anomaly/shapes',
-    '--test-ann-files': 'data/anomaly/classification/test.json',
-    '--test-data-roots': 'data/anomaly/shapes',
-    '--input': 'data/anomaly/shapes/test/hexagon',
-    'train_params': [],
+    "--train-ann-file": "data/anomaly/classification/train.json",
+    "--train-data-roots": "data/anomaly/shapes",
+    "--val-ann-file": "data/anomaly/classification/val.json",
+    "--val-data-roots": "data/anomaly/shapes",
+    "--test-ann-files": "data/anomaly/classification/test.json",
+    "--test-data-roots": "data/anomaly/shapes",
+    "--input": "data/anomaly/shapes/test/hexagon",
+    "train_params": [],
 }
 
-root = '/tmp/ote_cli/'
+root = "/tmp/ote_cli/"
 ote_dir = os.getcwd()
 
-templates = Registry('external').filter(task_type='ANOMALY_CLASSIFICATION').templates
+templates = Registry("external").filter(task_type="ANOMALY_CLASSIFICATION").templates
 templates_ids = [template.model_template_id for template in templates]
 
 
 class TestToolsAnomalyClassification:
     @e2e_pytest_component
     def test_create_venv(self):
-        work_dir, template_work_dir, algo_backend_dir = get_some_vars(templates[0], root)
-        create_venv(algo_backend_dir, work_dir, template_work_dir)
+        work_dir, _, algo_backend_dir = get_some_vars(templates[0], root)
+        create_venv(algo_backend_dir, work_dir)
 
     @e2e_pytest_component
     @pytest.mark.parametrize("template", templates, ids=templates_ids)

diff --git a/...y/test_ote_cli_tools_anomaly_detection.py → ...y/tests/ote_cli/test_anomaly_detection.py b/...y/test_ote_cli_tools_anomaly_detection.py → ...y/tests/ote_cli/test_anomaly_detection.py
@@ -17,7 +17,7 @@
 import os
 
 import pytest
-from common import (
+from ote_cli.utils.tests import (
     create_venv,
     get_some_vars,
     nncf_eval_openvino_testing,
@@ -41,28 +41,28 @@
 from ote_cli.registry import Registry
 
 args = {
-    '--train-ann-file': 'data/anomaly/detection/train.json',
-    '--train-data-roots': 'data/anomaly/shapes',
-    '--val-ann-file': 'data/anomaly/detection/val.json',
-    '--val-data-roots': 'data/anomaly/shapes',
-    '--test-ann-files': 'data/anomaly/detection/test.json',
-    '--test-data-roots': 'data/anomaly/shapes',
-    '--input': 'data/anomaly/shapes/test/hexagon',
-    'train_params': [],
+    "--train-ann-file": "data/anomaly/detection/train.json",
+    "--train-data-roots": "data/anomaly/shapes",
+    "--val-ann-file": "data/anomaly/detection/val.json",
+    "--val-data-roots": "data/anomaly/shapes",
+    "--test-ann-files": "data/anomaly/detection/test.json",
+    "--test-data-roots": "data/anomaly/shapes",
+    "--input": "data/anomaly/shapes/test/hexagon",
+    "train_params": [],
 }
 
-root = '/tmp/ote_cli/'
+root = "/tmp/ote_cli/"
 ote_dir = os.getcwd()
 
-templates = Registry('external').filter(task_type='ANOMALY_DETECTION').templates
+templates = Registry("external").filter(task_type="ANOMALY_DETECTION").templates
 templates_ids = [template.model_template_id for template in templates]
 
 
 class TestToolsAnomalyDetection:
     @e2e_pytest_component
     def test_create_venv(self):
-        work_dir, template_work_dir, algo_backend_dir = get_some_vars(templates[0], root)
-        create_venv(algo_backend_dir, work_dir, template_work_dir)
+        work_dir, _, algo_backend_dir = get_some_vars(templates[0], root)
+        create_venv(algo_backend_dir, work_dir)
 
     @e2e_pytest_component
     @pytest.mark.parametrize("template", templates, ids=templates_ids)

diff --git a/...est_ote_cli_tools_anomaly_segmentation.py → ...ests/ote_cli/test_anomaly_segmentation.py b/...est_ote_cli_tools_anomaly_segmentation.py → ...ests/ote_cli/test_anomaly_segmentation.py
@@ -20,7 +20,7 @@
 from ote_sdk.test_suite.e2e_test_system import e2e_pytest_component
 
 from ote_cli.registry import Registry
-from common import (
+from ote_cli.utils.tests import (
     create_venv,
     get_some_vars,
     ote_demo_deployment_testing,
@@ -42,28 +42,28 @@
 
 
 args = {
-    '--train-ann-file': 'data/anomaly/segmentation/train.json',
-    '--train-data-roots': 'data/anomaly/shapes',
-    '--val-ann-file': 'data/anomaly/segmentation/val.json',
-    '--val-data-roots': 'data/anomaly/shapes',
-    '--test-ann-files': 'data/anomaly/segmentation/test.json',
-    '--test-data-roots': 'data/anomaly/shapes',
-    '--input': 'data/anomaly/shapes/test/hexagon',
-    'train_params': [],
+    "--train-ann-file": "data/anomaly/segmentation/train.json",
+    "--train-data-roots": "data/anomaly/shapes",
+    "--val-ann-file": "data/anomaly/segmentation/val.json",
+    "--val-data-roots": "data/anomaly/shapes",
+    "--test-ann-files": "data/anomaly/segmentation/test.json",
+    "--test-data-roots": "data/anomaly/shapes",
+    "--input": "data/anomaly/shapes/test/hexagon",
+    "train_params": [],
 }
 
-root = '/tmp/ote_cli/'
+root = "/tmp/ote_cli/"
 ote_dir = os.getcwd()
 
-templates = Registry('external').filter(task_type='ANOMALY_SEGMENTATION').templates
+templates = Registry("external").filter(task_type="ANOMALY_SEGMENTATION").templates
 templates_ids = [template.model_template_id for template in templates]
 
 
 class TestToolsAnomalySegmentation:
     @e2e_pytest_component
     def test_create_venv(self):
-        work_dir, template_work_dir, algo_backend_dir = get_some_vars(templates[0], root)
-        create_venv(algo_backend_dir, work_dir, template_work_dir)
+        work_dir, _, algo_backend_dir = get_some_vars(templates[0], root)
+        create_venv(algo_backend_dir, work_dir)
 
     @e2e_pytest_component
     @pytest.mark.parametrize("template", templates, ids=templates_ids)

diff --git a/external/deep-object-reid/tests/conftest.py b/external/deep-object-reid/tests/conftest.py
@@ -5,15 +5,14 @@
     from e2e.conftest_utils import pytest_addoption as _e2e_pytest_addoption # noqa
     from e2e import config # noqa
     from e2e.utils import get_plugins_from_packages
-    from ote_sdk.test_suite.pytest_insertions import *
-    from ote_sdk.test_suite.training_tests_common import REALLIFE_USECASE_CONSTANT
     pytest_plugins = get_plugins_from_packages([e2e])
 except ImportError:
     _e2e_pytest_addoption = None
     pass
-
 import config
-
+import pytest
+from ote_sdk.test_suite.pytest_insertions import *
+from ote_sdk.test_suite.training_tests_common import REALLIFE_USECASE_CONSTANT
 
 pytest_plugins = get_pytest_plugins_from_ote()
 

diff --git a/...reid/test_ote_cli_tools_classification.py → ...reid/tests/ote_cli/test_classification.py b/...reid/test_ote_cli_tools_classification.py → ...reid/tests/ote_cli/test_classification.py
@@ -20,7 +20,7 @@
 from ote_sdk.test_suite.e2e_test_system import e2e_pytest_component
 
 from ote_cli.registry import Registry
-from common import (
+from ote_cli.utils.tests import (
     create_venv,
     get_some_vars,
     ote_demo_deployment_testing,
@@ -69,8 +69,8 @@
 class TestToolsClassification:
     @e2e_pytest_component
     def test_create_venv(self):
-        work_dir, template_work_dir, algo_backend_dir = get_some_vars(templates[0], root)
-        create_venv(algo_backend_dir, work_dir, template_work_dir)
+        work_dir, _, algo_backend_dir = get_some_vars(templates[0], root)
+        create_venv(algo_backend_dir, work_dir)
 
     @e2e_pytest_component
     @pytest.mark.parametrize("template", templates, ids=templates_ids)

diff --git a/...detection/test_ote_cli_tools_detection.py → ...detection/tests/ote_cli/test_detection.py b/...detection/test_ote_cli_tools_detection.py → ...detection/tests/ote_cli/test_detection.py
@@ -21,7 +21,7 @@
 from ote_sdk.test_suite.e2e_test_system import e2e_pytest_component
 
 from ote_cli.registry import Registry
-from common import (
+from ote_cli.utils.tests import (
     collect_env_vars,
     create_venv,
     get_some_vars,
@@ -71,8 +71,8 @@
 class TestToolsDetection:
     @e2e_pytest_component
     def test_create_venv(self):
-        work_dir, template_work_dir, algo_backend_dir = get_some_vars(templates[0], root)
-        create_venv(algo_backend_dir, work_dir, template_work_dir)
+        work_dir, _, algo_backend_dir = get_some_vars(templates[0], root)
+        create_venv(algo_backend_dir, work_dir)
 
     @e2e_pytest_component
     @pytest.mark.parametrize("template", templates, ids=templates_ids)
@@ -92,7 +92,7 @@ def test_ote_eval(self, template):
     @e2e_pytest_component
     @pytest.mark.parametrize("template", templates, ids=templates_ids)
     def test_ote_eval_openvino(self, template):
-        ote_eval_openvino_testing(template, root, ote_dir, args, threshold=0.1)
+        ote_eval_openvino_testing(template, root, ote_dir, args, threshold=0.2)
 
     @e2e_pytest_component
     @pytest.mark.parametrize("template", templates, ids=templates_ids)

diff --git a/...st_ote_cli_tools_instance_segmentation.py → ...sts/ote_cli/test_instance_segmentation.py b/...st_ote_cli_tools_instance_segmentation.py → ...sts/ote_cli/test_instance_segmentation.py
@@ -22,7 +22,7 @@
 
 from ote_cli.registry import Registry
 
-from common import (
+from ote_cli.utils.tests import (
     create_venv,
     get_some_vars,
     ote_demo_deployment_testing,
@@ -64,8 +64,8 @@
 class TestToolsInstanceSegmentation:
     @e2e_pytest_component
     def test_create_venv(self):
-        work_dir, template_work_dir, algo_backend_dir = get_some_vars(templates[0], root)
-        create_venv(algo_backend_dir, work_dir, template_work_dir)
+        work_dir, _, algo_backend_dir = get_some_vars(templates[0], root)
+        create_venv(algo_backend_dir, work_dir)
 
     @e2e_pytest_component
     @pytest.mark.parametrize("template", templates, ids=templates_ids)

diff --git a/...n/test_ote_cli_tools_rotated_detection.py → ...n/tests/ote_cli/test_rotated_detection.py b/...n/test_ote_cli_tools_rotated_detection.py → ...n/tests/ote_cli/test_rotated_detection.py
@@ -22,7 +22,7 @@
 
 from ote_cli.registry import Registry
 
-from common import (
+from ote_cli.utils.tests import (
     create_venv,
     get_some_vars,
     ote_demo_deployment_testing,
@@ -64,8 +64,8 @@
 class TestToolsRotatedDetection:
     @e2e_pytest_component
     def test_create_venv(self):
-        work_dir, template_work_dir, algo_backend_dir = get_some_vars(templates[0], root)
-        create_venv(algo_backend_dir, work_dir, template_work_dir)
+        work_dir, _, algo_backend_dir = get_some_vars(templates[0], root)
+        create_venv(algo_backend_dir, work_dir)
 
     @e2e_pytest_component
     @pytest.mark.parametrize("template", templates, ids=templates_ids)

diff --git a/...tation/test_ote_cli_tools_segmentation.py → ...tation/tests/ote_cli/test_segmentation.py b/...tation/test_ote_cli_tools_segmentation.py → ...tation/tests/ote_cli/test_segmentation.py
@@ -22,7 +22,7 @@
 
 from ote_cli.registry import Registry
 
-from common import (
+from ote_cli.utils.tests import (
     create_venv,
     get_some_vars,
     ote_demo_deployment_testing,
@@ -75,8 +75,8 @@
 class TestToolsSegmentation:
     @e2e_pytest_component
     def test_create_venv(self):
-        work_dir, template_work_dir, algo_backend_dir = get_some_vars(templates[0], root)
-        create_venv(algo_backend_dir, work_dir, template_work_dir)
+        work_dir, _, algo_backend_dir = get_some_vars(templates[0], root)
+        create_venv(algo_backend_dir, work_dir)
 
     @e2e_pytest_component
     @pytest.mark.parametrize("template", templates, ids=templates_ids)