Refactor integration tests

azure-pipelines.yml

@@ -52,23 +52,23 @@ jobs:
     displayName: 'Run flake8 tests to find Python syntax errors and undefined names'
   - script: |
       cd test
-      source unittest.sh
+      source scripts/unittest.sh
     displayName: 'Unit test'
   - script: |
       cd test
-      python3 naive_test.py
+      python3 nni_test/nnitest/naive_test.py
     displayName: 'Naive test'
   - script: |
       cd test
-      python3 tuner_test.py
-    displayName: 'Built-in tuners / assessors tests'
+      python3 nni_test/nnitest/tuner_test.py
+    displayName: 'Built-in tuners / assessors with naive trial tests'
   - script: |
       cd test
-      python3 metrics_test.py
+      python3 nni_test/nnitest/run_tests.py --config config/integration_tests.yml --cases metrics
     displayName: 'Trial job metrics test'
   - script: |
       cd test
-      python3 cli_test.py
+      python3 nni_test/nnitest/run_tests.py --config config/integration_tests.yml --cases nnicli
     displayName: 'nnicli test'
   - script: |
       cd docs/en_US/
@@ -101,19 +101,19 @@ jobs:
     displayName: 'Install dependencies'
   - script: |
       cd test
-      source unittest.sh
+      source scripts/unittest.sh
     displayName: 'Unit test'
   - script: |
       cd test
-      python3 naive_test.py
+      python3 nni_test/nnitest/naive_test.py
     displayName: 'Naive test'
   - script: |
       cd test
-      python3 tuner_test.py
-    displayName: 'Built-in tuners / assessors tests'
+      python3 nni_test/nnitest/tuner_test.py
+    displayName: 'Built-in tuners / assessors with naive trial tests'
   - script: |
       cd test
-      python3 cli_test.py
+      python3 nni_test/nnitest/run_tests.py --config config/integration_tests.yml --cases nnicli
     displayName: 'nnicli test'
 
 - job: 'basic_test_pr_Windows'
@@ -137,13 +137,13 @@ jobs:
     displayName: 'Install dependencies'
   - script: |
       cd test
-      powershell.exe -file unittest.ps1
+      powershell.exe -file scripts/unittest.ps1
     displayName: 'unit test'
   - script: |
       cd test
-      python tuner_test.py
-    displayName: 'Built-in tuners / assessors tests'
+      python nni_test/nnitest/tuner_test.py
+    displayName: 'Built-in tuners / assessors with naive trial tests'
   - script: |
       cd test
-      PATH=$HOME/.local/bin:$PATH python3 cli_test.py
+      python nni_test/nnitest/run_tests.py --config config/integration_tests.yml --cases nnicli
     displayName: 'nnicli test'

examples/model_compress/model_prune_torch.py

@@ -67,7 +67,7 @@
         'dataset_name': 'cifar10',
         'model_name': 'vgg16',
         'pruner_class': ActivationMeanRankFilterPruner,
-        'configure_list': [{
+        'config_list': [{
             'sparsity': 0.5,
             'op_types': ['default'],
             'op_names': ['feature.0', 'feature.24', 'feature.27', 'feature.30', 'feature.34', 'feature.37']

test/config_test/tuners/mnist-curvefitting.test.yml → test/config/assessors/curvefitting.yml

@@ -1,8 +1,8 @@
 authorName: nni
 experimentName: default_test
 maxExecDuration: 5m
-maxTrialNum: 2
-trialConcurrency: 1
+maxTrialNum: 8
+trialConcurrency: 8
 searchSpacePath: search_space.json
 
 tuner:
@@ -17,12 +17,12 @@ assessor:
       start_step: 6
       threshold: 0.95
 trial:
-  codeDir: ../../../examples/trials/mnist-tfv1
-  command: python3 mnist.py --batch_num 100
+  codeDir: ./
+  command: python3 trial.py
   gpuNum: 0
 
 useAnnotation: false
 multiPhase: false
 multiThread: false
 
-trainingServicePlatform: local
+trainingServicePlatform: local

test/config_test/multi_phase/multi_phase_tpe.test.yml → test/config/assessors/medianstop.yml

@@ -2,21 +2,24 @@ authorName: nni
 experimentName: default_test
 maxExecDuration: 5m
 maxTrialNum: 8
-trialConcurrency: 4
-searchSpacePath: ./search_space.json
+trialConcurrency: 8
+searchSpacePath: search_space.json
 
 tuner:
   builtinTunerName: TPE
   classArgs:
     optimize_mode: maximize
 
+assessor:
+  builtinAssessorName: Medianstop
+  classArgs:
+    optimize_mode: maximize
+
 trial:
-  codeDir: .
-  command: python3 multi_phase.py
+  codeDir: ./
+  command: python3 trial.py
   gpuNum: 0
 
 useAnnotation: false
-multiPhase: true
-multiThread: false
 
 trainingServicePlatform: local

test/config/assessors/search_space.json

@@ -0,0 +1,7 @@
+{
+    "test":
+    {
+        "_type" : "choice",
+        "_value" : [1, 100]
+    }
+}

test/config/assessors/trial.py

@@ -0,0 +1,25 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+import random
+import time
+import nni
+
+if __name__ == '__main__':
+    print('trial start')
+    if random.random() > 0.5:
+        up = True
+    else:
+        up = False
+    v = 0.5
+    nni.get_next_parameter()
+    for i in range(20):
+        time.sleep(1)
+        for _ in range(2):
+            if up:
+                v *= 1.1
+            else:
+                v *= 0.9
+            nni.report_intermediate_result(v)
+    nni.report_final_result(v)
+    print('trial done')

test/config_test/examples/mnist-annotation.test.yml → test/config/examples/mnist-annotation.yml

@@ -12,7 +12,7 @@ assessor:
     optimize_mode: maximize
 trial:
   codeDir: ../../../examples/trials/mnist-annotation
-  command: python3 mnist.py --batch_num 100
+  command: python3 mnist.py --batch_num 10
   gpuNum: 0
 
 useAnnotation: true

test/config_test/examples/mnist-nested-search-space.test.yml → test/config/examples/mnist-nested-search-space.yml

@@ -14,7 +14,7 @@ assessor:
     optimize_mode: maximize
 trial:
   codeDir: ../../../examples/trials/mnist-nested-search-space
-  command: python3 mnist.py --batch_num 100
+  command: python3 mnist.py --batch_num 10
   gpuNum: 0
 
 useAnnotation: false

test/config_test/examples/mnist-pytorch.test.yml → test/config/examples/mnist-pytorch.yml

@@ -13,7 +13,7 @@ assessor:
     optimize_mode: maximize
 trial:
   codeDir: ../../../examples/trials/mnist-pytorch
-  command: python3 mnist.py  --epochs 2
+  command: python3 mnist.py  --epochs 1
   gpuNum: 0
 
 useAnnotation: false

test/config_test/examples/mnist-tfv1.test.yml → test/config/examples/mnist-tfv1.yml

@@ -13,7 +13,7 @@ assessor:
     optimize_mode: maximize
 trial:
   codeDir: ../../../examples/trials/mnist-tfv1
-  command: python3 mnist.py --batch_num 100
+  command: python3 mnist.py --batch_num 10
   gpuNum: 0
 
 useAnnotation: false