Add platform match support

test/config/assessors/trial.py

@@ -7,10 +7,7 @@
 
 if __name__ == '__main__':
     print('trial start')
-    if random.random() > 0.5:
-        up = True
-    else:
-        up = False
+    up = random.random() > 0.5
     v = 0.5
     nni.get_next_parameter()
     for i in range(20):

test/config/integration_tests.yml

@@ -3,6 +3,7 @@ defaultTestCaseConfig:
   launchCommand: nnictl create --config $configFile
   stopCommand: nnictl stop
   experimentStatusCheck: True
+  platform: linux darwin win32
 
 testCases:
 #######################################################################
@@ -96,12 +97,14 @@ testCases:
   stopCommand: python3 -c 'import nnicli as nc; nc.stop_nni()'
   validator:
     class: NnicliValidator
+  platform: linux darwin
 
 - name: foreground
   configFile: test/config/examples/sklearn-regression.yml
   launchCommand: python3 nni_test/nnitest/foreground.py --config $configFile --timeout 45
   stopCommand:
   experimentStatusCheck: False
+  platform: linux darwin
 
 # Experiment resume test part 1
 - name: nnictl-resume-1
@@ -167,6 +170,7 @@ testCases:
 
 - name: tuner-smac
   configFile: test/config/tuners/mnist-annotation-smac.yml
+  platform: linux darwin
 
 - name: tuner-tpe
   configFile: test/config/tuners/mnist-annotation-tpe.yml
@@ -176,6 +180,7 @@ testCases:
 
 - name: tuner-bohb
   configFile: test/config/tuners/mnist-bohb.yml
+  platform: linux darwin
 
 - name: tuner-gp
   configFile: test/config/tuners/mnist-gp.yml

test/nni_test/nnitest/run_tests.py

@@ -14,7 +14,7 @@
 
 from utils import setup_experiment, get_experiment_status, get_yml_content, dump_yml_content, get_experiment_id, \
     parse_max_duration_time, get_trial_stats, deep_update, print_trial_job_log, get_failed_trial_jobs
-from utils import GREEN, RED, CLEAR, STATUS_URL, TRIAL_JOBS_URL, EXPERIMENT_URL, REST_ENDPOINT
+from utils import GREEN, RED, CLEAR, STATUS_URL, TRIAL_JOBS_URL, EXPERIMENT_URL, REST_ENDPOINT, detect_port
 import validators
 
 it_variables = {}
@@ -63,12 +63,6 @@ def prepare_config_file(test_case_config, it_config, args):
     return new_config_file
 
 def run_test_case(test_case_config, it_config, args):
-    # fill test case default config
-    for k in it_config['defaultTestCaseConfig']:
-        if k not in test_case_config:
-            test_case_config[k] = it_config['defaultTestCaseConfig'][k]
-    print(json.dumps(test_case_config, indent=4))
-
     new_config_file = prepare_config_file(test_case_config, it_config, args)
     # set configFile variable
     it_variables['$configFile'] = new_config_file
@@ -96,7 +90,6 @@ def invoke_validator(test_case_config, nni_source_dir):
     validator(REST_ENDPOINT, None, nni_source_dir, **kwargs)
 
 def get_max_values(config_file):
-    '''Get maxExecDuration and maxTrialNum of experiment'''
     experiment_config = get_yml_content(config_file)
     return parse_max_duration_time(experiment_config['maxExecDuration']), experiment_config['maxTrialNum']
 
@@ -118,7 +111,6 @@ def get_command(test_case_config, commandKey):
     return command
 
 def launch_test(config_file, training_service, test_case_config):
-    '''run test per configuration file'''
     launch_command = get_command(test_case_config, 'launchCommand')
     print('launch command: ', launch_command, flush=True)
 
@@ -181,6 +173,19 @@ def case_included(name, cases):
             return True
     return False
 
+def wait_for_port_available(port, timeout):
+    begin_time = time.time()
+    while True:
+        if not detect_port(port):
+            return
+        if time.time() - begin_time > timeout:
+            msg = 'port {} is not available in {} seconds.'.format(port, timeout)
+            raise RuntimeError(msg)
+        time.sleep(5)
+
+def match_platform(test_case_config):
+    return sys.platform in test_case_config['platform'].split(' ')
+
 def run(args):
     it_config = get_yml_content(args.config)
 
@@ -191,8 +196,19 @@ def run(args):
             continue
         if args.cases and not case_included(name, args.cases):
             continue
+
+        # fill test case default config
+        for k in it_config['defaultTestCaseConfig']:
+            if k not in test_case_config:
+                test_case_config[k] = it_config['defaultTestCaseConfig'][k]
+        print(json.dumps(test_case_config, indent=4))
+
+        if not match_platform(test_case_config):
+            print('skipped {}, platform {} not match [{}]'.format(name, sys.platform, test_case_config['platform']))
+            continue
+
+        wait_for_port_available(8080, 30)
         print('{}Testing: {}{}'.format(GREEN, name, CLEAR))
-        time.sleep(5)
         begin_time = time.time()
 
         run_test_case(test_case_config, it_config, args)

test/pipelines/pipelines-it-local-windows.yml

@@ -22,5 +22,5 @@ jobs:
     displayName: 'Naive test'
   - script: |
       cd test
-      python nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts local --exclude smac,bohb,nnicli
+      python nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts local
     displayName: 'Integration tests'

test/pipelines/pipelines-it-pai-windows.yml

@@ -64,5 +64,5 @@ jobs:
       set PATH=$(ENV_PATH)
       python --version
       python nni_test/nnitest/generate_ts_config.py --ts pai --pai_host $(pai_host) --pai_user $(pai_user) --pai_pwd $(pai_pwd) --vc $(pai_virtual_cluster) --nni_docker_image $(docker_image) --data_dir $(data_dir) --output_dir $(output_dir) --nni_manager_ip $(nni_manager_ip)
-      python nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts pai --exclude multi-phase,smac,bohb,nnicli
+      python nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts pai --exclude multi-phase
     displayName: 'Examples and advanced features tests on pai'

test/pipelines/pipelines-it-remote-windows.yml

@@ -44,7 +44,7 @@ jobs:
       cd test
       python nni_test/nnitest/generate_ts_config.py --ts remote --remote_user $(docker_user) --remote_host $(remote_host) --remote_port $(Get-Content port) --remote_pwd $(docker_pwd) --nni_manager_ip $(nni_manager_ip)
       Get-Content config/training_service.yml
-      python nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts remote --exclude cifar10,smac,bohb,nnicli
+      python nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts remote --exclude cifar10
     displayName: 'integration test'
   - task: SSH@0
     inputs: