Check pylint for nni_cmd

tools/nni_cmd/command_utils.py

@@ -3,7 +3,7 @@
 import os
 import signal
 import psutil
-from .common_utils import print_error, print_normal, print_warning
+from .common_utils import print_error
 
 
 def check_output_command(file_path, head=None, tail=None):

tools/nni_cmd/common_utils.py

@@ -21,10 +21,10 @@
 import os
 import sys
 import json
-import ruamel.yaml as yaml
-import psutil
 import socket
 from pathlib import Path
+import ruamel.yaml as yaml
+import psutil
 from .constants import ERROR_INFO, NORMAL_INFO, WARNING_INFO, COLOR_RED_FORMAT, COLOR_YELLOW_FORMAT
 
 def get_yml_content(file_path):
@@ -34,6 +34,7 @@ def get_yml_content(file_path):
             return yaml.load(file, Loader=yaml.Loader)
     except yaml.scanner.ScannerError as err:
         print_error('yaml file format error!')
+        print_error(err)
         exit(1)
     except Exception as exception:
         print_error(exception)
@@ -46,6 +47,7 @@ def get_json_content(file_path):
             return json.load(file)
     except TypeError as err:
         print_error('json file format error!')
+        print_error(err)
         return None
 
 def print_error(content):
@@ -70,7 +72,7 @@ def detect_process(pid):
 
 def detect_port(port):
     '''Detect if the port is used'''
-    socket_test = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
+    socket_test = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
     try:
         socket_test.connect(('127.0.0.1', int(port)))
         socket_test.close()
@@ -79,7 +81,7 @@ def detect_port(port):
         return False
 
 def get_user():
-    if sys.platform =='win32':
+    if sys.platform == 'win32':
         return os.environ['USERNAME']
     else:
         return os.environ['USER']

tools/nni_cmd/config_schema.py

@@ -19,13 +19,13 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 import os
-from schema import Schema, And, Use, Optional, Regex, Or
+from schema import Schema, And, Optional, Regex, Or
 from .constants import SCHEMA_TYPE_ERROR, SCHEMA_RANGE_ERROR, SCHEMA_PATH_ERROR
 
 
-def setType(key, type):
+def setType(key, valueType):
     '''check key type'''
-    return And(type, error=SCHEMA_TYPE_ERROR % (key, type.__name__))
+    return And(valueType, error=SCHEMA_TYPE_ERROR % (key, valueType.__name__))
 
 def setChoice(key, *args):
     '''check choice'''
@@ -47,7 +47,7 @@ def setPathCheck(key):
     'experimentName': setType('experimentName', str),
     Optional('description'): setType('description', str),
     'trialConcurrency': setNumberRange('trialConcurrency', int, 1, 99999),
-    Optional('maxExecDuration'): And(Regex(r'^[1-9][0-9]*[s|m|h|d]$',error='ERROR: maxExecDuration format is [digit]{s,m,h,d}')),
+    Optional('maxExecDuration'): And(Regex(r'^[1-9][0-9]*[s|m|h|d]$', error='ERROR: maxExecDuration format is [digit]{s,m,h,d}')),
     Optional('maxTrialNum'): setNumberRange('maxTrialNum', int, 1, 99999),
     'trainingServicePlatform': setChoice('trainingServicePlatform', 'remote', 'local', 'pai', 'kubeflow', 'frameworkcontroller'),
     Optional('searchSpacePath'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'searchSpacePath'),
@@ -106,7 +106,7 @@ def setPathCheck(key):
         'builtinTunerName': 'NetworkMorphism',
         Optional('classArgs'): {
             Optional('optimize_mode'): setChoice('optimize_mode', 'maximize', 'minimize'),
-            Optional('task'): setChoice('task', 'cv','nlp','common'),
+            Optional('task'): setChoice('task', 'cv', 'nlp', 'common'),
             Optional('input_width'): setType('input_width', int),
             Optional('input_channel'): setType('input_channel', int),
             Optional('n_output_node'): setType('n_output_node', int),
@@ -139,7 +139,7 @@ def setPathCheck(key):
             Optional('selection_num_warm_up'):  setType('selection_num_warm_up', int),
             Optional('selection_num_starting_points'):  setType('selection_num_starting_points', int),
             },
-        Optional('includeIntermediateResults'): setType('includeIntermediateResults', bool), 
+        Optional('includeIntermediateResults'): setType('includeIntermediateResults', bool),
         Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'),
     },
     'PPOTuner': {
@@ -232,35 +232,35 @@ def setPathCheck(key):
 }
 
 common_trial_schema = {
-'trial':{
-    'command': setType('command', str),
-    'codeDir': setPathCheck('codeDir'),
-    Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999),
-    Optional('nasMode'): setChoice('nasMode', 'classic_mode', 'enas_mode', 'oneshot_mode', 'darts_mode')
+    'trial':{
+        'command': setType('command', str),
+        'codeDir': setPathCheck('codeDir'),
+        Optional('gpuNum'): setNumberRange('gpuNum', int, 0, 99999),
+        Optional('nasMode'): setChoice('nasMode', 'classic_mode', 'enas_mode', 'oneshot_mode', 'darts_mode')
     }
 }
 
 pai_trial_schema = {
-'trial':{
-    'command': setType('command', str),
-    'codeDir': setPathCheck('codeDir'),
-    'gpuNum': setNumberRange('gpuNum', int, 0, 99999),
-    'cpuNum': setNumberRange('cpuNum', int, 0, 99999),
-    'memoryMB': setType('memoryMB', int),
-    'image': setType('image', str),
-    Optional('authFile'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'authFile'),
-    Optional('shmMB'): setType('shmMB', int),
-    Optional('dataDir'): And(Regex(r'hdfs://(([0-9]{1,3}.){3}[0-9]{1,3})(:[0-9]{2,5})?(/.*)?'),\
-                         error='ERROR: dataDir format error, dataDir format is hdfs://xxx.xxx.xxx.xxx:xxx'),
-    Optional('outputDir'): And(Regex(r'hdfs://(([0-9]{1,3}.){3}[0-9]{1,3})(:[0-9]{2,5})?(/.*)?'),\
-                         error='ERROR: outputDir format error, outputDir format is hdfs://xxx.xxx.xxx.xxx:xxx'),
-    Optional('virtualCluster'): setType('virtualCluster', str),
-    Optional('nasMode'): setChoice('nasMode', 'classic_mode', 'enas_mode', 'oneshot_mode', 'darts_mode'),
-    Optional('portList'): [{
-        "label": setType('label', str),
-        "beginAt": setType('beginAt', int),
-        "portNumber": setType('portNumber', int)
-    }]
+    'trial':{
+        'command': setType('command', str),
+        'codeDir': setPathCheck('codeDir'),
+        'gpuNum': setNumberRange('gpuNum', int, 0, 99999),
+        'cpuNum': setNumberRange('cpuNum', int, 0, 99999),
+        'memoryMB': setType('memoryMB', int),
+        'image': setType('image', str),
+        Optional('authFile'): And(os.path.exists, error=SCHEMA_PATH_ERROR % 'authFile'),
+        Optional('shmMB'): setType('shmMB', int),
+        Optional('dataDir'): And(Regex(r'hdfs://(([0-9]{1,3}.){3}[0-9]{1,3})(:[0-9]{2,5})?(/.*)?'),\
+                            error='ERROR: dataDir format error, dataDir format is hdfs://xxx.xxx.xxx.xxx:xxx'),
+        Optional('outputDir'): And(Regex(r'hdfs://(([0-9]{1,3}.){3}[0-9]{1,3})(:[0-9]{2,5})?(/.*)?'),\
+                            error='ERROR: outputDir format error, outputDir format is hdfs://xxx.xxx.xxx.xxx:xxx'),
+        Optional('virtualCluster'): setType('virtualCluster', str),
+        Optional('nasMode'): setChoice('nasMode', 'classic_mode', 'enas_mode', 'oneshot_mode', 'darts_mode'),
+        Optional('portList'): [{
+            "label": setType('label', str),
+            "beginAt": setType('beginAt', int),
+            "portNumber": setType('portNumber', int)
+        }]
     }
 }
 
@@ -273,7 +273,7 @@ def setPathCheck(key):
 }
 
 kubeflow_trial_schema = {
-'trial':{
+    'trial':{
         'codeDir':  setPathCheck('codeDir'),
         Optional('nasMode'): setChoice('nasMode', 'classic_mode', 'enas_mode', 'oneshot_mode', 'darts_mode'),
         Optional('ps'): {
@@ -315,7 +315,7 @@ def setPathCheck(key):
             'server': setType('server', str),
             'path': setType('path', str)
         }
-    },{
+    }, {
         'operator': setChoice('operator', 'tf-operator', 'pytorch-operator'),
         'apiVersion': setType('apiVersion', str),
         Optional('storage'): setChoice('storage', 'nfs', 'azureStorage'),
@@ -363,7 +363,7 @@ def setPathCheck(key):
             'server': setType('server', str),
             'path': setType('path', str)
         }
-    },{
+    }, {
         Optional('storage'): setChoice('storage', 'nfs', 'azureStorage'),
         Optional('serviceAccountName'): setType('serviceAccountName', str),
         'keyVault': {
@@ -383,24 +383,24 @@ def setPathCheck(key):
 }
 
 machine_list_schema = {
-Optional('machineList'):[Or({
-    'ip': setType('ip', str),
-    Optional('port'): setNumberRange('port', int, 1, 65535),
-    'username': setType('username', str),
-    'passwd': setType('passwd', str),
-    Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'),
-    Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int),
-    Optional('useActiveGpu'): setType('useActiveGpu', bool)
-    },{
-    'ip': setType('ip', str),
-    Optional('port'): setNumberRange('port', int, 1, 65535),
-    'username': setType('username', str),
-    'sshKeyPath': setPathCheck('sshKeyPath'),
-    Optional('passphrase'): setType('passphrase', str),
-    Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'),
-    Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int),
-    Optional('useActiveGpu'): setType('useActiveGpu', bool)
-})]
+    Optional('machineList'):[Or({
+        'ip': setType('ip', str),
+        Optional('port'): setNumberRange('port', int, 1, 65535),
+        'username': setType('username', str),
+        'passwd': setType('passwd', str),
+        Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'),
+        Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int),
+        Optional('useActiveGpu'): setType('useActiveGpu', bool)
+        }, {
+            'ip': setType('ip', str),
+            Optional('port'): setNumberRange('port', int, 1, 65535),
+            'username': setType('username', str),
+            'sshKeyPath': setPathCheck('sshKeyPath'),
+            Optional('passphrase'): setType('passphrase', str),
+            Optional('gpuIndices'): Or(int, And(str, lambda x: len([int(i) for i in x.split(',')]) > 0), error='gpuIndex format error!'),
+            Optional('maxTrialNumPerGpu'): setType('maxTrialNumPerGpu', int),
+            Optional('useActiveGpu'): setType('useActiveGpu', bool)
+        })]
 }
 
 LOCAL_CONFIG_SCHEMA = Schema({**common_schema, **common_trial_schema})

tools/nni_cmd/config_utils.py

@@ -21,7 +21,6 @@
 
 import os
 import json
-import shutil
 from .constants import NNICTL_HOME_DIR
 
 class Config:
@@ -73,29 +72,29 @@ def __init__(self):
         self.experiment_file = os.path.join(NNICTL_HOME_DIR, '.experiment')
         self.experiments = self.read_file()
 
-    def add_experiment(self, id, port, time, file_name, platform):
+    def add_experiment(self, expId, port, time, file_name, platform):
         '''set {key:value} paris to self.experiment'''
-        self.experiments[id] = {}
-        self.experiments[id]['port'] = port
-        self.experiments[id]['startTime'] = time
-        self.experiments[id]['endTime'] = 'N/A'
-        self.experiments[id]['status'] = 'INITIALIZED'
-        self.experiments[id]['fileName'] = file_name
-        self.experiments[id]['platform'] = platform
+        self.experiments[expId] = {}
+        self.experiments[expId]['port'] = port
+        self.experiments[expId]['startTime'] = time
+        self.experiments[expId]['endTime'] = 'N/A'
+        self.experiments[expId]['status'] = 'INITIALIZED'
+        self.experiments[expId]['fileName'] = file_name
+        self.experiments[expId]['platform'] = platform
         self.write_file()
 
-    def update_experiment(self, id, key, value):
+    def update_experiment(self, expId, key, value):
         '''Update experiment'''
-        if id not in self.experiments:
+        if expId not in self.experiments:
             return False
-        self.experiments[id][key] = value
+        self.experiments[expId][key] = value
         self.write_file()
         return True
 
-    def remove_experiment(self, id):
+    def remove_experiment(self, expId):
         '''remove an experiment by id'''
         if id in self.experiments:
-            self.experiments.pop(id)
+            self.experiments.pop(expId)
         self.write_file()
 
     def get_all_experiments(self):
@@ -109,7 +108,7 @@ def write_file(self):
                 json.dump(self.experiments, file)
         except IOError as error:
             print('Error:', error)
-            return
+            return ''
 
     def read_file(self):
         '''load config from local file'''
@@ -119,4 +118,4 @@ def read_file(self):
                     return json.load(file)
             except ValueError:
                 return {}
-        return {} 
+        return {}

tools/nni_cmd/constants.py

@@ -21,7 +21,7 @@
 import os
 from colorama import Fore
 
-NNICTL_HOME_DIR = os.path.join(os.path.expanduser('~'), '.local',  'nnictl')
+NNICTL_HOME_DIR = os.path.join(os.path.expanduser('~'), '.local', 'nnictl')
 
 ERROR_INFO = 'ERROR: %s'
 
@@ -58,7 +58,8 @@
              '-----------------------------------------------------------------------\n'
 
 EXPERIMENT_START_FAILED_INFO = 'There is an experiment running in the port %d, please stop it first or set another port!\n' \
-                               'You could use \'nnictl stop --port [PORT]\' command to stop an experiment!\nOr you could use \'nnictl create --config [CONFIG_PATH] --port [PORT]\' to set port!\n'
+                               'You could use \'nnictl stop --port [PORT]\' command to stop an experiment!\nOr you could ' \
+                               'use \'nnictl create --config [CONFIG_PATH] --port [PORT]\' to set port!\n'
 
 EXPERIMENT_INFORMATION_FORMAT = '----------------------------------------------------------------------------------------\n' \
                      '                Experiment information\n' \