Added tests dir back

tests/script/check.py

@@ -0,0 +1,14 @@
+def check_return(r):
+    if 'return' not in r:
+        raise Exception('CM access function should always return key \'return\'!')
+    if 'error' in r:
+        raise Exception(r['error'])
+
+def check_list(r, string, found=True):
+    check_return(r)
+    if 'list' not in r:
+        raise Exception('CM search should return a list!')
+    if len(r['list']) < 1 and found:
+        raise Exception('CM search returned an empty list for ' + string)
+    if len(r['list']) > 0 and not found:
+        raise Exception('CM search returned at lease one entry for ' + string)

tests/script/process_dockerfile.py

@@ -0,0 +1,30 @@
+import sys
+import os
+import cmind as cm
+import check as checks
+import json
+import yaml
+
+files=sys.argv[1:]
+
+for file in files:
+    if not os.path.isfile(file):
+        continue
+    if not file.endswith("_cm.json") and not file.endswith("_cm.yaml"):
+        continue
+    if not file.startswith(os.path.join("cm-mlops", "script")):
+        continue
+
+    script_path = os.path.dirname(file)
+
+    f = open(file)
+
+    if file.endswith(".json"):
+        data = json.load(f)
+    elif file.endswith(".yaml"):
+        data = yaml.safe_load(f)
+
+    uid = data['uid']
+
+    r = cm.access({'action':'dockerfile', 'automation':'script', 'artifact': uid, 'quiet': 'yes'})
+    checks.check_return(r)

tests/script/process_readme.py

@@ -0,0 +1,26 @@
+import sys
+import os
+import cmind as cm
+import check as checks
+import json
+import yaml
+
+files=sys.argv[1:]
+
+for file in files:
+    if not os.path.isfile(file):
+        continue
+    if not file.endswith("_cm.json") and not file.endswith("_cm.yaml"):
+        continue
+    if not file.startswith(os.path.join("cm-mlops", "script")):
+        continue
+    script_path = os.path.dirname(file)
+    f = open(file)
+    if file.endswith(".json"):
+        data = json.load(f)
+    elif file.endswith(".yaml"):
+        data = yaml.safe_load(f)
+    uid = data['uid']
+
+    r = cm.access({'action':'doc', 'automation':'script', 'artifact': uid, 'quiet': 'yes'})
+    checks.check_return(r)

tests/script/test_deps.py

@@ -0,0 +1,23 @@
+# This test covers version, variation, compilation from src, add_deps, add_deps_recursive, deps, post_deps
+
+import cmind as cm
+import check as checks
+
+# MLPerf v3.0 inference is now very outdated and we are testing inference in separate tests
+
+#r = cm.access({'action':'run', 'automation':'script', 'tags': 'generate-run-cmds,mlperf', 'adr':
+#    {'loadgen': {'version': 'r3.0'}, 'compiler': {'tags': "gcc"}}, 'env': {'CM_MODEL': 'resnet50',
+#        'CM_DEVICE': 'cpu', 'CM_BACKEND': 'onnxruntime'}, 'quiet': 'yes'})
+#checks.check_return(r)
+#
+#r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'loadgen,version-r3.0,deps-python-non-virtual'})
+#checks.check_list(r, "loadgen,version-r3.0,deps-python-non-virtual")
+#
+#r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'inference,src,version-r3.0'})
+#checks.check_list(r, "inference,src,version-r3.0")
+#
+#r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_python,_resnet50,_onnxruntime,_cpu,_r3.0_default', 'adr': {'mlperf-implementation': { 'version': 'master'}}, 'quiet': 'yes'})
+#checks.check_return(r)
+#
+#r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_python,_resnet50,_tf,_cpu,_r3.0_default', 'adr': {'mlperf-implementation': { 'version': 'master'}}, 'quiet': 'yes'})
+#checks.check_return(r)

tests/script/test_docker.py

@@ -0,0 +1,21 @@
+# This test covers version, variation, compilation from src, add_deps_recursive, post_deps
+
+import cmind as cm
+import check as checks
+
+r = cm.access({'action':'run', 
+               'automation':'script', 
+               'tags': 'run,docker,container', 
+               'add_deps_recursive': {
+                  'compiler': {'tags': "gcc"}
+               }, 
+               'image_name':'cm-script-app-image-classification-onnx-py',
+               'env': {
+                 'CM_DOCKER_RUN_SCRIPT_TAGS': 'app,image-classification,onnx,python',
+                 'CM_MLOPS_REPO': 'ctuning@mlcommons-ck', 
+                 'CM_DOCKER_IMAGE_BASE': 'ubuntu:22.04'
+               }, 
+               'quiet': 'yes'
+              })
+
+checks.check_return(r)

tests/script/test_features.py

@@ -0,0 +1,21 @@
+# This test covers
+# 1. python-virtual-env and update_deps inside customize.py
+# 2. cache search using "-" prefix
+
+import cmind as cm
+import check as checks
+
+r = cm.access({'action':'run', 'automation':'script', 'tags': 'install,python-venv', 'name':'test', 'quiet': 'yes'})
+checks.check_return(r)
+
+r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'get,python,virtual,name-test'})
+checks.check_list(r, "get,python-venv")
+
+r = cm.access({'action':'run', 'automation':'script', 'tags': 'get,dataset,preprocessed,imagenet,_NHWC', 'quiet': 'yes'})
+checks.check_return(r)
+
+r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'get,dataset,preprocessed,imagenet,-_NCHW'})
+checks.check_list(r, "_NHWC")
+
+r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'get,dataset,preprocessed,imagenet,-_NHWC'})
+checks.check_list(r, "_NHWC", False)

tests/script/test_install.py

@@ -0,0 +1,10 @@
+# This test covers script installation, version, shared library install
+
+import cmind as cm
+import check as checks
+
+r = cm.access({'action':'run', 'automation':'script', 'tags': 'python,src,install,_shared', 'version': '3.9.10', 'quiet': 'true'})
+checks.check_return(r)
+
+r = cm.access({'action':'search', 'automation':'cache', 'tags': 'python,src,install,_shared,version-3.9.10'})
+checks.check_list(r, "python,src,install,_shared,version-3.9.10")

tests/test_cm.py

@@ -0,0 +1,14 @@
+try:
+    import cmind as cm
+
+    r = cm.access(['test', 'script'])
+    if 'return' not in r:
+        raise Exception('CM access function should always return key \'return\'!')
+    exit(0)
+
+except ImportError as e:
+    from sys import stderr
+    from subprocess import call
+    print('WARNING: CM module for python is not installed & jupyter notebooks will not be supported', file=stderr)
+    retcode = call(['cm', 'test', 'script'])
+    exit(retcode)

tests/test_search_speed.py

@@ -0,0 +1,25 @@
+import cmind as cm
+import time
+
+times = []
+
+steps = 10
+
+print ('Running search with tags {} times ...'.format(steps))
+
+for step in range(steps):
+
+    start = time.time()
+    r = cm.access({'action':'search',
+                   'automation':'script',
+                   'tags':'detect,os'})
+    timer = time.time() - start
+
+    if r['return']>0: cm.error(r)
+
+    times.append(timer)
+
+step = 0
+for t in times:
+    step += 1
+    print ("{}) {:0.3f} sec.".format(step, t))

tests/tutorials/test_tutorial_retinanet.py

@@ -0,0 +1,28 @@
+# This test covers version, variation, compilation from src, add_deps, add_deps_recursive, deps, post_deps
+
+import cmind as cm
+from pathlib import Path
+import sys
+import os
+
+sys.path.insert(1, os.path.join(Path(__file__).parent.parent.resolve(), "script"))
+import check as checks
+
+r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_cpp,_retinanet,_onnxruntime,_cpu', 'adr': \
+    {'python': {'version_min': '3.8'}, 'compiler': {'tags': "gcc"}, 'openimages-preprocessed': {'tags': '_50'}}, 'scenario': 'Offline', \
+    'mode': 'accuracy', 'test_query_count': '10', 'rerun': 'true', 'quiet': 'yes'})
+checks.check_return(r)
+
+r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_cpp,_retinanet,_onnxruntime,_cpu', 'adr': \
+    {'python': {'version_min': '3.8'}, 'compiler': {'tags': "gcc"}, 'openimages-preprocessed': {'tags': '_50'}}, 'scenario': 'Offline', \
+    'mode': 'performance', 'test_query_count': '10', 'rerun': 'true', 'quiet': 'yes'})
+checks.check_return(r)
+
+r = cm.access({'action':'run', 'automation':'script', 'tags': 'install,python-venv', 'version': '3.10.8', 'name': 'mlperf' })
+checks.check_return(r)
+
+r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_short,_dashboard', 'adr': \
+        {'python': {'name': 'mlperf', 'version_min': '3.8'}, 'compiler': {'tags': "gcc"}, 'openimages-preprocessed': {'tags': '_50'}}, 'submitter': 'Community', \
+        'implementation': 'cpp', 'hw_name': 'default', 'model': 'retinanet', 'backend': 'onnxruntime', 'device': 'cpu', 'scenario': 'Offline', \
+        'test_query_count': '10', 'clean': 'true', 'quiet': 'yes'})
+checks.check_return(r)

tests/tutorials/test_tutorial_tvm.py

@@ -0,0 +1,24 @@
+# This test covers version, variation, compilation from src, add_deps, add_deps_recursive, deps, post_deps
+
+import cmind as cm
+
+from pathlib import Path
+import sys
+import os
+
+sys.path.insert(1, os.path.join(Path(__file__).parent.parent.resolve(), "script"))
+import check as checks
+
+r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds', 'adr': \
+        {'python': {'name': 'mlperf', 'version_min': '3.8'}}, 'submitter': 'Community', \
+        'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx', 'device': 'cpu', 'scenario': 'Offline', \
+        'mode': 'accuracy', 'test_query_count': '5', 'clean': 'true', 'quiet': 'yes'})
+checks.check_return(r)
+
+
+r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_dashboard', 'adr': \
+        {'python': {'name': 'mlperf', 'version_min': '3.8'}}, 'submitter': 'Community', \
+        'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx', 'device': 'cpu', 'scenario': 'Offline', \
+        'test_query_count': '500', 'clean': 'true', 'quiet': 'yes'})
+checks.check_return(r)
+

tests/tutorials/test_tutorial_tvm_pip_ge.py

@@ -0,0 +1,21 @@
+import cmind as cm
+
+from pathlib import Path
+import sys
+import os
+
+sys.path.insert(1, os.path.join(Path(__file__).parent.parent.resolve(), "script"))
+import check as checks
+
+r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds', 'adr': \
+    {'python': {'name': 'mlperf', 'version_min': '3.8'}, 'tvm': {'tags': '_pip-install'}, 'tvm-model': {'tags': '_graph_executor'}}, \
+    'submitter': 'Community', 'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx', \
+    'device': 'cpu', 'scenario': 'Offline', 'mode': 'accuracy', 'test_query_count': '5', 'clean': 'true', 'quiet': 'yes'})
+checks.check_return(r)
+
+r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_short,_dashboard', 'adr': \
+    {'python': {'name': 'mlperf', 'version_min': '3.8'}, 'tvm': {'tags': '_pip-install'}, 'tvm-model': {'tags': '_graph_executor'}}, \
+    'submitter': 'Community', 'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx', \
+    'device': 'cpu', 'scenario': 'Offline', 'test_query_count': '500', 'clean': 'true', 'quiet': 'yes'})
+checks.check_return(r)
+

tests/tutorials/test_tutorial_tvm_pip_vm.py

@@ -0,0 +1,23 @@
+# This test covers version, variation, compilation from src, add_deps, add_deps_recursive, deps, post_deps
+
+import cmind as cm
+
+from pathlib import Path
+import sys
+import os
+
+sys.path.insert(1, os.path.join(Path(__file__).parent.parent.resolve(), "script"))
+import check as checks
+
+r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds', 'adr': \
+    {'python': {'name': 'mlperf', 'version_min': '3.8'}, 'tvm': {'tags': '_pip-install'}}, 'submitter': 'Community', \
+    'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx', 'device': 'cpu', 'scenario': 'Offline', \
+    'mode': 'accuracy', 'test_query_count': '5', 'clean': 'true', 'quiet': 'yes'})
+checks.check_return(r)
+
+r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_short,_dashboard', 'adr': \
+    {'python': {'name': 'mlperf', 'version_min': '3.8'}, 'tvm': {'tags': '_pip-install'}}, 'submitter': 'Community', \
+    'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx', 'device': 'cpu', 'scenario': 'Offline', \
+    'test_query_count': '500', 'clean': 'true', 'quiet': 'yes'})
+checks.check_return(r)
+