From 751b98880f6607f4c4117147f3491f7863b78c42 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 1 May 2024 14:55:51 +0530 Subject: [PATCH] Added tests dir back --- tests/script/check.py | 14 ++++++++++ tests/script/process_dockerfile.py | 30 +++++++++++++++++++++ tests/script/process_readme.py | 26 ++++++++++++++++++ tests/script/test_deps.py | 23 ++++++++++++++++ tests/script/test_docker.py | 21 +++++++++++++++ tests/script/test_features.py | 21 +++++++++++++++ tests/script/test_install.py | 10 +++++++ tests/test_cm.py | 14 ++++++++++ tests/test_search_speed.py | 25 +++++++++++++++++ tests/tutorials/test_tutorial_retinanet.py | 28 +++++++++++++++++++ tests/tutorials/test_tutorial_tvm.py | 24 +++++++++++++++++ tests/tutorials/test_tutorial_tvm_pip_ge.py | 21 +++++++++++++++ tests/tutorials/test_tutorial_tvm_pip_vm.py | 23 ++++++++++++++++ 13 files changed, 280 insertions(+) create mode 100644 tests/script/check.py create mode 100644 tests/script/process_dockerfile.py create mode 100644 tests/script/process_readme.py create mode 100644 tests/script/test_deps.py create mode 100644 tests/script/test_docker.py create mode 100644 tests/script/test_features.py create mode 100644 tests/script/test_install.py create mode 100644 tests/test_cm.py create mode 100644 tests/test_search_speed.py create mode 100644 tests/tutorials/test_tutorial_retinanet.py create mode 100644 tests/tutorials/test_tutorial_tvm.py create mode 100644 tests/tutorials/test_tutorial_tvm_pip_ge.py create mode 100644 tests/tutorials/test_tutorial_tvm_pip_vm.py diff --git a/tests/script/check.py b/tests/script/check.py new file mode 100644 index 0000000000..7394406d8f --- /dev/null +++ b/tests/script/check.py @@ -0,0 +1,14 @@ +def check_return(r): + if 'return' not in r: + raise Exception('CM access function should always return key \'return\'!') + if 'error' in r: + raise Exception(r['error']) + +def check_list(r, string, found=True): + check_return(r) + if 'list' not in r: + raise Exception('CM search should return a list!') + if len(r['list']) < 1 and found: + raise Exception('CM search returned an empty list for ' + string) + if len(r['list']) > 0 and not found: + raise Exception('CM search returned at lease one entry for ' + string) diff --git a/tests/script/process_dockerfile.py b/tests/script/process_dockerfile.py new file mode 100644 index 0000000000..9a58fce389 --- /dev/null +++ b/tests/script/process_dockerfile.py @@ -0,0 +1,30 @@ +import sys +import os +import cmind as cm +import check as checks +import json +import yaml + +files=sys.argv[1:] + +for file in files: + if not os.path.isfile(file): + continue + if not file.endswith("_cm.json") and not file.endswith("_cm.yaml"): + continue + if not file.startswith(os.path.join("cm-mlops", "script")): + continue + + script_path = os.path.dirname(file) + + f = open(file) + + if file.endswith(".json"): + data = json.load(f) + elif file.endswith(".yaml"): + data = yaml.safe_load(f) + + uid = data['uid'] + + r = cm.access({'action':'dockerfile', 'automation':'script', 'artifact': uid, 'quiet': 'yes'}) + checks.check_return(r) diff --git a/tests/script/process_readme.py b/tests/script/process_readme.py new file mode 100644 index 0000000000..492813a050 --- /dev/null +++ b/tests/script/process_readme.py @@ -0,0 +1,26 @@ +import sys +import os +import cmind as cm +import check as checks +import json +import yaml + +files=sys.argv[1:] + +for file in files: + if not os.path.isfile(file): + continue + if not file.endswith("_cm.json") and not file.endswith("_cm.yaml"): + continue + if not file.startswith(os.path.join("cm-mlops", "script")): + continue + script_path = os.path.dirname(file) + f = open(file) + if file.endswith(".json"): + data = json.load(f) + elif file.endswith(".yaml"): + data = yaml.safe_load(f) + uid = data['uid'] + + r = cm.access({'action':'doc', 'automation':'script', 'artifact': uid, 'quiet': 'yes'}) + checks.check_return(r) diff --git a/tests/script/test_deps.py b/tests/script/test_deps.py new file mode 100644 index 0000000000..aaf19bc81b --- /dev/null +++ b/tests/script/test_deps.py @@ -0,0 +1,23 @@ +# This test covers version, variation, compilation from src, add_deps, add_deps_recursive, deps, post_deps + +import cmind as cm +import check as checks + +# MLPerf v3.0 inference is now very outdated and we are testing inference in separate tests + +#r = cm.access({'action':'run', 'automation':'script', 'tags': 'generate-run-cmds,mlperf', 'adr': +# {'loadgen': {'version': 'r3.0'}, 'compiler': {'tags': "gcc"}}, 'env': {'CM_MODEL': 'resnet50', +# 'CM_DEVICE': 'cpu', 'CM_BACKEND': 'onnxruntime'}, 'quiet': 'yes'}) +#checks.check_return(r) +# +#r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'loadgen,version-r3.0,deps-python-non-virtual'}) +#checks.check_list(r, "loadgen,version-r3.0,deps-python-non-virtual") +# +#r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'inference,src,version-r3.0'}) +#checks.check_list(r, "inference,src,version-r3.0") +# +#r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_python,_resnet50,_onnxruntime,_cpu,_r3.0_default', 'adr': {'mlperf-implementation': { 'version': 'master'}}, 'quiet': 'yes'}) +#checks.check_return(r) +# +#r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_python,_resnet50,_tf,_cpu,_r3.0_default', 'adr': {'mlperf-implementation': { 'version': 'master'}}, 'quiet': 'yes'}) +#checks.check_return(r) diff --git a/tests/script/test_docker.py b/tests/script/test_docker.py new file mode 100644 index 0000000000..5a02c932ed --- /dev/null +++ b/tests/script/test_docker.py @@ -0,0 +1,21 @@ +# This test covers version, variation, compilation from src, add_deps_recursive, post_deps + +import cmind as cm +import check as checks + +r = cm.access({'action':'run', + 'automation':'script', + 'tags': 'run,docker,container', + 'add_deps_recursive': { + 'compiler': {'tags': "gcc"} + }, + 'image_name':'cm-script-app-image-classification-onnx-py', + 'env': { + 'CM_DOCKER_RUN_SCRIPT_TAGS': 'app,image-classification,onnx,python', + 'CM_MLOPS_REPO': 'ctuning@mlcommons-ck', + 'CM_DOCKER_IMAGE_BASE': 'ubuntu:22.04' + }, + 'quiet': 'yes' + }) + +checks.check_return(r) diff --git a/tests/script/test_features.py b/tests/script/test_features.py new file mode 100644 index 0000000000..d116cbd5bf --- /dev/null +++ b/tests/script/test_features.py @@ -0,0 +1,21 @@ +# This test covers +# 1. python-virtual-env and update_deps inside customize.py +# 2. cache search using "-" prefix + +import cmind as cm +import check as checks + +r = cm.access({'action':'run', 'automation':'script', 'tags': 'install,python-venv', 'name':'test', 'quiet': 'yes'}) +checks.check_return(r) + +r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'get,python,virtual,name-test'}) +checks.check_list(r, "get,python-venv") + +r = cm.access({'action':'run', 'automation':'script', 'tags': 'get,dataset,preprocessed,imagenet,_NHWC', 'quiet': 'yes'}) +checks.check_return(r) + +r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'get,dataset,preprocessed,imagenet,-_NCHW'}) +checks.check_list(r, "_NHWC") + +r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'get,dataset,preprocessed,imagenet,-_NHWC'}) +checks.check_list(r, "_NHWC", False) diff --git a/tests/script/test_install.py b/tests/script/test_install.py new file mode 100644 index 0000000000..66fa164d50 --- /dev/null +++ b/tests/script/test_install.py @@ -0,0 +1,10 @@ +# This test covers script installation, version, shared library install + +import cmind as cm +import check as checks + +r = cm.access({'action':'run', 'automation':'script', 'tags': 'python,src,install,_shared', 'version': '3.9.10', 'quiet': 'true'}) +checks.check_return(r) + +r = cm.access({'action':'search', 'automation':'cache', 'tags': 'python,src,install,_shared,version-3.9.10'}) +checks.check_list(r, "python,src,install,_shared,version-3.9.10") diff --git a/tests/test_cm.py b/tests/test_cm.py new file mode 100644 index 0000000000..41fb402c22 --- /dev/null +++ b/tests/test_cm.py @@ -0,0 +1,14 @@ +try: + import cmind as cm + + r = cm.access(['test', 'script']) + if 'return' not in r: + raise Exception('CM access function should always return key \'return\'!') + exit(0) + +except ImportError as e: + from sys import stderr + from subprocess import call + print('WARNING: CM module for python is not installed & jupyter notebooks will not be supported', file=stderr) + retcode = call(['cm', 'test', 'script']) + exit(retcode) diff --git a/tests/test_search_speed.py b/tests/test_search_speed.py new file mode 100644 index 0000000000..3086a83408 --- /dev/null +++ b/tests/test_search_speed.py @@ -0,0 +1,25 @@ +import cmind as cm +import time + +times = [] + +steps = 10 + +print ('Running search with tags {} times ...'.format(steps)) + +for step in range(steps): + + start = time.time() + r = cm.access({'action':'search', + 'automation':'script', + 'tags':'detect,os'}) + timer = time.time() - start + + if r['return']>0: cm.error(r) + + times.append(timer) + +step = 0 +for t in times: + step += 1 + print ("{}) {:0.3f} sec.".format(step, t)) diff --git a/tests/tutorials/test_tutorial_retinanet.py b/tests/tutorials/test_tutorial_retinanet.py new file mode 100644 index 0000000000..404b69fe93 --- /dev/null +++ b/tests/tutorials/test_tutorial_retinanet.py @@ -0,0 +1,28 @@ +# This test covers version, variation, compilation from src, add_deps, add_deps_recursive, deps, post_deps + +import cmind as cm +from pathlib import Path +import sys +import os + +sys.path.insert(1, os.path.join(Path(__file__).parent.parent.resolve(), "script")) +import check as checks + +r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_cpp,_retinanet,_onnxruntime,_cpu', 'adr': \ + {'python': {'version_min': '3.8'}, 'compiler': {'tags': "gcc"}, 'openimages-preprocessed': {'tags': '_50'}}, 'scenario': 'Offline', \ + 'mode': 'accuracy', 'test_query_count': '10', 'rerun': 'true', 'quiet': 'yes'}) +checks.check_return(r) + +r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_cpp,_retinanet,_onnxruntime,_cpu', 'adr': \ + {'python': {'version_min': '3.8'}, 'compiler': {'tags': "gcc"}, 'openimages-preprocessed': {'tags': '_50'}}, 'scenario': 'Offline', \ + 'mode': 'performance', 'test_query_count': '10', 'rerun': 'true', 'quiet': 'yes'}) +checks.check_return(r) + +r = cm.access({'action':'run', 'automation':'script', 'tags': 'install,python-venv', 'version': '3.10.8', 'name': 'mlperf' }) +checks.check_return(r) + +r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_short,_dashboard', 'adr': \ + {'python': {'name': 'mlperf', 'version_min': '3.8'}, 'compiler': {'tags': "gcc"}, 'openimages-preprocessed': {'tags': '_50'}}, 'submitter': 'Community', \ + 'implementation': 'cpp', 'hw_name': 'default', 'model': 'retinanet', 'backend': 'onnxruntime', 'device': 'cpu', 'scenario': 'Offline', \ + 'test_query_count': '10', 'clean': 'true', 'quiet': 'yes'}) +checks.check_return(r) diff --git a/tests/tutorials/test_tutorial_tvm.py b/tests/tutorials/test_tutorial_tvm.py new file mode 100644 index 0000000000..930e3622df --- /dev/null +++ b/tests/tutorials/test_tutorial_tvm.py @@ -0,0 +1,24 @@ +# This test covers version, variation, compilation from src, add_deps, add_deps_recursive, deps, post_deps + +import cmind as cm + +from pathlib import Path +import sys +import os + +sys.path.insert(1, os.path.join(Path(__file__).parent.parent.resolve(), "script")) +import check as checks + +r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds', 'adr': \ + {'python': {'name': 'mlperf', 'version_min': '3.8'}}, 'submitter': 'Community', \ + 'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx', 'device': 'cpu', 'scenario': 'Offline', \ + 'mode': 'accuracy', 'test_query_count': '5', 'clean': 'true', 'quiet': 'yes'}) +checks.check_return(r) + + +r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_dashboard', 'adr': \ + {'python': {'name': 'mlperf', 'version_min': '3.8'}}, 'submitter': 'Community', \ + 'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx', 'device': 'cpu', 'scenario': 'Offline', \ + 'test_query_count': '500', 'clean': 'true', 'quiet': 'yes'}) +checks.check_return(r) + diff --git a/tests/tutorials/test_tutorial_tvm_pip_ge.py b/tests/tutorials/test_tutorial_tvm_pip_ge.py new file mode 100644 index 0000000000..0c9a4b9c33 --- /dev/null +++ b/tests/tutorials/test_tutorial_tvm_pip_ge.py @@ -0,0 +1,21 @@ +import cmind as cm + +from pathlib import Path +import sys +import os + +sys.path.insert(1, os.path.join(Path(__file__).parent.parent.resolve(), "script")) +import check as checks + +r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds', 'adr': \ + {'python': {'name': 'mlperf', 'version_min': '3.8'}, 'tvm': {'tags': '_pip-install'}, 'tvm-model': {'tags': '_graph_executor'}}, \ + 'submitter': 'Community', 'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx', \ + 'device': 'cpu', 'scenario': 'Offline', 'mode': 'accuracy', 'test_query_count': '5', 'clean': 'true', 'quiet': 'yes'}) +checks.check_return(r) + +r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_short,_dashboard', 'adr': \ + {'python': {'name': 'mlperf', 'version_min': '3.8'}, 'tvm': {'tags': '_pip-install'}, 'tvm-model': {'tags': '_graph_executor'}}, \ + 'submitter': 'Community', 'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx', \ + 'device': 'cpu', 'scenario': 'Offline', 'test_query_count': '500', 'clean': 'true', 'quiet': 'yes'}) +checks.check_return(r) + diff --git a/tests/tutorials/test_tutorial_tvm_pip_vm.py b/tests/tutorials/test_tutorial_tvm_pip_vm.py new file mode 100644 index 0000000000..81069194d4 --- /dev/null +++ b/tests/tutorials/test_tutorial_tvm_pip_vm.py @@ -0,0 +1,23 @@ +# This test covers version, variation, compilation from src, add_deps, add_deps_recursive, deps, post_deps + +import cmind as cm + +from pathlib import Path +import sys +import os + +sys.path.insert(1, os.path.join(Path(__file__).parent.parent.resolve(), "script")) +import check as checks + +r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds', 'adr': \ + {'python': {'name': 'mlperf', 'version_min': '3.8'}, 'tvm': {'tags': '_pip-install'}}, 'submitter': 'Community', \ + 'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx', 'device': 'cpu', 'scenario': 'Offline', \ + 'mode': 'accuracy', 'test_query_count': '5', 'clean': 'true', 'quiet': 'yes'}) +checks.check_return(r) + +r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_short,_dashboard', 'adr': \ + {'python': {'name': 'mlperf', 'version_min': '3.8'}, 'tvm': {'tags': '_pip-install'}}, 'submitter': 'Community', \ + 'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx', 'device': 'cpu', 'scenario': 'Offline', \ + 'test_query_count': '500', 'clean': 'true', 'quiet': 'yes'}) +checks.check_return(r) +