diff --git a/.github/workflows/test-mlperf-inference-abtf-poc.yml b/.github/workflows/test-mlperf-inference-abtf-poc.yml index e1e352c612..c08d89ed0e 100644 --- a/.github/workflows/test-mlperf-inference-abtf-poc.yml +++ b/.github/workflows/test-mlperf-inference-abtf-poc.yml @@ -3,7 +3,7 @@ name: MLPerf inference ABTF POC Test on: - pull_request_target: + pull_request: branches: [ "main", "mlperf-inference" ] paths: - '.github/workflows/test-mlperf-inference-abtf-poc.yml' @@ -37,7 +37,7 @@ jobs: cm pull repo mlcommons@cm4abtf --branch=poc - name: Test MLPerf Inference ABTF POC using ${{ matrix.backend }} on docker run: | - cm run script --tags=run-abtf,inference,_poc-demo --adr.compiler.tags=gcc --quiet --docker --docker_it=no -v --gh_token=${{ secrets.ABTF_ACCESS_TOKEN }} + cm run script --tags=run-abtf,inference,_poc-demo --adr.compiler.tags=gcc --quiet -v build2: runs-on: ${{ matrix.os }} @@ -65,5 +65,31 @@ jobs: cm pull repo mlcommons@cm4abtf --branch=poc - name: Test MLPerf Inference ABTF POC using ${{ matrix.backend }} on ${{ matrix.os }} run: | - cm run script --tags=run-abtf,inference,_poc-demo --adr.compiler.tags=gcc --quiet -v --gh_token=${{ secrets.ABTF_ACCESS_TOKEN }} + cm run script --tags=run-abtf,inference,_poc-demo --adr.compiler.tags=gcc --quiet -v + build3: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [windows-latest] + python-version: [ "3.8", "3.12" ] + backend: [ "pytorch" ] + implementation: [ "python" ] + exclude: + - python-version: "3.8" + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python3 -m pip install cmind + cm pull repo --url=${{ github.event.pull_request.head.repo.html_url }} --checkout=${{ github.event.pull_request.head.ref }} + cm pull repo mlcommons@cm4abtf --branch=poc + - name: Test MLPerf Inference ABTF POC using ${{ matrix.backend }} on ${{ matrix.os }} + run: | + cm run script --tags=run-abtf,inference,_poc-demo --quiet --env.CM_MLPERF_LOADGEN_BUILD_FROM_SRC=off -v \ No newline at end of file diff --git a/automation/script/module.py b/automation/script/module.py index b98a3906a2..c9ddb07e5e 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -3,7 +3,7 @@ # as portable and reusable automation recipes with simple tags, native scripts # and a unified CLI, Python API and JSON/YAML meta descriptions. # -# This is a stable prototype of the CM script automation +# This is a stable prototype of the CM script automation being developed by Grigori Fursin and Arjun Suresh # # TBD: when we have bandwidth and resources, we should refactor it # and make it cleaner and simpler while keeping full backwards compatibility. @@ -4443,7 +4443,11 @@ def update_env_with_values(env, fail_on_not_found=False): if tmp_value not in env and fail_on_not_found: return {'return':1, 'error':'variable {} is not in env'.format(tmp_value)} if tmp_value in env: - value = value.replace("<<<"+tmp_value+">>>", str(env[tmp_value])) + if type(value) == str: + value = value.replace("<<<"+tmp_value+">>>", str(env[tmp_value])) + elif type(value) == list: + for i,val in enumerate(value): + value[i] = value[i].replace("<<<"+tmp_value+">>>", str(env[tmp_value])) env[key] = value diff --git a/cm-repro/cm-run-script-input.json b/cm-repro/cm-run-script-input.json new file mode 100644 index 0000000000..9fe20fd5e8 --- /dev/null +++ b/cm-repro/cm-run-script-input.json @@ -0,0 +1,41 @@ +{ + "action": "run", + "automation": "script", + "tags": "run-mlperf,inference,_find-performance,_full", + "model": "mixtral-8x7b-99", + "implementation": "reference", + "framework": "pytorch", + "category": "edge", + "scenario": "Offline", + "execution_mode": "test", + "device": "cuda", + "test_query_count": "100", + "adr": { + "cuda": { + "version": "12.4.1" + } + }, + "quiet": true, + "repro": true, + "cmd": [ + "--tags=run-mlperf,inference,_find-performance,_full", + "--model=mixtral-8x7b-99", + "--implementation=reference", + "--framework=pytorch", + "--category=edge", + "--scenario=Offline", + "--execution_mode=test", + "--device=cuda", + "--test_query_count=100", + "--adr.cuda.version=12.4.1", + "--quiet", + "--repro" + ], + "out": "con", + "parsed_automation": [ + [ + "script", + "5b4e0237da074764" + ] + ] +} \ No newline at end of file diff --git a/script/app-mlperf-inference-intel/_cm.yaml b/script/app-mlperf-inference-intel/_cm.yaml index 01735e0e77..55cbe426cf 100644 --- a/script/app-mlperf-inference-intel/_cm.yaml +++ b/script/app-mlperf-inference-intel/_cm.yaml @@ -112,20 +112,22 @@ deps: ######################################################################## # Install OpenImages + - enable_if_env: CM_MODEL: - retinanet - tags: compile,intel,model,_retinanet names: - - retinanet-compiler + - openimages-original + - dataset-original + tags: get,dataset,original,openimages,_validation,_custom-annotations,_full - enable_if_env: CM_MODEL: - retinanet names: - - openimages-preprocessed - - dataset-preprocessed - tags: get,dataset,preprocessed,openimages,_for.retinanet.onnx,_NCHW,_validation,_custom-annotations + - openimages-calibration + - dataset-calibration + tags: get,dataset,original,openimages,_calibration @@ -214,11 +216,6 @@ variations: default: true env: CM_MODEL: resnet50 - dataset_imagenet_preprocessed_input_square_side: 224 - ml_model_has_background_class: "YES" - ml_model_image_height: 224 - loadgen_buffer_size: 1024 - loadgen_dataset_size: 50000 CM_BENCHMARK: STANDALONE_CLASSIFICATION resnet50,int8: @@ -239,23 +236,39 @@ variations: env: CM_MODEL: retinanet CM_ML_MODEL_STARTING_WEIGHTS_FILENAME: "https://zenodo.org/record/6617981/files/resnext50_32x4d_fpn.pth" - dataset_imagenet_preprocessed_input_square_side: 224 - ml_model_image_height: 800 - ml_model_image_width: 800 - loadgen_buffer_size: 64 - loadgen_dataset_size: 24576 CM_BENCHMARK: STANDALONE_OBJECT_DETECTION deps: - tags: get,generic-python-lib,_numpy + names: + - pip-package + - numpy + version: "1.23.5" + 3d-unet-99: + group: model + base: + - 3d-unet_ + env: + CM_MODEL: 3d-unet-99 + + 3d-unet-99.9: + group: model + base: + - 3d-unet_ + env: + CM_MODEL: 3d-unet-99.9 + + 3d-unet_: + env: + CM_BENCHMARK: MEDICAL_IMAGING + deps: + - tags: get,dataset,kits19,preprocessed + - tags: get,ml-model,medical-imaging,3d-unet,_pytorch,_weights bert_: env: CM_BENCHMARK: STANDALONE_BERT - dataset_squad_tokenized_max_seq_length: 384 - loadgen_buffer_size: 10833 - loadgen_dataset_size: 10833 bert_,pytorch: deps: @@ -327,9 +340,6 @@ variations: - conda-package - mkl tags: get,generic,conda-package,_package.mkl,_source.conda-forge - enable_if_env: - CM_MLPERF_INFERENCE_CODE_VERSION: - - v4.0 - names: - conda-package - mkl-include @@ -342,10 +352,6 @@ variations: - conda-package - ncurses tags: get,generic,conda-package,_package.ncurses,_source.conda-forge - - names: - - conda-package - - ncurses - tags: get,generic,conda-package,_package.ncurses,_source.conda-forge - tags: get,generic-sys-util,_numactl - tags: get,generic,conda-package,_package.jemalloc,_source.conda-forge names: @@ -388,8 +394,6 @@ variations: - tags: get,mlcommons,inference,loadgen,_custom-python,_keep-build names: - inference-loadgen - env: - CM_PYTHON_BIN_WITH_PATH: "<<>>/python3" resnet50,build-harness: @@ -410,6 +414,194 @@ variations: - run-mode - loadgen-scenario + retinanet,pytorch: + adr: + conda-package: + tags: _name.retinanet-pt + compiler: + tags: gcc + conda-python: + version: "3.9" + deps: + - tags: get,conda,_name.retinanet-pt + - tags: get,python,_conda.retinanet-pt + - names: + - conda-package + - mkl + tags: get,generic,conda-package,_package.mkl,_source.conda-forge + - names: + - conda-package + - libstdcxx-ng + tags: get,generic,conda-package,_package.libstdcxx-ng,_source.conda-forge + + - names: + - conda-package + - mkl + tags: get,generic,conda-package,_package.mkl,_source.intel + - names: + - conda-package + - mkl-include + tags: get,generic,conda-package,_package.mkl-include,_source.intel + - names: + - conda-package + - intel-openmp + tags: get,generic,conda-package,_package.intel-openmp,_source.intel + - names: + - conda-package + - llvm-openmp + tags: get,generic,conda-package,_package.llvm-openmp,_source.conda-forge + - names: + - conda-package + - ncurses + tags: get,generic,conda-package,_package.ncurses,_source.conda-forge + - tags: get,generic-sys-util,_numactl + - tags: get,generic,conda-package,_package.cmake,_source.conda-forge + names: + - conda-package + - cmake + - tags: get,generic,conda-package,_package.jemalloc,_source.conda-forge + names: + - conda-package + - jemalloc + - tags: get,pytorch,from.src,_for-intel-mlperf-inference-retinanet + names: + - pytorch + - tags: get,torchvision,from.src,_sha.8e078971b8aebdeb1746fea58851e3754f103053 + update_tags_from_env_with_prefix: + "_python.": + - CM_PYTHON_BIN_WITH_PATH + names: + - torchvision + - tags: install,opencv,from.src,_branch.4.x + names: + - opencv-from-src + - tags: get,git,repo,_repo.https://github.com/Tencent/rapidjson.git,_sha.e4bde977 + names: + - rapidjson-src + env: + CM_GIT_CHECKOUT_PATH_ENV_NAME: CM_RAPIDJSON_SRC_REPO_PATH + - tags: install,gflags,from.src + names: + - gflags-from-src + - tags: install,onednn,from.src,_branch.rls-v2.6 + names: + - onednn-from-src + - tags: get,generic-python-lib,_package.scikit-learn + names: + - pip-package + - scikit-learn + - tags: get,generic-python-lib,_package.opencv-python + names: + - pip-package + - opencv-python + - tags: get,generic-python-lib,_package.pycocotools + names: + - pip-package + - pycocotools + - tags: install,ipex,from.src,_for-intel-mlperf-inference-v3.1-retinanet + names: + - ipex-from-src + - tags: get,generic,conda-package,_package.ninja + names: + - conda-package + - ninja + - tags: get,mlcommons,inference,src + names: + - inference-src + - tags: get,mlcommons,inference,loadgen,_custom-python,_keep-build + names: + - inference-loadgen + + + retinanet,build-harness: + deps: + - tags: reproduce,mlperf,inference,intel,_compile-model + inherit_variation_tags: true + force_cache: true + skip_inherit_variation_groups: + - run-mode + - loadgen-scenario + new_env_keys: + - CM_ML_MODEL_RETINANET_INT8_FILE_WITH_PATH + + retinanet,compile-model: + deps: + - tags: get,ml-model,retinanet,_pytorch,_fp32 + new_env_keys: + - CM_ML_MODEL_RETINANET_INT8_FILE_WITH_PATH + + 3d-unet_,pytorch: + adr: + conda-package: + tags: _name.3d-unet-pt + deps: + - tags: get,generic-sys-util,_libffi7 + - tags: get,conda,_name.3d-unet-pt + - tags: get,python,_conda.3d-unet-pt + adr: + conda-python: + version: "3.8" + - names: + - conda-package + - mkl + tags: get,generic,conda-package,_package.mkl,_source.conda-forge + - names: + - conda-package + - mkl-include + tags: get,generic,conda-package,_package.mkl-include,_source.intel + - names: + - conda-package + - mkl-service + tags: get,generic,conda-package,_package.mkl-service,_source.intel + - names: + - conda-package + - mkl_fft + tags: get,generic,conda-package,_package.mkl_fft,_source.intel + - names: + - conda-package + - mkl_random + tags: get,generic,conda-package,_package.mkl_random,_source.intel + - names: + - conda-package + - ncurses + tags: get,generic,conda-package,_package.ncurses,_source.conda-forge + - tags: get,generic-sys-util,_numactl + - tags: get,generic,conda-package,_package.jemalloc,_source.conda-forge + names: + - conda-package + - jemalloc + - tags: install,ipex,from.src,_for-intel-mlperf-inference-v3.1-3d-unet + names: + - ipex-from-src + - tags: get,generic,conda-package,_package.ninja + names: + - conda-package + - ninja + - tags: get,mlcommons,inference,src + names: + - inference-src + - tags: get,mlcommons,inference,loadgen,_custom-python,_keep-build + names: + - inference-loadgen + + + 3d-unet_,build-harness: + deps: + - tags: reproduce,mlperf,inference,intel,_compile-model + inherit_variation_tags: true + force_cache: true + skip_inherit_variation_groups: + - run-mode + - loadgen-scenario + + 3d-unet_,compile-model: + deps: + - tags: reproduce,mlperf,inference,intel,_calibration + inherit_variation_tags: true + force_cache: true + skip_inherit_variation_groups: + - run-mode + - loadgen-scenario gptj_,pytorch: adr: @@ -670,6 +862,7 @@ variations: - tags: generate,user-conf,mlperf,inference names: - user-conf-generator + - tags: get,generic-sys-util,_rsync env: CM_LOCAL_MLPERF_INFERENCE_INTEL_RUN_MODE: run_harness diff --git a/script/app-mlperf-inference-intel/build_retinanet_harness.sh b/script/app-mlperf-inference-intel/build_retinanet_harness.sh new file mode 100644 index 0000000000..0d577b26b2 --- /dev/null +++ b/script/app-mlperf-inference-intel/build_retinanet_harness.sh @@ -0,0 +1,44 @@ +export PATH=${CM_CONDA_BIN_PATH}:$PATH + +cd ${CM_HARNESS_CODE_ROOT} + + +export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"} + +export IPEX_PATH=${CM_IPEX_INSTALLED_PATH} +export TORCH_PATH=`python -c 'import torch;print(torch.utils.cmake_prefix_path)'` + +if [[ -z ${TORCH_PATH} ]]; then + echo "Torch not found" + exit 1 +fi + +export LOADGEN_DIR="${CM_MLPERF_INFERENCE_LOADGEN_INSTALL_PATH}/../" +export OPENCV_DIR=${CM_OPENCV_BUILD_PATH} +export RAPIDJSON_INCLUDE_DIR=${CM_RAPIDJSON_SRC_REPO_PATH}/include +export GFLAGS_DIR=${CM_GFLAGS_BUILD_PATH} +export USE_CUDA=0 + +BUILD_DIR=${PWD}/build +rm -rf "$BUILD_DIR" + +SRC_DIR=${PWD}/src + +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${OPENCV_DIR}/lib:${ONEDNN_DIR}/build/src +export LIBRARY_PATH=${LIBRARY_PATH}:${CONDA_PREFIX}/lib + +cmd="cmake -DCMAKE_PREFIX_PATH=${TORCH_PATH} \ + -DLOADGEN_DIR=${LOADGEN_DIR} \ + -DOpenCV_DIR=${OPENCV_DIR} \ + -DRapidJSON_INCLUDE_DIR=${RAPIDJSON_INCLUDE_DIR} \ + -Dgflags_DIR=${GFLAGS_DIR} \ + -DINTEL_EXTENSION_FOR_PYTORCH_PATH=${IPEX_PATH} \ + -DCMAKE_BUILD_TYPE=Release \ + -B${BUILD_DIR} \ + -H${SRC_DIR}" +echo "$cmd" +eval "$cmd" +test "$?" -eq 0 || exit "$?" + +cmake --build ${BUILD_DIR} --config Release -j$(nproc) +test "$?" -eq 0 || exit "$?" diff --git a/script/app-mlperf-inference-intel/compile_resnet50.sh b/script/app-mlperf-inference-intel/compile_resnet50.sh index 657d871efe..ee81956ec9 100644 --- a/script/app-mlperf-inference-intel/compile_resnet50.sh +++ b/script/app-mlperf-inference-intel/compile_resnet50.sh @@ -1,12 +1,9 @@ export PATH=${CM_CONDA_BIN_PATH}:$PATH -echo $PWD - -export DATA_CAL_DIR=calibration_dataset +export DATA_CAL_DIR=${CM_HARNESS_CODE_ROOT}/calibration_dataset export CHECKPOINT=${CM_ML_MODEL_FILE_WITH_PATH} cd ${CM_HARNESS_CODE_ROOT} -#mkdir -p -#cp src/ckernels/scripts/resnet50-int8-scales.json + bash generate_torch_model.sh test "$?" -eq 0 || exit "$?" diff --git a/script/app-mlperf-inference-intel/compile_retinanet.sh b/script/app-mlperf-inference-intel/compile_retinanet.sh new file mode 100644 index 0000000000..7e23b889af --- /dev/null +++ b/script/app-mlperf-inference-intel/compile_retinanet.sh @@ -0,0 +1,11 @@ +export PATH=${CM_CONDA_BIN_PATH}:$PATH + +export CALIBRATION_ANNOTATIONS=${CM_DATASET_CALIBRATION_ANNOTATIONS_FILE_PATH} + +export CALIBRATION_DATA_DIR=${CM_CALIBRATION_DATASET_PATH} +export MODEL_CHECKPOINT=${CM_ML_MODEL_FILE_WITH_PATH} + +cd ${CM_HARNESS_CODE_ROOT} + +bash run_calibration.sh +test "$?" -eq 0 || exit "$?" diff --git a/script/app-mlperf-inference-intel/customize.py b/script/app-mlperf-inference-intel/customize.py index 4efff4bb81..4f8c8a27ba 100644 --- a/script/app-mlperf-inference-intel/customize.py +++ b/script/app-mlperf-inference-intel/customize.py @@ -22,7 +22,7 @@ def preprocess(i): return {'return': 1, 'error': 'Please select a variation specifying the device to run on'} ml_model = env['CM_MODEL'] - master_model = ml_model.replace("-99", "").replace("-99.9","") + master_model = ml_model.replace("-99.9", "").replace("-99","") master_model = master_model.replace("gptj", "gpt-j") backend = env['CM_MLPERF_BACKEND'] @@ -61,9 +61,12 @@ def preprocess(i): loadgen_mode = env['CM_MLPERF_LOADGEN_MODE'] env['CONDA_PREFIX'] = env['CM_CONDA_PREFIX'] + if env['CM_LOCAL_MLPERF_INFERENCE_INTEL_RUN_MODE'] == "calibration": if master_model == "resnet50": i['run_script_input']['script_name'] = "prepare_imagenet_calibration" + if master_model == "3d-unet": + i['run_script_input']['script_name'] = "prepare_3d-unet_data_model" else: calibration_root = os.path.join(env['CM_MLPERF_INFERENCE_RESULTS_PATH'], 'closed', 'Intel', 'calibration', master_model, backend+"-"+device) @@ -76,6 +79,9 @@ def preprocess(i): elif env['CM_LOCAL_MLPERF_INFERENCE_INTEL_RUN_MODE'] == "compilation": if master_model == "resnet50": i['run_script_input']['script_name'] = "compile_resnet50" + elif master_model == "retinanet": + i['run_script_input']['script_name'] = "compile_retinanet" + env['CM_ML_MODEL_RETINANET_INT8_FILE_WITH_PATH'] = os.path.join(os.path.dirname(env['CM_ML_MODEL_FILE_WITH_PATH']), 'retinanet-int8-model.pth') elif env['CM_LOCAL_MLPERF_INFERENCE_INTEL_RUN_MODE'] == "build_harness": print(f"Harness Root: {harness_root}") @@ -83,10 +89,13 @@ def preprocess(i): i['run_script_input']['script_name'] = "build_bert_harness" env['CM_MLPERF_INFERENCE_INTEL_HARNESS_PATH'] = os.path.join(os.getcwd(), "harness", "build", "bert_inference") env['DATA_PATH'] = os.path.join(os.getcwd(), "harness", "bert") - if "resnet50" in env['CM_MODEL']: + elif "resnet50" in env['CM_MODEL']: i['run_script_input']['script_name'] = "build_resnet50_harness" env['CM_MLPERF_INFERENCE_INTEL_HARNESS_PATH'] = os.path.join(os.getcwd(), "harness", "build", "resnet50_inference") env['DATA_PATH'] = os.path.join(os.getcwd(), "harness", "resnet50") + elif "retinanet" in env['CM_MODEL']: + i['run_script_input']['script_name'] = "build_retinanet_harness" + env['CM_MLPERF_INFERENCE_INTEL_HARNESS_PATH'] = os.path.join(os.getcwd(), "harness", "build", "retinanet_inference") elif "gpt" in env['CM_MODEL']: i['run_script_input']['script_name'] = "build_gptj_harness" env['CM_MLPERF_INFERENCE_INTEL_HARNESS_PATH'] = os.path.join(os.getcwd(), "harness", "build", "gptj_inference") @@ -138,8 +147,18 @@ def preprocess(i): elif 'resnet50' in env['CM_MODEL']: env['MODEL_PATH'] = os.path.dirname(os.path.dirname(env['CM_MLPERF_INFERENCE_INTEL_HARNESS_PATH'])) env['DATASET_PATH'] = os.path.dirname(os.path.dirname(env['CM_MLPERF_INFERENCE_INTEL_HARNESS_PATH'])) - env['CM_RUN_DIR'] = i['run_script_input']['path'] - env['CM_RUN_CMD'] = "bash run_resnet50_harness.sh " + ("--accuracy" if env['CM_MLPERF_LOADGEN_MODE'] == "accuracy" else "") + env['CM_RUN_DIR'] = env['CM_MLPERF_OUTPUT_DIR'] + env['CM_RUN_CMD'] = f"bash {os.path.join(i['run_script_input']['path'],'run_resnet50_harness.sh')} " + + elif 'retinanet' in env['CM_MODEL']: + env['MODEL_PATH'] = env['CM_ML_MODEL_RETINANET_INT8_FILE_WITH_PATH'] + env['DATA_DIR'] = env['CM_DATASET_PATH_ROOT'] + env['CM_RUN_DIR'] = env['CM_MLPERF_OUTPUT_DIR'] + env['CM_RUN_CMD'] = f"bash {os.path.join(i['run_script_input']['path'],'run_retinanet_harness.sh')} " + + elif '3d-unet' in env['CM_MODEL']: + env['CM_RUN_DIR'] = env['CM_MLPERF_OUTPUT_DIR'] + env['CM_RUN_CMD'] = f"bash {os.path.join(i['run_script_input']['path'],'run_3d-unet_harness.sh')} " elif "gptj" in env['CM_MODEL']: env['CM_RUN_DIR'] = i['run_script_input']['path'] diff --git a/script/app-mlperf-inference-intel/prepare_3d-unet_data_model.sh b/script/app-mlperf-inference-intel/prepare_3d-unet_data_model.sh new file mode 100644 index 0000000000..2633881471 --- /dev/null +++ b/script/app-mlperf-inference-intel/prepare_3d-unet_data_model.sh @@ -0,0 +1,19 @@ +#!/bin/bash + + +export DOWNLOAD_DATA_DIR=${CM_DATASET_PATH} +cd ${CM_HARNESS_CODE_ROOT} + +mkdir -p build +ln -sf ${CM_DATASET_PREPROCESSED_PATH} build/preprocessed_data +mkdir -p build/model +ln -sf ${CM_ML_MODEL_FILE_WITH_PATH} build/model/3dunet_kits19_pytorch_checkpoint.pth +#make setup +#make duplicate_kits19_case_00185 + +#make preprocess_data +make preprocess_calibration_data +make preprocess_gaussian_patches + +export LD_PRELOAD=${CONDA_PREFIX}/lib/libiomp5.so:$LD_PRELOAD +python trace_model.py diff --git a/script/app-mlperf-inference-intel/run_3d-unet_harness.sh b/script/app-mlperf-inference-intel/run_3d-unet_harness.sh new file mode 100644 index 0000000000..78f44fb2b9 --- /dev/null +++ b/script/app-mlperf-inference-intel/run_3d-unet_harness.sh @@ -0,0 +1,49 @@ +#!/bin/bash + + +scenario=${CM_MLPERF_LOADGEN_SCENARIO} +OUTDIR="${CM_MLPERF_OUTPUT_DIR}" +#python ../../user_config.py + + +CPUS_PER_INSTANCE=8 + +export DNNL_MAX_CPU_ISA=AVX512_CORE_AMX + +number_threads=`nproc --all` +export number_cores=`lscpu -b -p=Core,Socket | grep -v '^#' | sort -u | wc -l` +num_numa=$(numactl --hardware|grep available|awk -F' ' '{ print $2 }') +num_instance=$((number_cores/CPUS_PER_INSTANCE)) +export PYTHONPATH=${CM_HARNESS_CODE_ROOT}/common:$PYTHONPATH +cp -r ${CM_HARNESS_CODE_ROOT}/meta $OUTDIR/ +cp ${CM_HARNESS_CODE_ROOT}/unet3d_jit_model.pt $OUTDIR/ +cp ${CM_HARNESS_CODE_ROOT}/calibration_result.json $OUTDIR/ +ln -sf ${CM_HARNESS_CODE_ROOT}/build $OUTDIR/build +#the log path is hardcoded in the intel implementation. This is a hack to get them to where we want +rm -rf $OUTDIR/output_logs +ln -sf $OUTDIR $OUTDIR/output_logs + +PYTHON_VERSION=`python -c 'import sys; print ("{}.{}".format(sys.version_info.major, sys.version_info.minor))'` +SITE_PACKAGES=`python -c 'import site; print (site.getsitepackages()[0])'` +IPEX_VERSION=`conda list |grep torch-ipex | awk '{print $2}' ` +export LD_LIBRARY_PATH=$SITE_PACKAGES/torch_ipex-${IPEX_VERSION}-py$PYTHON_VERSION-linux-x86_64.egg/lib/:$LD_LIBRARY_PATH +export LD_PRELOAD=$CONDA_PREFIX/lib/libjemalloc.so:$LD_PRELOAD +export MALLOC_CONF="oversize_threshold:1,background_thread:true,percpu_arena:percpu,metadata_thp:always,dirty_decay_ms:9000000000,muzzy_decay_ms:9000000000"; + + +#cd ${CM_HARNESS_CODE_ROOT} +cmd="python ${CM_HARNESS_CODE_ROOT}/run.py \ + --mode ${LOADGEN_MODE} \ + --workload-name 3dunet \ + --mlperf-conf ${CM_MLPERF_CONF} \ + --user-conf ${CM_MLPERF_USER_CONF} \ + --workload-config ${CM_HARNESS_CODE_ROOT}/config.json \ + --num-instance $num_instance \ + --cpus-per-instance $CPUS_PER_INSTANCE \ + --scenario $scenario \ + --warmup 1 \ + --precision=int8" + +echo "$cmd" +eval "$cmd" +test "$?" -eq 0 || exit "$?" diff --git a/script/app-mlperf-inference-intel/run_resnet50_harness.sh b/script/app-mlperf-inference-intel/run_resnet50_harness.sh index 9ec4ba245f..861d891aab 100644 --- a/script/app-mlperf-inference-intel/run_resnet50_harness.sh +++ b/script/app-mlperf-inference-intel/run_resnet50_harness.sh @@ -11,11 +11,11 @@ THREADS_PER_INSTANCE=$((( ${WORKERS_PER_PROC} * ${CM_HOST_CPU_THREADS_PER_CORE}) export LD_PRELOAD=${CONDA_PREFIX}/lib/libjemalloc.so export LD_PRELOAD=${CONDA_PREFIX}/lib/libiomp5.so -#export MALLOC_CONF="oversize_threshold:1,background_thread:true,metadata_thp:auto,dirty_decay_ms:9000000000,muzzy_decay_ms:9000000000"; +export MALLOC_CONF="oversize_threshold:1,background_thread:true,metadata_thp:auto,dirty_decay_ms:9000000000,muzzy_decay_ms:9000000000"; KMP_SETTING="KMP_AFFINITY=granularity=fine,compact,1,0" export KMP_BLOCKTIME=1 -#export $KMP_SETTING +export $KMP_SETTING export DATA_DIR=${CM_HARNESS_CODE_ROOT}/ILSVRC2012_img_val @@ -25,6 +25,7 @@ export RN50_FULL=${CM_HARNESS_CODE_ROOT}/models/resnet50-full.pth export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${CONDA_PREFIX}/lib +rsync -avz ${CM_HARNESS_CODE_ROOT}/val_data/ ${DATA_DIR}/ executable="${CM_HARNESS_CODE_ROOT}/build/bin/mlperf_runner" number_threads=`nproc --all` @@ -34,17 +35,16 @@ num_instance=$(($number_cores / $THREADS_PER_INSTANCE)) scenario=${CM_MLPERF_LOADGEN_SCENARIO} OUTDIR="${CM_MLPERF_OUTPUT_DIR}" - +scenario="Offline" #python ../../user_config.py -USER_CONF="${CM_MLPERF_USER_CONF}" #--warmup_iters 20 \ CONFIG=" --scenario ${scenario} --mode ${LOADGEN_MODE} --model_name resnet50 \ --rn50-part1 ${RN50_START} --rn50-part3 ${RN50_END} --rn50-full-model ${RN50_FULL} \ --data_path ${DATA_DIR} \ - --mlperf_conf ${CM_MLPERF_CONF} --user_conf ${USER_CONF} \ - --num_instance $number_cores \ + --mlperf_conf ${CM_MLPERF_CONF} --user_conf ${CM_MLPERF_USER_CONF} \ --cpus_per_instance $CPUS_PER_INSTANCE \ + --num_instance $number_cores \ --total_sample_count 50000 \ --batch_size 256 " diff --git a/script/app-mlperf-inference-intel/run_retinanet_harness.sh b/script/app-mlperf-inference-intel/run_retinanet_harness.sh new file mode 100644 index 0000000000..98ca3a5b26 --- /dev/null +++ b/script/app-mlperf-inference-intel/run_retinanet_harness.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +CPUS_PER_INSTANCE=8 +number_threads=`nproc --all` +export number_cores=`lscpu -b -p=Core,Socket | grep -v '^#' | sort -u | wc -l` +number_sockets=`grep physical.id /proc/cpuinfo | sort -u | wc -l` +cpu_per_socket=$((number_cores/number_sockets)) +number_instance=$((number_cores/CPUS_PER_INSTANCE)) + +WORKERS_PER_PROC=${WORKERS_PER_PROC:-4} +THREADS_PER_INSTANCE=$((( ${WORKERS_PER_PROC} * ${CM_HOST_CPU_THREADS_PER_CORE}) / ${CM_HOST_CPU_SOCKETS})) + +export LD_PRELOAD=${CONDA_PREFIX}/lib/libjemalloc.so +export LD_PRELOAD=${CONDA_PREFIX}/lib/libiomp5.so +export MALLOC_CONF="oversize_threshold:1,background_thread:true,metadata_thp:auto,dirty_decay_ms:9000000000,muzzy_decay_ms:9000000000"; + +KMP_SETTING="KMP_AFFINITY=granularity=fine,compact,1,0" +export KMP_BLOCKTIME=1 +export $KMP_SETTING + + +export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${CONDA_PREFIX}/lib + +executable="${CM_HARNESS_CODE_ROOT}/build/bin/mlperf_runner" + +number_threads=`nproc --all` +export number_cores=`lscpu -b -p=Core,Socket | grep -v '^#' | sort -u | wc -l` +num_numa=$(numactl --hardware|grep available|awk -F' ' '{ print $2 }') +num_instance=$(($number_cores / $THREADS_PER_INSTANCE)) + +scenario=${CM_MLPERF_LOADGEN_SCENARIO} +OUTDIR="${CM_MLPERF_OUTPUT_DIR}" +scenario="Offline" +#python ../../user_config.py + +#--warmup_iters 20 \ +CONFIG=" --scenario ${scenario} --mode ${LOADGEN_MODE} --model_name retinanet \ + --model_path ${MODEL_PATH} \ + --data_path ${DATA_DIR} \ + --mlperf_conf ${CM_MLPERF_CONF} --user_conf ${CM_MLPERF_USER_CONF} \ + --cpus_per_instance $CPUS_PER_INSTANCE \ + --num_instance $number_instance \ + --total_sample_count 24781 \ + --batch_size 1 + " + +cmd=" ${executable} ${CONFIG}" +echo "$cmd" +eval "$cmd" +test "$?" -eq 0 || exit "$?" diff --git a/script/app-mlperf-inference-mlcommons-cpp/dockerfiles/ubuntu_22.04.Dockerfile b/script/app-mlperf-inference-mlcommons-cpp/dockerfiles/ubuntu_22.04.Dockerfile deleted file mode 100644 index 7f9b8c493c..0000000000 --- a/script/app-mlperf-inference-mlcommons-cpp/dockerfiles/ubuntu_22.04.Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM ubuntu:22.04 - -# Maintained by the MLCommons taskforce on automation and reproducibility -LABEL github="https://github.com/mlcommons/ck" -LABEL maintainer="https://cKnowledge.org/mlcommons-taskforce" - -SHELL ["/bin/bash", "-c"] -ARG CM_GH_TOKEN - -# Notes: https://runnable.com/blog/9-common-dockerfile-mistakes -# Install system dependencies -RUN apt-get update -y -RUN apt-get install -y python3 python3-pip git sudo wget - -# Install python packages -RUN python3 -m pip install cmind requests giturlparse - -# Setup docker environment -ENTRYPOINT ["/bin/bash", "-c"] -ENV TZ="US/Pacific" -ENV PATH="${PATH}:/home/cmuser/.local/bin" -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ >/etc/timezone - -# Setup docker user -RUN groupadd cm -RUN useradd -g cm --create-home --shell /bin/bash cmuser -RUN echo "cmuser ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers -USER cmuser:cm -WORKDIR /home/cmuser - -# Download CM repo for scripts -RUN cm pull repo ctuning@mlcommons-ck - -# Install all system dependencies -RUN cm run script --tags=get,sys-utils-cm --quiet - -# Run commands -RUN cm run script --tags=app,mlperf,inference,_intel-original,_gptj-99 --quiet --fake_run --env.CM_RUN_STATE_DOCKER=True diff --git a/script/app-mlperf-inference-mlcommons-python/_cm.yaml b/script/app-mlperf-inference-mlcommons-python/_cm.yaml index 0b92244768..2cb9367dd9 100644 --- a/script/app-mlperf-inference-mlcommons-python/_cm.yaml +++ b/script/app-mlperf-inference-mlcommons-python/_cm.yaml @@ -407,6 +407,18 @@ deps: CM_MLPERF_CUSTOM_MODEL_PATH: - "on" + ## mixtral-8x7b + - tags: get,ml-model,mixtral + names: + - ml-model + - mixtral-model + enable_if_env: + CM_MODEL: + - mixtral-8x7b + skip_if_env: + CM_MLPERF_CUSTOM_MODEL_PATH: + - "on" + ## 3d-unet - tags: get,ml-model,medical-imaging,3d-unet names: @@ -513,6 +525,14 @@ deps: - llama2-70b-99 - llama2-70b-99.9 + ## OpenOrca,mbxp,gsm8k combined dataset for mixtral-8x7b + - tags: get,dataset-mixtral,openorca-mbxp-gsm8k-combined + names: + - openorca-mbxp-gsm8k-combined-preprocessed + enable_if_env: + CM_MODEL: + - mixtral-8x7b + ## Kits19 for 3d-unet - tags: get,dataset,kits19,preprocessed names: @@ -914,6 +934,9 @@ variations: - tags: get,generic-python-lib,_package.nltk names: - nltk + - tags: get,generic-python-lib,_package.numpy + names: + - numpy - tags: get,generic-python-lib,_package.rouge-score names: - rouge-score @@ -936,6 +959,52 @@ variations: base: - llama2-70b_ + mixtral-8x7b: + group: models + env: + CM_MODEL: mixtral-8x7b + deps: + - tags: get,rust-compiler + names: + - rustup + - tags: get,generic-python-lib,_package.transformers + names: + - transformers + - tags: get,generic-python-lib,_package.datasets + names: + - datasets + - tags: get,generic-python-lib,_package.sentencepiece + names: + - sentencepiece + - tags: get,generic-python-lib,_package.protobuf + names: + - protobuf + - tags: get,generic-python-lib,_package.accelerate + names: + - accelerate + - tags: get,generic-python-lib,_package.absl-py + names: + - absl-py + - tags: get,generic-python-lib,_package.evaluate + names: + - evaluate + - tags: get,generic-python-lib,_package.nltk + names: + - nltk + - tags: get,generic-python-lib,_package.rouge-score + names: + - rouge-score + - tags: get,generic-python-lib,_package.pybind11 + names: + - rouge-score + - tags: get,generic-python-lib,_mxeval + names: + - rouge-score + + mixtral-8x7b,cuda: + default_env: + CM_MLPERF_LOADGEN_BATCH_SIZE: 1 + 3d-unet-99.9: group: models base: diff --git a/script/app-mlperf-inference-mlcommons-python/customize.py b/script/app-mlperf-inference-mlcommons-python/customize.py index b172293c0f..70415fc484 100644 --- a/script/app-mlperf-inference-mlcommons-python/customize.py +++ b/script/app-mlperf-inference-mlcommons-python/customize.py @@ -70,7 +70,7 @@ def preprocess(i): x="" if os_info['platform'] == 'windows' else "'" - if "llama2-70b" in env['CM_MODEL']: + if "llama2-70b" in env['CM_MODEL'] or "mixtral-8x7b" in env["CM_MODEL"]: env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --mlperf-conf " + x+ env['CM_MLPERF_CONF'] + x else: env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --mlperf_conf "+ x + env['CM_MLPERF_CONF'] + x @@ -101,7 +101,7 @@ def preprocess(i): if 'CM_MLPERF_USER_CONF' in env: user_conf_path = env['CM_MLPERF_USER_CONF'] x="" if os_info['platform'] == 'windows' else "'" - if 'llama2-70b' in env['CM_MODEL']: + if 'llama2-70b' in env['CM_MODEL'] or "mixtral-8x7b" in env["CM_MODEL"]: scenario_extra_options += " --user-conf " + x + user_conf_path + x else: scenario_extra_options += " --user_conf " + x + user_conf_path + x @@ -307,6 +307,22 @@ def get_run_cmd_reference(os_info, env, scenario_extra_options, mode_extra_optio ' --dtype ' + env['CM_MLPERF_MODEL_PRECISION'] + \ " --model-path " + env['MODEL_DIR'] cmd = cmd.replace("--count", "--total-sample-count") + + elif "mixtral-8x7b" in env['CM_MODEL']: + env['RUN_DIR'] = os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "language", "mixtral-8x7b") + backend = env['CM_MLPERF_BACKEND'] + device = env['CM_MLPERF_DEVICE'] if env['CM_MLPERF_DEVICE'] != "gpu" else "cuda" + cmd = env['CM_PYTHON_BIN_WITH_PATH'] + " main.py " \ + " --scenario " + env['CM_MLPERF_LOADGEN_SCENARIO'] + \ + " --dataset-path " + env['CM_DATASET_PREPROCESSED_PATH'] + \ + " --device " + device.replace("cuda", "cuda:0") + \ + env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] + \ + scenario_extra_options + mode_extra_options + \ + " --output-log-dir " + env['CM_MLPERF_OUTPUT_DIR'] + \ + ' --dtype ' + env['CM_MLPERF_MODEL_PRECISION'] + \ + " --model-path " + env['MODEL_DIR'] + cmd = cmd.replace("--count", "--total-sample-count") + elif "3d-unet" in env['CM_MODEL']: env['RUN_DIR'] = env['CM_MLPERF_INFERENCE_3DUNET_PATH'] diff --git a/script/app-mlperf-inference/_cm.yaml b/script/app-mlperf-inference/_cm.yaml index 56578b746c..087a89fcba 100644 --- a/script/app-mlperf-inference/_cm.yaml +++ b/script/app-mlperf-inference/_cm.yaml @@ -227,6 +227,10 @@ variations: reference,llama2-70b_: default_variations: backend: pytorch + + reference,mixtral-8x7b: + default_variations: + backend: pytorch reference,resnet50: default_variations: @@ -349,9 +353,6 @@ variations: cnndm-accuracy-script: tags: _int32 - intel-original,gptj_,build-harness: - docker: - run: false redhat: group: @@ -417,6 +418,9 @@ variations: intel-original,resnet50: default_variations: precision: int8 + add_deps_recursive: + imagenet-accuracy-script: + tags: _int32 intel-original,retinanet: default_variations: @@ -462,9 +466,10 @@ variations: - mlperf-accuracy-script - imagenet-accuracy-script tags: run,accuracy,mlperf,_imagenet + docker: deps: - - tags: get,dataset,imagenet,validation,original + - tags: get,dataset,imagenet,validation,original,_full names: - imagenet-original - dataset-original @@ -517,6 +522,7 @@ variations: 3d-unet_: env: CM_MLPERF_MODEL_EQUAL_ISSUE_MODE: 'yes' + CM_MLPERF_INFERENCE_TEST_QPS: "0.01" posthook_deps: - enable_if_env: CM_MLPERF_LOADGEN_MODE: @@ -536,11 +542,10 @@ variations: group: model env: - CM_MODEL: - stable-diffusion-xl + CM_MODEL: stable-diffusion-xl + CM_MLPERF_INFERENCE_TEST_QPS: "0.05" default_variations: precision: float16 - backend: pytorch device: cuda add_deps_recursive: mlperf-inference-implementation: @@ -563,6 +568,7 @@ variations: llama2-70b_: env: CM_MLPERF_MODEL_EQUAL_ISSUE_MODE: 'yes' + CM_MLPERF_INFERENCE_TEST_QPS: "0.01" posthook_deps: - enable_if_env: CM_MLPERF_LOADGEN_MODE: @@ -601,6 +607,34 @@ variations: add_deps_recursive: mlperf-inference-implementation: tags: _llama2-70b-99.9 + + mixtral-8x7b: + group: + model + base: + - mixtral-8x7b + env: + CM_MODEL: + mixtral-8x7b + add_deps_recursive: + mlperf-inference-implementation: + tags: _mixtral-8x7b + env: + CM_MLPERF_MODEL_EQUAL_ISSUE_MODE: 'yes' + posthook_deps: + - enable_if_env: + CM_MLPERF_LOADGEN_MODE: + - accuracy + - all + CM_MLPERF_ACCURACY_RESULTS_DIR: + - 'on' + skip_if_env: + CM_MLPERF_IMPLEMENTATION: + - nvidia + names: + - mlperf-accuracy-script + - openorca-gsm8k-mbxp-combined-accuracy-script + tags: run,accuracy,mlperf,_openorca-gsm8k-mbxp,_int32 rnnt: group: @@ -1327,6 +1361,7 @@ docker: - "${{ CM_MLPERF_INFERENCE_RESULTS_DIR }}:${{ CM_MLPERF_INFERENCE_RESULTS_DIR }}" - "${{ CM_MLPERF_INFERENCE_SUBMISSION_DIR }}:${{ CM_MLPERF_INFERENCE_SUBMISSION_DIR }}" - "${{ GPTJ_CHECKPOINT_PATH }}:${{ GPTJ_CHECKPOINT_PATH }}" + - "${{ LLAMA2_CHECKPOINT_PATH }}:${{ LLAMA2_CHECKPOINT_PATH }}" - "${{ DLRM_DATA_PATH }}:/home/mlperf_inf_dlrmv2" skip_run_cmd: 'no' shm_size: '32gb' diff --git a/script/app-mlperf-inference/customize.py b/script/app-mlperf-inference/customize.py index 1832908280..6ab5bb51da 100644 --- a/script/app-mlperf-inference/customize.py +++ b/script/app-mlperf-inference/customize.py @@ -108,6 +108,8 @@ def postprocess(i): pass #No compliance check for now elif 'llama2-70b' in model: pass #No compliance check for now + elif 'mixtral-8x7b' in model: + pass #No compliance check for now else: pass # Not giving an error now. But accuracy paths need to be done for other benchmarks which may need the non-determinism test #return {'return': 1, 'error': f'Accuracy paths not done for model {model}'} diff --git a/script/build-dockerfile/dockerfiles/rhel_9.Dockerfile b/script/build-dockerfile/dockerfiles/rhel_9.Dockerfile deleted file mode 100644 index 4deef4ec75..0000000000 --- a/script/build-dockerfile/dockerfiles/rhel_9.Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM registry.access.redhat.com/ubi9 -SHELL ["/bin/bash", "-c"] -ARG CM_GH_TOKEN - -# Notes: https://runnable.com/blog/9-common-dockerfile-mistakes -# Install system dependencies -RUN dnf update -y -RUN dnf install -y python3 python-pip git wget sudo binutils - -# Install python packages -RUN python3 -m pip install cmind requests - -# Setup docker environment -ENTRYPOINT ["/bin/bash", "-c"] -ENV TZ=US/Pacific -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ >/etc/timezone - -# Setup docker user -RUN groupadd cm -RUN useradd -g cm --create-home --shell /bin/bash cmuser -RUN echo "cmuser ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers -USER cmuser:cm -WORKDIR /home/cmuser - -# Download CM repo for scripts -RUN cm pull repo mlcommons@ck - -# Install all system dependencies -RUN cm run script --quiet --tags=get,sys-utils-cm - -# Run command -RUN cm version diff --git a/script/build-dockerfile/dockerfiles/ubuntu_18.04.Dockerfile b/script/build-dockerfile/dockerfiles/ubuntu_18.04.Dockerfile deleted file mode 100644 index 73a9b0fd37..0000000000 --- a/script/build-dockerfile/dockerfiles/ubuntu_18.04.Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM ubuntu:18.04 -SHELL ["/bin/bash", "-c"] -ARG CM_GH_TOKEN - -# Notes: https://runnable.com/blog/9-common-dockerfile-mistakes -# Install system dependencies -RUN apt-get update -y -RUN apt-get install -y python3 python3-pip git sudo wget - -# Install python packages -RUN python3 -m pip install cmind requests - -# Setup docker environment -ENTRYPOINT ["/bin/bash", "-c"] -ENV TZ=US/Pacific -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ >/etc/timezone - -# Setup docker user -RUN groupadd cm -RUN useradd -g cm --create-home --shell /bin/bash cmuser -RUN echo "cmuser ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers -USER cmuser:cm -WORKDIR /home/cmuser - -# Download CM repo for scripts -RUN cm pull repo mlcommons@ck - -# Install all system dependencies -RUN cm run script --quiet --tags=get,sys-utils-cm - -# Run command -RUN cm version diff --git a/script/build-dockerfile/dockerfiles/ubuntu_20.04.Dockerfile b/script/build-dockerfile/dockerfiles/ubuntu_20.04.Dockerfile deleted file mode 100644 index c07b594335..0000000000 --- a/script/build-dockerfile/dockerfiles/ubuntu_20.04.Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM ubuntu:20.04 -SHELL ["/bin/bash", "-c"] -ARG CM_GH_TOKEN - -# Notes: https://runnable.com/blog/9-common-dockerfile-mistakes -# Install system dependencies -RUN apt-get update -y -RUN apt-get install -y python3 python3-pip git sudo wget - -# Install python packages -RUN python3 -m pip install cmind requests - -# Setup docker environment -ENTRYPOINT ["/bin/bash", "-c"] -ENV TZ=US/Pacific -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ >/etc/timezone - -# Setup docker user -RUN groupadd cm -RUN useradd -g cm --create-home --shell /bin/bash cmuser -RUN echo "cmuser ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers -USER cmuser:cm -WORKDIR /home/cmuser - -# Download CM repo for scripts -RUN cm pull repo mlcommons@ck - -# Install all system dependencies -RUN cm run script --quiet --tags=get,sys-utils-cm - -# Run command -RUN cm version diff --git a/script/build-dockerfile/dockerfiles/ubuntu_22.04.Dockerfile b/script/build-dockerfile/dockerfiles/ubuntu_22.04.Dockerfile deleted file mode 100644 index 05b1d69b37..0000000000 --- a/script/build-dockerfile/dockerfiles/ubuntu_22.04.Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM ubuntu:22.04 -SHELL ["/bin/bash", "-c"] -ARG CM_GH_TOKEN - -# Notes: https://runnable.com/blog/9-common-dockerfile-mistakes -# Install system dependencies -RUN apt-get update -y -RUN apt-get install -y python3 python3-pip git sudo wget - -# Install python packages -RUN python3 -m pip install cmind requests - -# Setup docker environment -ENTRYPOINT ["/bin/bash", "-c"] -ENV TZ=US/Pacific -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ >/etc/timezone - -# Setup docker user -RUN groupadd cm -RUN useradd -g cm --create-home --shell /bin/bash cmuser -RUN echo "cmuser ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers -USER cmuser:cm -WORKDIR /home/cmuser - -# Download CM repo for scripts -RUN cm pull repo mlcommons@ck - -# Install all system dependencies -RUN cm run script --quiet --tags=get,sys-utils-cm - -# Run command -RUN cm version diff --git a/script/download-file/customize.py b/script/download-file/customize.py index 78362181db..80e389d86f 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -48,7 +48,7 @@ def preprocess(i): verify_ssl = env.get('CM_VERIFY_SSL', "True") if str(verify_ssl).lower() in [ "no", "false" ]: verify_ssl = False - if tool == 'wget': + if tool == 'wget' or tool == "gdown": extra_download_options += " --no-check-certificate" else: verify_ssl = True @@ -113,7 +113,13 @@ def preprocess(i): if env.get('CM_RCLONE_CONFIG_CMD', '') != '': env['CM_DOWNLOAD_CONFIG_CMD'] = env['CM_RCLONE_CONFIG_CMD'] rclone_copy_using = env.get('CM_RCLONE_COPY_USING', 'sync') - env['CM_DOWNLOAD_CMD'] = f"rclone {rclone_copy_using} {q}{url}{q} {q}{os.path.join(os.getcwd(), env['CM_DOWNLOAD_FILENAME'])}{q} -P" + if env["CM_HOST_OS_TYPE"] == "windows": + # have to modify the variable from url to temp_url if it is going to be used anywhere after this point + url = url.replace("%", "%%") + temp_download_file = env['CM_DOWNLOAD_FILENAME'].replace("%", "%%") + env['CM_DOWNLOAD_CMD'] = f"rclone {rclone_copy_using} {q}{url}{q} {q}{os.path.join(os.getcwd(), temp_download_file)}{q} -P" + else: + env['CM_DOWNLOAD_CMD'] = f"rclone {rclone_copy_using} {q}{url}{q} {q}{os.path.join(os.getcwd(), env['CM_DOWNLOAD_FILENAME'])}{q} -P" filename = env['CM_DOWNLOAD_FILENAME'] env['CM_DOWNLOAD_DOWNLOADED_FILENAME'] = filename diff --git a/script/extract-file/customize.py b/script/extract-file/customize.py index a8da7ec0d5..af769aecf1 100644 --- a/script/extract-file/customize.py +++ b/script/extract-file/customize.py @@ -25,6 +25,9 @@ def preprocess(i): filename = env.get('CM_EXTRACT_FILEPATH','') if filename == '': return {'return': 1, 'error': 'Extract with no download requested and CM_EXTRACT_FILEPATH is not set'} + + if windows: + filename = filename.replace("%", "%%") env['CM_EXTRACT_FILENAME'] = filename diff --git a/script/get-brew/dockerfiles/ubuntu_22.04.Dockerfile b/script/get-brew/dockerfiles/ubuntu_22.04.Dockerfile deleted file mode 100644 index 15fa071d94..0000000000 --- a/script/get-brew/dockerfiles/ubuntu_22.04.Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -FROM ubuntu:22.04 -SHELL ["/bin/bash", "-c"] -ARG CM_GH_TOKEN - -# Notes: https://runnable.com/blog/9-common-dockerfile-mistakes -# Install system dependencies -RUN apt-get update -y -RUN apt-get install -y python3 python3-pip git sudo wget - -# Install python packages -RUN python3 -m pip install cmind requests - -# Setup docker environment -ENTRYPOINT ["/bin/bash", "-c"] -ENV TZ=US/Pacific -ENV PATH=${PATH}:$HOME/.local/bin -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ >/etc/timezone - -# Setup docker user -RUN groupadd cm -RUN useradd -g cm --create-home --shell /bin/bash cmuser -RUN echo "cmuser ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers -USER cmuser:cm -WORKDIR /home/cmuser - -# Download CM repo for scripts -RUN cm pull repo mlcommons@ck - -# Install all system dependencies -RUN cm run script --quiet --tags=get,sys-utils-cm - -# Run commands - -# Run CM workflow -RUN cm run script --tags=get,brew --fake_run -RUN cm run script --tags=get,brew diff --git a/script/get-conda/_cm.json b/script/get-conda/_cm.json index 3253a3ff56..a7d73f922e 100644 --- a/script/get-conda/_cm.json +++ b/script/get-conda/_cm.json @@ -13,6 +13,7 @@ "new_env_keys": [ "+PATH", "+LD_LIBRARY_PATH", + "+LIBRARY_PATH", "CM_CONDA_PREFIX", "CONDA_PREFIX", "CM_CONDA_BIN_PATH", diff --git a/script/get-conda/customize.py b/script/get-conda/customize.py index d8ef13e343..d8fcf6eacd 100644 --- a/script/get-conda/customize.py +++ b/script/get-conda/customize.py @@ -78,6 +78,8 @@ def postprocess(i): conda_bin_path = os.path.dirname(env['CM_CONDA_BIN_WITH_PATH']) env['CM_CONDA_BIN_PATH'] = conda_bin_path + env['+PATH'] = [ conda_bin_path ] + conda_prefix = os.path.dirname(conda_bin_path) env['CM_CONDA_PREFIX'] = conda_prefix env['CONDA_PREFIX'] = conda_prefix @@ -87,6 +89,7 @@ def postprocess(i): if os.path.exists(conda_lib_path): env['CM_CONDA_LIB_PATH'] = conda_lib_path env['+LD_LIBRARY_PATH'] = [ conda_lib_path ] + env['+LIBRARY_PATH'] = [ conda_lib_path ] version = r['version'] diff --git a/script/get-dataset-mixtral/_cm.json b/script/get-dataset-mixtral/_cm.json new file mode 100644 index 0000000000..e0ddd31eca --- /dev/null +++ b/script/get-dataset-mixtral/_cm.json @@ -0,0 +1,48 @@ +{ + "alias": "get-dataset-mixtral", + "automation_alias": "script", + "automation_uid": "5b4e0237da074764", + "cache": true, + "category": "AI/ML datasets", + "new_env_keys": [ + "CM_DATASET_*" + ], + "deps":[ + { + "tags": "detect,detect-os" + }, + { + "skip_if_env": { + "CM_HOST_OS_TYPE": [ + "windows" + ] + }, + "tags": "get,rclone" + }, + { + "enable_if_env": { + "CM_HOST_OS_TYPE": [ + "windows" + ] + }, + "tags": "get,rclone" + } + ], + "tags": [ + "get", + "dataset-mixtral", + "openorca-mbxp-gsm8k-combined" + ], + "uid": "89e7c91444804775", + "variations": { + "mlcommons-storage":{ + "default":true, + "env":{ + "CM_RCLONE_WINDOWS_URL": "https://inference.mlcommons-storage.org/mixtral_8x7b%%2F2024.06.06_mixtral_15k_v4.pkl", + "CM_RCLONE_LINUX_URL": "https://inference.mlcommons-storage.org/mixtral_8x7b%2F2024.06.06_mixtral_15k_v4.pkl", + "CM_DATASET_FILE_NAME": "2024.06.06_mixtral_15k_v4.pkl" + }, + "group": "download-source" + } + } +} diff --git a/script/get-dataset-mixtral/run.bat b/script/get-dataset-mixtral/run.bat new file mode 100644 index 0000000000..bf1e128dd5 --- /dev/null +++ b/script/get-dataset-mixtral/run.bat @@ -0,0 +1,7 @@ +echo. + +rclone copyurl %CM_RCLONE_WINDOWS_URL% . -a -P +IF %ERRORLEVEL% NEQ 0 EXIT 1 + +echo CM_DATASET_PREPROCESSED_PATH=%CD%\%CM_DATASET_FILE_NAME% > tmp-run-env.out +echo %CD%\%CM_DATASET_FILE_NAME% diff --git a/script/get-dataset-mixtral/run.sh b/script/get-dataset-mixtral/run.sh new file mode 100644 index 0000000000..ed3b3142f1 --- /dev/null +++ b/script/get-dataset-mixtral/run.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +echo "" + +rclone copyurl ${CM_RCLONE_LINUX_URL} ./ -a -P +test $? -eq 0 || exit 1 + +echo "CM_DATASET_PREPROCESSED_PATH=$PWD/${CM_DATASET_FILE_NAME}" > tmp-run-env.out diff --git a/script/get-dataset-openimages/_cm.json b/script/get-dataset-openimages/_cm.json index 06ae8c2bb9..3f3904076b 100644 --- a/script/get-dataset-openimages/_cm.json +++ b/script/get-dataset-openimages/_cm.json @@ -69,11 +69,14 @@ "new_env_keys": [ "CM_DATASET_PATH", "CM_DATASET_PATH_ROOT", + "CM_DATASET_OPENIMAGES_DATASET_PATH", + "CM_DATASET_OPENIMAGES_DATASET_PATH_ROOT", "CM_DATASET_ANNOTATIONS_DIR_PATH", "CM_DATASET_ANNOTATIONS_FILE_PATH", "CM_DATASET_CALIBRATION_ANNOTATIONS_FILE_PATH", "CM_DATASET_VALIDATION_ANNOTATIONS_FILE_PATH", - "CM_CALIBRATION_DATASET_PATH" + "CM_CALIBRATION_DATASET_PATH", + "CM_CALIBRATION_DATASET_PATH_ROOT" ], "tags": [ "get", @@ -113,7 +116,12 @@ ], "env": { "CM_DATASET_CALIBRATION": "yes" - } + }, + "new_env_keys": [ + "CM_DATASET_CALIBRATION_ANNOTATIONS_FILE_PATH", + "CM_CALIBRATION_DATASET_PATH", + "CM_CALIBRATION_DATASET_PATH_ROOT" + ] }, "filter": { }, @@ -136,7 +144,16 @@ "default": true, "env": { "CM_DATASET_CALIBRATION": "no" - } + }, + "new_env_keys": [ + "CM_DATASET_PATH", + "CM_DATASET_PATH_ROOT", + "CM_DATASET_OPENIMAGES_DATASET_PATH", + "CM_DATASET_OPENIMAGES_DATASET_PATH_ROOT", + "CM_DATASET_ANNOTATIONS_DIR_PATH", + "CM_DATASET_ANNOTATIONS_FILE_PATH", + "CM_DATASET_VALIDATION_ANNOTATIONS_FILE_PATH" + ] }, "size.#": { "group": "size", diff --git a/script/get-dataset-openimages/customize.py b/script/get-dataset-openimages/customize.py index 3040fff2ba..6ccb558c59 100644 --- a/script/get-dataset-openimages/customize.py +++ b/script/get-dataset-openimages/customize.py @@ -73,8 +73,11 @@ def postprocess(i): if env.get("CM_DATASET_OPENIMAGES_CUSTOM_ANNOTATIONS",'') == "yes": annotations_file_src = env['CM_DATASET_OPENIMAGES_ANNOTATIONS_FILE_PATH'] shutil.copy(annotations_file_src, env['CM_DATASET_ANNOTATIONS_DIR_PATH']) + env['CM_DATASET_OPENIMAGES_PATH'] = env['CM_DATASET_PATH'] + env['CM_DATASET_OPENIMAGES_PATH_ROOT'] = env['CM_DATASET_PATH_ROOT'] else: env['CM_CALIBRATION_DATASET_PATH'] = os.path.join(os.getcwd(), 'install', 'calibration', 'data') + env['CM_CALIBRATION_DATASET_PATH_ROOT'] = os.path.join(os.getcwd(), 'install') annotations_file_path = os.path.join(env['CM_DATASET_ANNOTATIONS_DIR_PATH'], "openimages-calibration-mlperf.json") env['CM_DATASET_CALIBRATION_ANNOTATIONS_FILE_PATH'] = annotations_file_path diff --git a/script/get-generic-python-lib/_cm.json b/script/get-generic-python-lib/_cm.json index 4b40d51d4f..603aecafb4 100644 --- a/script/get-generic-python-lib/_cm.json +++ b/script/get-generic-python-lib/_cm.json @@ -231,6 +231,13 @@ "CM_GENERIC_PYTHON_PIP_URL": "git+https://github.com/NVIDIA/dllogger#egg=dllogger" } }, + "mxeval": { + "env": { + "CM_GENERIC_PYTHON_PACKAGE_NAME": "mxeval", + "CM_GENERIC_PYTHON_PIP_URL": "git+https://github.com/amazon-science/mxeval.git", + "CM_PIP_ERROR_SKIP": "true" + } + }, "fiftyone": { "env": { "CM_GENERIC_PYTHON_PACKAGE_NAME": "fiftyone" diff --git a/script/get-generic-python-lib/customize.py b/script/get-generic-python-lib/customize.py index 6dcf770f3d..18ae0149aa 100644 --- a/script/get-generic-python-lib/customize.py +++ b/script/get-generic-python-lib/customize.py @@ -33,6 +33,9 @@ def preprocess(i): if env.get('CM_GENERIC_PYTHON_PACKAGE_INSTALL_DEPS', '') == "no": env['CM_PYTHON_PIP_COMMON_EXTRA'] = " --no-deps" + if env.get('CM_PIP_INSTALL_NEEDS_USER', '') == "yes": + env['CM_PYTHON_PIP_COMMON_EXTRA'] = " --user" + if env.get('CM_GENERIC_PYTHON_PIP_UNINSTALL_DEPS', '') != '': r = automation.run_native_script({'run_script_input':run_script_input, 'env':env, 'script_name':'uninstall_deps'}) if r['return']>0: return r diff --git a/script/get-generic-python-lib/install.sh b/script/get-generic-python-lib/install.sh index 6029190cc3..5aca7aa7ca 100644 --- a/script/get-generic-python-lib/install.sh +++ b/script/get-generic-python-lib/install.sh @@ -7,19 +7,25 @@ if [[ ${CM_GENERIC_PYTHON_PACKAGE_VARIANT} == "nvidia-apex-depreciated" ]]; then cmd="${CM_PYTHON_BIN_WITH_PATH} -m pip install -v --disable-pip-version-check --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" ./" echo $cmd eval $cmd - test $? -eq 0 || exit $? + if [[ -n ${CM_PIP_ERROR_SKIP} ]]; then + test $? -eq 0 || exit $? + fi exit 0 fi if [[ ${CM_GENERIC_PYTHON_PACKAGE_NAME} == "tensorflow_old" ]]; then if [[ ${CM_HOST_OS_FLAVOR} == "macos" ]]; then . ${CM_TMP_CURRENT_SCRIPT_PATH}/tensorflow/run-macos.sh - test $? -eq 0 || exit $? + if [[ -n ${CM_PIP_ERROR_SKIP} ]]; then + test $? -eq 0 || exit $? + fi exit 0 fi if [[ ${CM_HOST_PLATFORM_FLAVOR} == "aarch64" ]]; then . ${CM_TMP_CURRENT_SCRIPT_PATH}/tensorflow/run-aarch64.sh - test $? -eq 0 || exit $? + if [[ -n ${CM_PIP_ERROR_SKIP} ]]; then + test $? -eq 0 || exit $? + fi exit 0 fi fi @@ -27,11 +33,16 @@ if [[ -n ${CM_GENERIC_PYTHON_PIP_URL} ]]; then cmd="${CM_PYTHON_BIN_WITH_PATH} -m pip install \"${CM_GENERIC_PYTHON_PIP_URL}\" ${CM_GENERIC_PYTHON_PIP_EXTRA}" echo $cmd eval $cmd - test $? -eq 0 || exit $? + if [[ -n ${CM_PIP_ERROR_SKIP} ]]; then + test $? -eq 0 || exit $? + fi exit 0 fi cmd="${CM_PYTHON_BIN_WITH_PATH} -m pip install \"${CM_GENERIC_PYTHON_PACKAGE_NAME}${CM_TMP_PIP_VERSION_STRING}\" ${CM_GENERIC_PYTHON_PIP_EXTRA}" echo $cmd eval $cmd -test $? -eq 0 || exit $? +if [[ -n ${CM_PIP_ERROR_SKIP} ]]; then + test $? -eq 0 || exit $? +fi +exit 0 diff --git a/script/get-git-repo/_cm.json b/script/get-git-repo/_cm.json index fffaabd108..dcb2ce9445 100644 --- a/script/get-git-repo/_cm.json +++ b/script/get-git-repo/_cm.json @@ -107,6 +107,11 @@ "CM_GIT_PR_TO_APPLY": "#" } }, + "cherrypicks.#": { + "env": { + "CM_GIT_CHERRYPICKS": "#" + } + }, "repo.#": { "group": "repo", "env": { diff --git a/script/get-git-repo/run.sh b/script/get-git-repo/run.sh index 2080b72eb6..87803526d2 100644 --- a/script/get-git-repo/run.sh +++ b/script/get-git-repo/run.sh @@ -1,6 +1,7 @@ #!/bin/bash CUR_DIR=$PWD +echo "$CUR_DIR" SCRIPT_DIR=${CM_TMP_CURRENT_SCRIPT_PATH} folder=${CM_GIT_CHECKOUT_FOLDER} @@ -50,11 +51,20 @@ else fi if [ ! -z ${CM_GIT_PR_TO_APPLY} ]; then + echo "" + echo "Fetching from ${CM_GIT_PR_TO_APPLY}" git fetch origin ${CM_GIT_PR_TO_APPLY}:tmp-apply - git config user.email "temp@example.com" - git merge tmp-apply --no-edit fi +IFS=',' read -r -a cherrypicks <<< "${CM_GIT_CHERRYPICKS}" +for cherrypick in "${cherrypicks[@]}" +do + echo "" + echo "Applying cherrypick $cherrypick" + git cherry-pick -n $cherrypick + test $? -eq 0 || exit $? +done + IFS=',' read -r -a submodules <<< "${CM_GIT_SUBMODULES}" for submodule in "${submodules[@]}" diff --git a/script/get-ml-model-3d-unet-kits19/_cm.json b/script/get-ml-model-3d-unet-kits19/_cm.json index 076d18c5f6..13de73c7e6 100644 --- a/script/get-ml-model-3d-unet-kits19/_cm.json +++ b/script/get-ml-model-3d-unet-kits19/_cm.json @@ -60,7 +60,7 @@ "pytorch,fp32,weights": { "env": { "CM_ML_MODEL_ACCURACY": "0.86170", - "CM_ML_MODEL_FILE": "retinanet_model_10.pth", + "CM_ML_MODEL_FILE": "3dunet_kits19_pytorch_checkpoint.pth", "CM_PACKAGE_URL": "https://zenodo.org/record/5597155/files/3dunet_kits19_pytorch_checkpoint.pth?download=1", "CM_UNZIP": "yes" } diff --git a/script/get-ml-model-llama2/_cm.json b/script/get-ml-model-llama2/_cm.json index 330465f8c1..3fc4381b97 100644 --- a/script/get-ml-model-llama2/_cm.json +++ b/script/get-ml-model-llama2/_cm.json @@ -11,6 +11,9 @@ "input_mapping": { "checkpoint": "LLAMA2_CHECKPOINT_PATH" }, + "docker": { + "real_run": false + }, "new_env_keys": [ "CM_ML_MODEL_*", "LLAMA2_CHECKPOINT_PATH" diff --git a/script/get-ml-model-llama2/customize.py b/script/get-ml-model-llama2/customize.py index 09f8148bf6..70a8974712 100644 --- a/script/get-ml-model-llama2/customize.py +++ b/script/get-ml-model-llama2/customize.py @@ -17,7 +17,9 @@ def postprocess(i): env = i['env'] - env['LLAMA2_CHECKPOINT_PATH'] = env['CM_ML_MODEL_PATH'] + if env.get('LLAMA2_CHECKPOINT_PATH', '' ) == '': + env['LLAMA2_CHECKPOINT_PATH'] = env['CM_ML_MODEL_PATH'] + env['CM_ML_MODEL_PATH'] = env['LLAMA2_CHECKPOINT_PATH'] env['CM_ML_MODEL_LLAMA2_FILE_WITH_PATH'] = env['LLAMA2_CHECKPOINT_PATH'] env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_ML_MODEL_PATH'] diff --git a/script/get-ml-model-mixtral/_cm.json b/script/get-ml-model-mixtral/_cm.json new file mode 100644 index 0000000000..6ef6359cb3 --- /dev/null +++ b/script/get-ml-model-mixtral/_cm.json @@ -0,0 +1,95 @@ +{ + "alias": "get-ml-model-mixtral", + "automation_alias": "script", + "automation_uid": "5b4e0237da074764", + "cache": true, + "category": "AI/ML models", + "env": { + "CM_ML_MODEL_DATASET": "", + "CM_ML_MODEL_WEIGHT_TRANSFORMATIONS": "no" + }, + "input_mapping": { + "checkpoint": "MIXTRAL_CHECKPOINT_PATH" + }, + "new_env_keys": [ + "CM_ML_MODEL_*", + "MIXTRAL_CHECKPOINT_PATH" + ], + "prehook_deps": [ + { + "enable_if_env": { + "CM_TMP_REQUIRE_DOWNLOAD": [ + "yes" + ] + }, + "env": {}, + "extra_cache_tags": "mixtral", + "force_env_keys": [ + "CM_GIT_CHECKOUT_FOLDER" + ], + "names": [ + "hf-zoo" + ], + "tags": "get,ml-model,huggingface,zoo,_clone-repo" + } + ], + "print_env_at_the_end": { + "MIXTRAL_CHECKPOINT_PATH": "MIXTRAL checkpoint path" + }, + "tags": [ + "get", + "raw", + "ml-model", + "language-processing", + "mixtral", + "mixtral-8x7b" + ], + "uid": "0c14127677f34ea2", + "variations": { + "batch_size.#": { + "env": { + "CM_ML_MODEL_BATCH_SIZE": "#" + } + }, + "fp32": { + "default": true, + "env": { + "CM_ML_MODEL_INPUT_DATA_TYPES": "fp32", + "CM_ML_MODEL_PRECISION": "fp32", + "CM_ML_MODEL_WEIGHT_DATA_TYPES": "fp32" + }, + "group": "precision" + }, + "mistralai/Mixtral-8x7B-Instruct-v0.1": { + "adr": { + "hf-zoo": { + "tags": "_model-stub.mistralai/Mixtral-8x7B-Instruct-v0.1" + } + }, + "default": true, + "env": { + "CM_GIT_CHECKOUT_FOLDER": "Mixtral-8x7B-Instruct-v0.1", + "CM_MODEL_ZOO_ENV_KEY": "MIXTRAL" + }, + "group": "huggingface-stub" + }, + "pytorch": { + "default": true, + "env": { + "CM_ML_MODEL_FRAMEWORK": "pytorch" + }, + "group": "framework" + }, + "stub.#": { + "adr": { + "hf-zoo": { + "tags": "_model-stub.#" + } + }, + "env": { + "CM_MODEL_ZOO_ENV_KEY": "MIXTRAL" + }, + "group": "huggingface-stub" + } + } +} diff --git a/script/get-ml-model-mixtral/customize.py b/script/get-ml-model-mixtral/customize.py new file mode 100644 index 0000000000..8b1c0af52d --- /dev/null +++ b/script/get-ml-model-mixtral/customize.py @@ -0,0 +1,24 @@ +from cmind import utils +import os + +def preprocess(i): + + os_info = i['os_info'] + env = i['env'] + + path = env.get('MIXTRAL_CHECKPOINT_PATH', '').strip() + + if path == '' or not os.path.exists(path): + env['CM_TMP_REQUIRE_DOWNLOAD'] = 'yes' + + return {'return':0} + +def postprocess(i): + + env = i['env'] + + env['MIXTRAL_CHECKPOINT_PATH'] = env['CM_ML_MODEL_PATH'] + env['CM_ML_MODEL_PATH'] = env['MIXTRAL_CHECKPOINT_PATH'] + env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_ML_MODEL_PATH'] + + return {'return':0} diff --git a/script/get-mlperf-inference-loadgen/run.sh b/script/get-mlperf-inference-loadgen/run.sh index a1828add8e..d9acb121b8 100644 --- a/script/get-mlperf-inference-loadgen/run.sh +++ b/script/get-mlperf-inference-loadgen/run.sh @@ -48,12 +48,12 @@ MLPERF_INFERENCE_PYTHON_SITE_BASE=${INSTALL_DIR}"/python" cd "${CM_MLPERF_INFERENCE_SOURCE}/loadgen" #CFLAGS="-std=c++14 -O3" ${CM_PYTHON_BIN_WITH_PATH} setup.py bdist_wheel #${CM_PYTHON_BIN_WITH_PATH} -m pip install --force-reinstall `ls dist/mlperf_loadgen-*cp3${PYTHON_MINOR_VERSION}*.whl` --target="${MLPERF_INFERENCE_PYTHON_SITE_BASE}" -${CM_PYTHON_BIN_WITH_PATH} -m pip install . +${CM_PYTHON_BIN_WITH_PATH} -m pip install . --target="${MLPERF_INFERENCE_PYTHON_SITE_BASE}" if [ "${?}" != "0" ]; then exit 1; fi # Clean the built wheel -find . -name 'mlcommons_loadgen*.whl' | xargs rm +#find . -name 'mlcommons_loadgen*.whl' | xargs rm echo "******************************************************" echo "Loadgen is built and installed to ${INSTALL_DIR} ..." diff --git a/script/get-mlperf-inference-results-dir/_cm.json b/script/get-mlperf-inference-results-dir/_cm.json index 3e9eb912b5..8017dbca8a 100644 --- a/script/get-mlperf-inference-results-dir/_cm.json +++ b/script/get-mlperf-inference-results-dir/_cm.json @@ -36,11 +36,11 @@ "CM_MLPERF_INFERENCE_RESULTS_VERSION": "#" } }, - "version.4_0": { + "version.4_1": { "group": "version", "default": true, "env": { - "CM_MLPERF_INFERENCE_RESULTS_VERSION": "4_0" + "CM_MLPERF_INFERENCE_RESULTS_VERSION": "4_1" } } }, diff --git a/script/get-mlperf-inference-src/_cm.json b/script/get-mlperf-inference-src/_cm.json index 82ba0d9338..9bd2f4c3aa 100644 --- a/script/get-mlperf-inference-src/_cm.json +++ b/script/get-mlperf-inference-src/_cm.json @@ -195,7 +195,7 @@ "versions": { "custom": { "env": { - "CM_MLPERF_LAST_RELEASE": "v4.0" + "CM_MLPERF_LAST_RELEASE": "v4.1" } }, "deepsparse": { @@ -208,13 +208,13 @@ "main": { "env": { "CM_TMP_GIT_CHECKOUT": "main", - "CM_MLPERF_LAST_RELEASE": "v4.0" + "CM_MLPERF_LAST_RELEASE": "v4.1" } }, "master": { "env": { "CM_TMP_GIT_CHECKOUT": "master", - "CM_MLPERF_LAST_RELEASE": "v4.0" + "CM_MLPERF_LAST_RELEASE": "v4.1" } }, "r2.1": { diff --git a/script/get-mlperf-inference-submission-dir/_cm.json b/script/get-mlperf-inference-submission-dir/_cm.json index 3c2f648f06..f056fd4c3c 100644 --- a/script/get-mlperf-inference-submission-dir/_cm.json +++ b/script/get-mlperf-inference-submission-dir/_cm.json @@ -36,10 +36,10 @@ }, "group": "version" }, - "version.4_0": { + "version.4_1": { "default": true, "env": { - "CM_MLPERF_INFERENCE_SUBMISSION_VERSION": "4_0" + "CM_MLPERF_INFERENCE_SUBMISSION_VERSION": "4_1" }, "group": "version" } diff --git a/script/get-preprocessed-dataset-generic/src/preprocess_object_detection_dataset.py b/script/get-preprocessed-dataset-generic/src/preprocess_object_detection_dataset.py index 84e18ee397..ef28014b94 100644 --- a/script/get-preprocessed-dataset-generic/src/preprocess_object_detection_dataset.py +++ b/script/get-preprocessed-dataset-generic/src/preprocess_object_detection_dataset.py @@ -38,7 +38,7 @@ def load_image(image_path, target_size, data_type='uint8', convert_to_bgr=False, def quantize_to_uint8(image, scale, offset): quantized_image = (image.astype(np.float64) / scale + offset).astype(np.float64) - output = np.round_(quantized_image) + output = np.round(quantized_image) output = np.clip(output, 0, 255) return output.astype(np.uint8) diff --git a/script/get-preprocessed-dataset-openorca/_cm.json b/script/get-preprocessed-dataset-openorca/_cm.json index 7934799c70..94a1336a05 100644 --- a/script/get-preprocessed-dataset-openorca/_cm.json +++ b/script/get-preprocessed-dataset-openorca/_cm.json @@ -110,5 +110,8 @@ }, "group": "dataset-type" } + }, + "docker": { + "real_run": false } } diff --git a/script/get-rclone/_cm.json b/script/get-rclone/_cm.json index 31e1e0a9a5..39f60ef1ab 100644 --- a/script/get-rclone/_cm.json +++ b/script/get-rclone/_cm.json @@ -6,6 +6,14 @@ "deps": [ { "tags": "detect,os" + }, + { + "enable_if_env": { + "CM_HOST_OS_TYPE": [ + "windows" + ] + }, + "tags": "get,sys-utils-min" } ], "category": "Detection or installation of tools and artifacts", diff --git a/script/get-rclone/customize.py b/script/get-rclone/customize.py index 84804d6e59..eef1ea9be6 100644 --- a/script/get-rclone/customize.py +++ b/script/get-rclone/customize.py @@ -53,7 +53,15 @@ def preprocess(i): cur_dir = os.getcwd() path_bin = os.path.join(cur_dir, file_name) env['CM_RCLONE_BIN_WITH_PATH'] = path_bin - env['+PATH']=[cur_dir] + + if not env.get('+PATH', []): + env['+PATH'] = [] + env['+PATH'].append(cur_dir) + + if not env.get('+PATH', []): + env['+PATH'] = [] + env['+PATH'].append(cur_dir) + r = automation.run_native_script({'run_script_input':run_script_input, 'env':env, diff --git a/script/install-generic-conda-package/_cm.json b/script/install-generic-conda-package/_cm.json index 7cc13f5e28..4cefd679d4 100644 --- a/script/install-generic-conda-package/_cm.json +++ b/script/install-generic-conda-package/_cm.json @@ -48,6 +48,14 @@ ], "uid": "d9275487f5314195", "variations": { + "package.python": { + "env": { + "CM_CONDA_PKG_NAME": "python" + }, + "new_env_keys": [ + "CM_PYTHON_BIN_WITH_PATH" + ] + }, "package.#": { "env": { "CM_CONDA_PKG_NAME": "#" diff --git a/script/install-generic-conda-package/customize.py b/script/install-generic-conda-package/customize.py index 5f7905d592..2e6486a54d 100644 --- a/script/install-generic-conda-package/customize.py +++ b/script/install-generic-conda-package/customize.py @@ -38,4 +38,7 @@ def postprocess(i): env = i['env'] version = env.get('CM_VERSION', '') + if env['CM_CONDA_PKG_NAME'] == "python": + env['CM_PYTHON_BIN_WITH_PATH'] = os.path.join(os.path.dirname(env['CM_CONDA_BIN_WITH_PATH']), "python") + return {'return':0, 'version': version} diff --git a/script/install-ipex-from-src/_cm.json b/script/install-ipex-from-src/_cm.json index ba6d9f6184..d9f4c358f8 100644 --- a/script/install-ipex-from-src/_cm.json +++ b/script/install-ipex-from-src/_cm.json @@ -152,6 +152,83 @@ "for-intel-mlperf-inference-v3.1-resnet50": { "alias": "for-intel-mlperf-inference-resnet50" }, + "for-intel-mlperf-inference-v3.1-retinanet": { + "alias": "for-intel-mlperf-inference-retinanet" + }, + "for-intel-mlperf-inference-v3.1-3d-unet": { + "alias": "for-intel-mlperf-inference-3d-unet" + }, + "for-intel-mlperf-inference-retinanet": { + "adr": { + "conda-package": { + "tags": "_name.retinanet-pt" + }, + "pytorch": { + "tags": "_for-intel-mlperf-inference-retinanet" + } + }, + "base": [ + "tag.v1.12.0" + ], + "deps": [ + { + "names": [ + "conda" + ], + "tags": "get,conda,_name.retinanet-pt" + }, + { + "names": [ + "conda-package", + "python3" + ], + "tags": "get,generic,conda-package,_package.python", + "version": "3.9" + }, + { + "names": [ + "conda-package", + "wheel" + ], + "tags": "get,generic,conda-package,_package.wheel,_source.conda-forge" + }, + { + "names": [ + "conda-package", + "setuptools" + ], + "version": "69.5.1", + "tags": "get,generic,conda-package,_package.setuptools,_source.conda-forge" + }, + { + "names": [ + "conda-package", + "typing-extensions" + ], + "tags": "get,generic,conda-package,_package.typing-extensions,_source.conda-forge" + }, + { + "names": [ + "conda-package", + "sympy" + ], + "tags": "get,generic,conda-package,_package.sympy,_source.conda-forge" + }, + { + "tags": "get,cmake", + "version_max": "3.26.4" + }, + { + "tags": "get,gcc", + "version_max": "12.3", + "version_max_usable": "12.3" + } + ], + "env": { + "CM_CONDA_ENV": "yes", + "CM_INTEL_IPEX_RETINANET_PATCH": "yes" + } + }, "for-intel-mlperf-inference-resnet50": { "adr": { "conda-package": { @@ -208,21 +285,96 @@ ], "tags": "get,generic,conda-package,_package.sympy,_source.conda-forge" }, - { - "tags": "get,cmake", - "version_max": "3.26.4" - }, - { - "tags": "get,gcc", - "version_max": "12.3", - "version_max_usable": "12.3" - } + { + "tags": "get,cmake", + "version_max": "3.26.4" + }, + { + "tags": "get,gcc", + "version_max": "12.3", + "version_max_usable": "12.3" + } ], "env": { "CM_CONDA_ENV": "yes", "CM_INTEL_IPEX_RESNET50_PATCH": "yes" } }, + "for-intel-mlperf-inference-3d-unet": { + "adr": { + "conda-package": { + "tags": "_name.3d-unet-pt" + }, + "pytorch": { + "tags": "_for-intel-mlperf-inference-3d-unet" + } + }, + "base": [ + "branch.1.9.0-rc" + ], + "deps": [ + { + "tags": "get,generic-sys-util,_libffi7" + }, + { + "names": [ + "conda" + ], + "tags": "get,conda,_name.3d-unet-pt" + }, + { + "names": [ + "conda-package", + "python3" + ], + "tags": "get,generic,conda-package,_package.python", + "version": "3.8" + }, + { + "names": [ + "conda-package", + "wheel" + ], + "tags": "get,generic,conda-package,_package.wheel,_source.conda-forge" + }, + { + "names": [ + "conda-package", + "setuptools" + ], + "version": "69.5.1", + "tags": "get,generic,conda-package,_package.setuptools,_source.conda-forge" + }, + { + "names": [ + "conda-package", + "typing-extensions" + ], + "tags": "get,generic,conda-package,_package.typing-extensions,_source.conda-forge" + }, + { + "names": [ + "conda-package", + "sympy" + ], + "tags": "get,generic,conda-package,_package.sympy,_source.conda-forge" + }, + { + "tags": "get,cmake", + "version_max": "3.26.4" + }, + { + "tags": "get,gcc", + "version_max": "12.3", + "version_max_usable": "12.3" + } + ], + "env": { + "CM_CONDA_ENV": "yes", + "CM_INTEL_IPEX_3D_UNET_PATCH": "yes", + "CM_IPEX_SKIP_PYTORCH": "yes" + } + }, "repo.#": { "env": { "CM_GIT_URL": "#" diff --git a/script/install-ipex-from-src/apply_intel_3d-unet_patch.sh b/script/install-ipex-from-src/apply_intel_3d-unet_patch.sh new file mode 100644 index 0000000000..f5a1b315ac --- /dev/null +++ b/script/install-ipex-from-src/apply_intel_3d-unet_patch.sh @@ -0,0 +1,5 @@ +rm -rf unet3d.patch +wget -nc https://raw.githubusercontent.com/mlcommons/inference_results_v3.1/main/closed/Intel/code/3d-unet-99/pytorch-cpu/unet3d.patch +test $? -eq 0 || exit $? +git apply unet3d.patch +test $? -eq 0 || exit $? diff --git a/script/install-ipex-from-src/apply_intel_retinanet_patch.sh b/script/install-ipex-from-src/apply_intel_retinanet_patch.sh new file mode 100644 index 0000000000..d219846fc1 --- /dev/null +++ b/script/install-ipex-from-src/apply_intel_retinanet_patch.sh @@ -0,0 +1,5 @@ +rm runtime_ignore_dequant_check.patch +wget -nc https://raw.githubusercontent.com/mlcommons/inference_results_v3.1/main/closed/Intel/code/retinanet/pytorch-cpu/runtime_ignore_dequant_check.patch +test $? -eq 0 || exit $? +git apply runtime_ignore_dequant_check.patch +test $? -eq 0 || exit $? diff --git a/script/install-ipex-from-src/run.sh b/script/install-ipex-from-src/run.sh index b079211e6a..b54041a89a 100644 --- a/script/install-ipex-from-src/run.sh +++ b/script/install-ipex-from-src/run.sh @@ -6,11 +6,24 @@ rm -rf ipex_src cp -r ${IPEX_DIR} ipex_src cd ipex_src +git submodule sync git submodule update --init --recursive -if [[ ${aCM_INTEL_IPEX_RESNET50_PATCH} == "yes" ]]; then +if [[ ${CM_INTEL_IPEX_RESNET50_PATCH} == "yes" ]]; then bash ${CM_TMP_CURRENT_SCRIPT_PATH}/apply_intel_resnet50_patch.sh test "$?" -eq 0 || exit "$?" + +elif [[ ${CM_INTEL_IPEX_RETINANET_PATCH} == "yes" ]]; then + bash ${CM_TMP_CURRENT_SCRIPT_PATH}/apply_intel_retinanet_patch.sh + test "$?" -eq 0 || exit "$?" + +elif [[ ${CM_INTEL_IPEX_3D_UNET_PATCH} == "yes" ]]; then + cd third_party/mkl-dnn + git fetch --tags && git checkout v2.7 + test "$?" -eq 0 || exit "$?" + cd ../../ + bash ${CM_TMP_CURRENT_SCRIPT_PATH}/apply_intel_3d-unet_patch.sh + test "$?" -eq 0 || exit "$?" fi rm -rf build diff --git a/script/install-pytorch-from-src/_cm.json b/script/install-pytorch-from-src/_cm.json index 445a82bc1e..2f3440db30 100644 --- a/script/install-pytorch-from-src/_cm.json +++ b/script/install-pytorch-from-src/_cm.json @@ -21,6 +21,10 @@ "python3" ] }, + { + "tags": "get,compiler", + "names": [ "compiler" ] + }, { "env": { "CM_GIT_CHECKOUT_PATH_ENV_NAME": "CM_PYTORCH_SRC_REPO_PATH" @@ -102,11 +106,20 @@ "env": { }, "ad": { - "pytorch-src-repo": { + "pytorch-src-repo": { "tags": "_pr-to-apply.#" } } }, + "cherrypicks.#": { + "env": { + }, + "ad": { + "pytorch-src-repo": { + "tags": "_cherrypicks.#" + } + } + }, "for-intel-mlperf-inference-resnet50": { "base": [ "tag.v1.12.0-rc7", @@ -120,7 +133,7 @@ "env": { "CM_CONDA_ENV": "yes", "CM_MLPERF_INFERENCE_INTEL": "yes", - "CM_MLPERF_INFERENCE_INTEL_RESNET50_MODEL": "yes", + "CM_MLPERF_INFERENCE_INTEL_MODEL": "resnet50", "USE_CUDA": "0" }, "deps": [ @@ -153,7 +166,74 @@ "conda-package", "cmake" ], - "version_min": "3.26", + "version_min": "3.26", + "tags": "get,generic,conda-package,_package.cmake" + }, + { + "tags": "get,generic-python-lib,_package.numpy", + "names": [ + "pip-package", + "numpy" + ], + "version_max": "1.26.4" + }, + { + "names": [ + "conda-package", + "libstdcxx-ng" + ], + "tags": "get,generic,conda-package,_package.libstdcxx-ng,_source.conda-forge" + } + ] + }, + "for-intel-mlperf-inference-retinanet": { + "base": [ + "tag.v1.12.0-rc7", + "pr-to-apply.pull/89925/head", + "cherrypicks.78cad998e505b667d25ac42f8aaa24409f5031e1" + ], + "adr": { + "conda-package": { + "tags": "_name.retinanet-pt" + } + }, + "env": { + "CM_CONDA_ENV": "yes", + "CM_MLPERF_INFERENCE_INTEL": "yes", + "CM_MLPERF_INFERENCE_INTEL_MODEL": "retinanet", + "USE_CUDA": "0" + }, + "deps": [ + { + "tags": "get,conda,_name.retinanet-pt", + "names": [ "conda" ] + }, + { + "names": [ + "conda-package", + "ncurses" + ], + "tags": "get,generic,conda-package,_package.ncurses,_source.conda-forge" + }, + { + "names": [ + "conda-package", + "python3" + ], + "tags": "get,generic,conda-package,_package.python", + "version": "3.9" + }, + { + "tags": "get,gcc", + "version_max": "11.9", + "version_max_usable": "11.3" + }, + { + "names": [ + "conda-package", + "cmake" + ], + "version_min": "3.26", "tags": "get,generic,conda-package,_package.cmake" }, { diff --git a/script/install-pytorch-from-src/customize.py b/script/install-pytorch-from-src/customize.py index c8e6c99574..67f2673d33 100644 --- a/script/install-pytorch-from-src/customize.py +++ b/script/install-pytorch-from-src/customize.py @@ -15,9 +15,8 @@ def preprocess(i): run_cmd="CC=clang CXX=clang++ USE_CUDA=OFF python -m pip install -e . " env['CM_RUN_CMD'] = run_cmd - elif env.get('CM_MLPERF_INFERENCE_INTEL_RESNET50_MODEL', '') == "yes": - i['run_script_input']['script_name'] = "run-intel-mlperf-inference-resnet50" - run_cmd="CC=clang CXX=clang++ USE_CUDA=OFF python -m pip install -e . " + elif env.get('CM_MLPERF_INFERENCE_INTEL_MODEL', '') in [ "resnet50", "retinanet" ]: + i['run_script_input']['script_name'] = "run-intel-mlperf-inference-vision" run_cmd=f"CC={env['CM_C_COMPILER_WITH_PATH']} CXX={env['CM_CXX_COMPILER_WITH_PATH']} USE_CUDA=OFF python -m pip install -e . " env['CM_RUN_CMD'] = run_cmd diff --git a/script/install-pytorch-from-src/run-intel-mlperf-inference-resnet50.sh b/script/install-pytorch-from-src/run-intel-mlperf-inference-vision.sh similarity index 69% rename from script/install-pytorch-from-src/run-intel-mlperf-inference-resnet50.sh rename to script/install-pytorch-from-src/run-intel-mlperf-inference-vision.sh index c828a61327..f3bd3d771c 100644 --- a/script/install-pytorch-from-src/run-intel-mlperf-inference-resnet50.sh +++ b/script/install-pytorch-from-src/run-intel-mlperf-inference-vision.sh @@ -1,21 +1,7 @@ #!/bin/bash -export PATH=${CM_CONDA_BIN_PATH}:$PATH - - -gcc() -{ - ${CM_GCC_BIN_WITH_PATH} "$@" -} -export -f gcc - -g++() -{ - ${CM_CXX_COMPILER_WITH_PATH} "$@" -} -export -f g++ - -#exit 1 +#export PATH=${CM_CONDA_BIN_PATH}:$PATH +#export LIBRARY_PATH=${CM_CONDA_LIB_PATH}:$LIBRARY_PATH CUR_DIR=$PWD rm -rf pytorch diff --git a/script/install-torchvision-from-src/_cm.json b/script/install-torchvision-from-src/_cm.json index f0e72134b7..50aa95dee8 100644 --- a/script/install-torchvision-from-src/_cm.json +++ b/script/install-torchvision-from-src/_cm.json @@ -23,6 +23,10 @@ }, "tags": "get,python3" }, + { + "tags": "get,compiler", + "names": [ "compiler" ] + }, { "env": { "CM_GIT_CHECKOUT_PATH_ENV_NAME": "CM_PYTORCH_VISION_SRC_REPO_PATH" @@ -74,6 +78,11 @@ "CM_GIT_CHECKOUT": "#" } }, + "python.#": { + "env": { + "CM_PYTHON_BIN_WITH_PATH": "#" + } + }, "cuda": { "deps": [ { diff --git a/script/install-torchvision-from-src/run.sh b/script/install-torchvision-from-src/run.sh index 06de574ada..3ba73deee8 100644 --- a/script/install-torchvision-from-src/run.sh +++ b/script/install-torchvision-from-src/run.sh @@ -7,8 +7,8 @@ cd pytorchvision test "${?}" -eq "0" || exit $? rm -rf build -python setup.py bdist_wheel +${CM_PYTHON_BIN_WITH_PATH} setup.py bdist_wheel test "${?}" -eq "0" || exit $? cd dist -python3 -m pip install torchvision*linux_x86_64.whl +${CM_PYTHON_BIN_WITH_PATH} -m pip install torchvision*linux_x86_64.whl test "${?}" -eq "0" || exit $? diff --git a/script/process-mlperf-accuracy/_cm.json b/script/process-mlperf-accuracy/_cm.json index 351434e7f8..2cc6631a77 100644 --- a/script/process-mlperf-accuracy/_cm.json +++ b/script/process-mlperf-accuracy/_cm.json @@ -45,7 +45,7 @@ }, { "tags": "get,generic-python-lib,_package.datasets", - "names": + "names": [ "pip-package", "datasets" @@ -85,55 +85,55 @@ }, { "tags": "download,file,_url.https://raw.githubusercontent.com/mlcommons/inference_results_v4.0/main/closed/Intel/code/gptj-99/ITREX/evaluation.py", - "enable_if_env": { - "CM_MLPERF_IMPLEMENTATION": - [ - "intel" - ] + "enable_if_env": { + "CM_MLPERF_IMPLEMENTATION": + [ + "intel" + ] + }, + "force_cache": true, + "extra_cache_tags": "intel,accuracy,file,gptj,mlperf,inference", + "env": { + "CM_DOWNLOAD_FINAL_ENV_NAME": "CM_MLPERF_INFERENCE_INTEL_GPTJ_ACCURACY_FILE_WITH_PATH" + } }, - "force_cache": true, - "extra_cache_tags": "intel,accuracy,file,gptj,mlperf,inference", - "env": { - "CM_DOWNLOAD_FINAL_ENV_NAME": "CM_MLPERF_INFERENCE_INTEL_GPTJ_ACCURACY_FILE_WITH_PATH" - } - }, { "tags": "download,file,_url.https://raw.githubusercontent.com/mlcommons/inference_results_v4.0/main/closed/Intel/code/gptj-99/ITREX/dataset.py", - "enable_if_env": { - "CM_MLPERF_IMPLEMENTATION": - [ - "intel" - ] + "enable_if_env": { + "CM_MLPERF_IMPLEMENTATION": + [ + "intel" + ] + }, + "force_cache": true, + "extra_cache_tags": "intel,dataset,file,gptj,mlperf,inference", + "env": { + "CM_DOWNLOAD_FINAL_ENV_NAME": "CM_MLPERF_INFERENCE_INTEL_GPTJ_DATASET_FILE_WITH_PATH" + } }, - "force_cache": true, - "extra_cache_tags": "intel,dataset,file,gptj,mlperf,inference", - "env": { - "CM_DOWNLOAD_FINAL_ENV_NAME": "CM_MLPERF_INFERENCE_INTEL_GPTJ_DATASET_FILE_WITH_PATH" - } - }, { "tags": "download,file,_url.https://raw.githubusercontent.com/mlcommons/inference_results_v4.0/main/closed/Intel/code/gptj-99/ITREX/item.py", - "enable_if_env": { - "CM_MLPERF_IMPLEMENTATION": - [ - "intel" - ] + "enable_if_env": { + "CM_MLPERF_IMPLEMENTATION": + [ + "intel" + ] + }, + "force_cache": true, + "extra_cache_tags": "intel,dataset,item,file,gptj,mlperf,inference", + "env": { + "CM_DOWNLOAD_FINAL_ENV_NAME": "CM_MLPERF_INFERENCE_INTEL_GPTJ_DATASET_ITEM_FILE_WITH_PATH" + } }, - "force_cache": true, - "extra_cache_tags": "intel,dataset,item,file,gptj,mlperf,inference", - "env": { - "CM_DOWNLOAD_FINAL_ENV_NAME": "CM_MLPERF_INFERENCE_INTEL_GPTJ_DATASET_ITEM_FILE_WITH_PATH" - } - }, { "tags": "get,ml-model,gptj,_fp32,_pytorch", - "enable_if_env": { - "CM_MLPERF_IMPLEMENTATION": - [ - "intel" - ] + "enable_if_env": { + "CM_MLPERF_IMPLEMENTATION": + [ + "intel" + ] + } } - } ], "env": { "CM_DATASET": "cnndm" @@ -202,6 +202,11 @@ "deps": [ { "tags": "get,dataset,preprocessed,medical-imaging,kits19" + }, + { + "tags": "get,generic-python-lib,_pandas", + "version_max": "1.53.0", + "version_max_usable": "1.53.0" } ], "env": { @@ -326,6 +331,26 @@ }, "group": "dataset" }, + "openorca-gsm8k-mbxp": { + "deps": [ + { + "names": [ + "openorca-gsm8k-mbxp-combined" + ], + "tags": "get,dataset-mixtral,openorca-mbxp-gsm8k-combined" + }, + { + "names": [ + "mixtral-8x7b-model" + ], + "tags": "get,ml-model,mixtral" + } + ], + "env": { + "CM_DATASET": "openorca-gsm8k-mbxp-combined" + }, + "group": "dataset" + }, "coco2014": { "deps": [ { diff --git a/script/process-mlperf-accuracy/customize.py b/script/process-mlperf-accuracy/customize.py index 25f81a0921..1dd68becac 100644 --- a/script/process-mlperf-accuracy/customize.py +++ b/script/process-mlperf-accuracy/customize.py @@ -79,6 +79,12 @@ def preprocess(i): "evaluate-accuracy.py") CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + accuracy_checker_file + "' --checkpoint-path '" + env['CM_ML_MODEL_LLAMA2_FILE_WITH_PATH'] + "' --mlperf-accuracy-file '" + os.path.join(result_dir, "mlperf_log_accuracy.json") + \ "' --dataset-file '" + env['CM_DATASET_PREPROCESSED_PATH'] + "'"+ " --dtype " + env.get('CM_ACCURACY_DTYPE', "int32") +" > '" + out_file + "'" + + elif dataset == "openorca-gsm8k-mbxp-combined": + accuracy_checker_file = os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "language", "mixtral-8x7b", + "evaluate-accuracy.py") + CMD = env['CM_PYTHON_BIN_WITH_PATH'] + " '" + accuracy_checker_file + "' --checkpoint-path '" + env['CM_ML_MODEL_MIXTRAL_FILE_WITH_PATH'] + "' --mlperf-accuracy-file '" + os.path.join(result_dir, "mlperf_log_accuracy.json") + \ + "' --dataset-file '" + env['CM_DATASET_PREPROCESSED_PATH'] + "'"+ " --dtype " + env.get('CM_ACCURACY_DTYPE', "float32") +" > '" + out_file + "'" elif dataset == "coco2014": diff --git a/script/reproduce-mlperf-octoml-tinyml-results/dockerfiles/ubuntu_20.04.Dockerfile b/script/reproduce-mlperf-octoml-tinyml-results/dockerfiles/ubuntu_20.04.Dockerfile deleted file mode 100644 index 5a1672f1d8..0000000000 --- a/script/reproduce-mlperf-octoml-tinyml-results/dockerfiles/ubuntu_20.04.Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM ubuntu:20.04 -SHELL ["/bin/bash", "-c"] -ARG CM_GH_TOKEN -RUN apt-get update -y -RUN apt-get install -y python3 python3-pip git sudo wget -RUN python3 -m pip install cmind requests -ENTRYPOINT ["/bin/bash", "-c"] -ENV TZ=US/Pacific -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ >/etc/timezone -RUN groupadd cm -RUN useradd -g cm --create-home --shell /bin/bash cmuser -RUN echo "cmuser ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers -USER cmuser:cm -WORKDIR /home/cmuser -RUN cm pull repo ctuning@mlcommons-ck -RUN cm run script --quiet --tags=get,sys-utils-cm -RUN cm run script --quiet --tags=reproduce,tiny,octoml --env.CM_GH_TOKEN=$CM_GH_TOKEN diff --git a/script/run-mlperf-inference-app/_cm.yaml b/script/run-mlperf-inference-app/_cm.yaml index 84990a6955..72762501d6 100644 --- a/script/run-mlperf-inference-app/_cm.yaml +++ b/script/run-mlperf-inference-app/_cm.yaml @@ -346,6 +346,7 @@ input_description: - sdxl - llama2-70b-99 - llama2-70b-99.9 + - mixtral-8x7b - mobilenet - efficientnet default: resnet50 diff --git a/script/run-mlperf-inference-app/customize.py b/script/run-mlperf-inference-app/customize.py index 861207c29b..67a60daabf 100644 --- a/script/run-mlperf-inference-app/customize.py +++ b/script/run-mlperf-inference-app/customize.py @@ -101,7 +101,7 @@ def preprocess(i): test_list = ["TEST01", "TEST05"] if env['CM_MODEL'] in ["resnet50"]: test_list.append("TEST04") - if "gpt" in env['CM_MODEL'] or "sdxl" in env['CM_MODEL'] or "llama2-70b" in env['CM_MODEL']: + if "gpt" in env['CM_MODEL'] or "sdxl" in env['CM_MODEL'] or "llama2-70b" in env['CM_MODEL'] or "mixtral-8x7b" in env['CM_MODEL']: test_list.remove("TEST01") test_list.remove("TEST05") @@ -555,6 +555,9 @@ def gui(i): elif model.startswith('llama2-'): github_doc_model = 'llama2-70b' + + elif model.startswith('mixtral-'): + github_doc_model = 'mixtral-8x7b' if github_doc_model == '': github_doc_model = model diff --git a/script/run-mlperf-inference-submission-checker/_cm.json b/script/run-mlperf-inference-submission-checker/_cm.json index ed3f45a5d3..3b567cc8e5 100644 --- a/script/run-mlperf-inference-submission-checker/_cm.json +++ b/script/run-mlperf-inference-submission-checker/_cm.json @@ -135,6 +135,13 @@ "version": "r4.0" } } + }, + "r4.1": { + "adr": { + "submission-checker-src": { + "version": "r4.1" + } + } } } }