Merge branch 'master' of https://github.com/ctuning/mlcommons-ck

cm-mlops/automation/script/module.py

@@ -2525,7 +2525,7 @@ def _call_run_deps(script, deps, local_env_keys, local_env_keys_from_meta, env,
     ##############################################################################
     def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, add_deps_recursive, recursion_spaces, 
                     remembered_selections, variation_tags_string='', from_cache=False, debug_script_tags='', 
-                    verbose=False, show_time=False, extra_recursion_spaces='  ', run_state={'deps':[], 'fake_deps':[]}):
+                  verbose=False, show_time=False, extra_recursion_spaces='  ', run_state={'deps':[], 'fake_deps':[], 'parent': None}):
         """
         Runs all the enabled dependencies and pass them env minus local env
         """

cm-mlops/automation/script/module_misc.py

@@ -1537,7 +1537,7 @@ def docker(i):
         return {'return':1, 'error':'no scripts were found'}
 
     env=i.get('env', {})
-    env['CM_RUN_STATE_DOCKER'] = True
+    env['CM_RUN_STATE_DOCKER'] = False
 
     docker_cache = i.get('docker_cache', "yes")
     if docker_cache in ["no", False, "False" ]:
@@ -1599,8 +1599,16 @@ def docker(i):
             update_path_for_docker('.', mounts, force_path_target=current_path_target)
 
 
-        _os = i.get('docker_os', meta.get('docker_os', 'ubuntu'))
-        version = i.get('docker_os_version', meta.get('docker_os_version', '22.04'))
+        _os = i.get('docker_os', docker_settings.get('docker_os', 'ubuntu'))
+        version = i.get('docker_os_version', docker_settings.get('docker_os_version', '22.04'))
+
+        deps = docker_settings.get('deps', [])
+        if deps:
+            # Todo: Support state, const and add_deps_recursive
+            script_automation = i['self_module']
+            r = script_automation._run_deps(deps, [], env, {}, {}, {}, {}, '',{})
+            if r['return'] > 0:
+                return r
 
         for key in docker_settings.get('mounts', []):
             mounts.append(key)
@@ -1694,6 +1702,11 @@ def docker(i):
 
         port_maps = i.get('docker_port_maps', docker_settings.get('port_maps', []))
 
+        if detached == '':
+            detached = docker_settings.get('detached', '')
+
+        if interactive == '':
+            interactive = docker_settings.get('interactive', '')
 
 #        # Regenerate run_cmd
 #        if i.get('cmd'):
@@ -1717,6 +1730,7 @@ def docker(i):
         if r['return']>0: return r
 
         run_cmd  = r['run_cmd_string']
+        env['CM_RUN_STATE_DOCKER'] = True
 
         if docker_settings.get('mount_current_dir','')=='yes':
             run_cmd = 'cd '+current_path_target+' && '+run_cmd

cm-mlops/script/build-mlperf-inference-server-nvidia/README.md

@@ -222,6 +222,8 @@ ___
      * get,nvidia,mlperf,inference,common-code
        * CM names: `--adr.['nvidia-inference-common-code']...`
        - CM script: [get-mlperf-inference-nvidia-common-code](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-mlperf-inference-nvidia-common-code)
+     * get,generic-python-lib,_package.pybind11
+       - CM script: [get-generic-python-lib](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-generic-python-lib)
      * get,generic-python-lib,_pycuda
        * `if (CM_RUN_STATE_DOCKER not in ['yes', True, 'True'])`
        - CM script: [get-generic-python-lib](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-generic-python-lib)

cm-mlops/script/build-mlperf-inference-server-nvidia/_cm.yaml

@@ -77,7 +77,7 @@ deps:
 
   # Detect CMake
   - tags: get,cmake
-    version_min: "3.18"
+    version_min: "3.25"
 
   # Detect Google Logger
   - tags: get,generic,sys-util,_glog-dev
@@ -106,6 +106,8 @@ deps:
     names:
     - nvidia-inference-common-code
 
+  - tags: get,generic-python-lib,_package.pybind11
+
   # Detect pycuda
   - tags: get,generic-python-lib,_pycuda
     skip_if_env:
@@ -125,6 +127,7 @@ deps:
     names:
     - nvidia-scratch-space
 
+
 post_deps:
   # Detect nvidia system
   - tags: add,custom,system,nvidia
@@ -185,31 +188,48 @@ versions:
     add_deps_recursive:
       nvidia-inference-common-code:
         version: r2.1
+      nvidia-scratch-space:
+        tags: version.2_1
 
   r3.0:
     add_deps_recursive:
       nvidia-inference-common-code:
         version: r3.0
+      nvidia-scratch-space:
+        tags: version.3_0
+
   r3.1:
     add_deps_recursive:
       nvidia-inference-common-code:
         version: r3.1
+      nvidia-scratch-space:
+        tags: version.3_1
+    deps:
+      - tags: install,nccl,libs,_cuda
+      - tags: install,pytorch,from.src,_for-nvidia-mlperf-inference-v3.1-gptj
+        names:
+        - pytorch
 
 docker:
   skip_run_cmd: 'no'
   all_gpus: 'yes'
   docker_os: ubuntu
-  docker_real_run: True
+  docker_real_run: False
+  interactive: True
   docker_os_version: '20.04'
   base_image: nvcr.io/nvidia/mlperf/mlperf-inference:mlpinf-v3.1-cuda12.2-cudnn8.9-x86_64-ubuntu20.04-public
   docker_input_mapping:
     imagenet_path: IMAGENET_PATH
+    gptj_checkpoint_path: GPTJ_CHECKPOINT_PATH
+    criteo_preprocessed_path: CRITEO_PREPROCESSED_PATH
     results_dir: RESULTS_DIR
     submission_dir: SUBMISSION_DIR
     cudnn_tar_file_path: CM_CUDNN_TAR_FILE_PATH
     tensorrt_tar_file_path: CM_TENSORRT_TAR_FILE_PATH
     cuda_run_file_path: CUDA_RUN_FILE_LOCAL_PATH
     scratch_path: MLPERF_SCRATCH_PATH
+  deps:
+    - tags: get,mlperf,inference,nvidia,scratch,space
   mounts:
    - "${{ IMAGENET_PATH }}:/data/imagenet-val"
    - "${{ RESULTS_DIR }}:/home/cmuser/results_dir"

cm-mlops/script/download-and-extract/README.md

@@ -136,6 +136,8 @@ ___
       - Workflow:
     * `_gdown`
       - Workflow:
+    * `_rclone`
+      - Workflow:
     * `_torrent`
       - Environment variables:
         - *CM_DAE_DOWNLOAD_USING_TORRENT*: `yes`

cm-mlops/script/download-and-extract/_cm.json

@@ -96,6 +96,14 @@
         "CM_DAE_EXTRACT_DOWNLOADED": "yes"
       }
     },
+    "rclone": {
+      "add_deps_recursive": {
+        "download-script": {
+          "tags": "_rclone"
+        }
+      },
+      "group": "download-tool"
+    },
     "gdown": {
       "add_deps_recursive": {
         "download-script": {

cm-mlops/script/download-file/README.md

@@ -133,6 +133,13 @@ ___
         1. ***Read "deps" on other CM scripts***
            * get,generic-python-lib,_package.gdown
              - CM script: [get-generic-python-lib](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-generic-python-lib)
+    * `_rclone`
+      - Environment variables:
+        - *CM_DOWNLOAD_TOOL*: `rclone`
+      - Workflow:
+        1. ***Read "deps" on other CM scripts***
+           * get,rclone
+             - CM script: [get-rclone](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-rclone)
     * `_wget`
       - Environment variables:
         - *CM_DOWNLOAD_TOOL*: `wget`

cm-mlops/script/download-file/_cm.json

@@ -63,6 +63,17 @@
       },
       "group": "download-tool"
     },
+    "rclone": {
+      "deps": [
+        {
+          "tags": "get,rclone"
+        }
+      ],
+      "env": {
+        "CM_DOWNLOAD_TOOL": "rclone"
+      },
+      "group": "download-tool"
+    },
     "url.#": {
       "env": {
         "CM_DOWNLOAD_URL": "#"

cm-mlops/script/download-file/customize.py

@@ -65,6 +65,8 @@ def preprocess(i):
                 if j>0:
                     urltail=urltail[:j]
                 env['CM_DOWNLOAD_FILENAME'] = urltail
+            elif env.get('CM_DOWNLOAD_TOOL', '') == "rclone":
+                env['CM_DOWNLOAD_FILENAME'] = urltail
             else:
                 env['CM_DOWNLOAD_FILENAME'] = "index.html"
 
@@ -104,6 +106,11 @@ def preprocess(i):
         elif tool == "gdown":
             env['CM_DOWNLOAD_CMD'] = f"gdown {extra_download_options} {url}"
 
+        elif tool == "rclone":
+            if env.get('CM_RCLONE_CONFIG_CMD', '') != '':
+                env['CM_DOWNLOAD_CONFIG_CMD'] = env['CM_RCLONE_CONFIG_CMD']
+            env['CM_DOWNLOAD_CMD'] = f"rclone copy {url} {os.path.join(os.getcwd(), env['CM_DOWNLOAD_FILENAME'])} -P"
+
         filename = env['CM_DOWNLOAD_FILENAME']
         env['CM_DOWNLOAD_DOWNLOADED_FILENAME'] = filename
 

cm-mlops/script/download-file/run.sh

@@ -1,5 +1,11 @@
 #!/bin/bash
 
+if [[ -n ${CM_DOWNLOAD_CONFIG_CMD} ]]; then
+  echo ""
+  echo "${CM_DOWNLOAD_CONFIG_CMD}"
+  eval "${CM_DOWNLOAD_CONFIG_CMD}"
+fi
+
 if [ -e ${CM_DOWNLOAD_DOWNLOADED_PATH} ]; then
   if [[ "${CM_DOWNLOAD_CHECKSUM_CMD}" != "" ]]; then
     echo ""

cm-mlops/script/get-lib-armnn/README.md

@@ -102,10 +102,11 @@ These keys can be updated via `--env.KEY=VALUE` or `env` dictionary in `@input.j
 </details>
 
 #### Versions
-Default version: `23.05`
+Default version: `23.11`
 
 * `22.11`
 * `23.05`
+* `23.11`
 ___
 ### Script workflow, dependencies and native scripts
 

cm-mlops/script/get-lib-armnn/_cm.json

@@ -4,7 +4,7 @@
   "automation_uid": "5b4e0237da074764",
   "cache": true,
   "category": "Detection or installation of tools and artifacts",
-  "default_version": "23.05",
+  "default_version": "23.11",
   "deps": [
     {
       "tags": "detect,os"
@@ -36,6 +36,12 @@
   ],
   "uid": "9603a2e90fd44587",
   "versions": {
+    "23.11": {
+      "env": {
+        "CM_LIB_ARMNN_VERSION": "v23.11",
+        "CM_TMP_GIT_BRANCH_NAME": "branches/armnn_23_11"
+      }
+    },
     "23.05": {
       "env": {
         "CM_LIB_ARMNN_VERSION": "v23.05",

cm-mlops/script/get-ml-model-gptj/README.md

@@ -114,6 +114,8 @@ ___
         - *CM_UNZIP*: `yes`
         - *CM_DOWNLOAD_CHECKSUM_NOT_USED*: `e677e28aaf03da84584bb3073b7ee315`
         - *CM_PACKAGE_URL*: `https://cloud.mlcommons.org/index.php/s/QAZ2oM94MkFtbQx/download`
+        - *CM_RCLONE_CONFIG*: `rclone config create mlc-inference s3 provider=LyveCloud access_key_id=0LITLNQMHZALM5AK secret_access_key=YQKYTMBY23TMZHLOYFJKL5CHHS0CWYUC endpoint=s3.us-east-1.lyvecloud.seagate.com`
+        - *CM_RCLONE_URL*: `mlc-inference:mlcommons-inference-wg-s3/gpt-j`
       - Workflow:
     * `_pytorch,int4,intel`
       - Workflow:
@@ -126,6 +128,7 @@ ___
         1. ***Read "deps" on other CM scripts***
            * get,mlperf,inference,results
              - CM script: [get-mlperf-inference-results](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-mlperf-inference-results)
+             - CM script: [get-mlperf-inference-results-dir](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-mlperf-inference-results-dir)
            * get,ml-model,gpt-j,_fp32,_pytorch
              - CM script: [get-ml-model-gptj](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-ml-model-gptj)
            * get,conda,_name.gptj-pt
@@ -146,6 +149,22 @@ ___
     </details>
 
 
+  * Group "**download-tool**"
+    <details>
+    <summary>Click here to expand this section.</summary>
+
+    * `_rclone`
+      - Environment variables:
+        - *CM_DOWNLOAD_URL*: `<<<CM_RCLONE_URL>>>`
+      - Workflow:
+    * **`_wget`** (default)
+      - Environment variables:
+        - *CM_DOWNLOAD_URL*: `<<<CM_PACKAGE_URL>>>`
+      - Workflow:
+
+    </details>
+
+
   * Group "**framework**"
     <details>
     <summary>Click here to expand this section.</summary>
@@ -205,7 +224,7 @@ ___
 
 #### Default variations
 
-`_mlcommons,_pytorch`
+`_mlcommons,_pytorch,_wget`
 
 #### Script flags mapped to environment
 <details>
@@ -240,7 +259,7 @@ ___
   1. Read "deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-ml-model-gptj/_cm.json)
   1. ***Run "preprocess" function from [customize.py](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-ml-model-gptj/customize.py)***
   1. ***Read "prehook_deps" on other CM scripts from [meta](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/get-ml-model-gptj/_cm.json)***
-     * download-and-extract,_wget
+     * download-and-extract
        * `if (CM_TMP_REQUIRE_DOWNLOAD  == yes)`
        * CM names: `--adr.['dae']...`
        - CM script: [download-and-extract](https://github.com/mlcommons/ck/tree/master/cm-mlops/script/download-and-extract)