Support hf_token in CM docker runs (#114)

* Support GH_PAT for windows in push-mlperf-inference-results-to-github

* Added huggingface-cli script

* Added support to modify cache via meta

Author: arjunsuresh

automation/script/module.py

@@ -32,6 +32,7 @@ def __init__(self, cmind, automation_file):
         self.run_state['fake_deps'] = False
         self.run_state['parent'] = None
         self.run_state['version_info'] = []
+        self.run_state['cache'] = False
 
         self.file_with_cached_state = 'cm-cached-state.json'
 
@@ -74,7 +75,8 @@ def __init__(self, cmind, automation_file):
                                              'accept_license',
                                              'skip_system_deps',
                                              'git_ssh',
-                                             'gh_token']
+                                             'gh_token',
+                                             'hf_token']
 
     ############################################################
 
@@ -826,6 +828,7 @@ def _run(self, i):
             'alias', '')
         run_state['script_repo_git'] = script_artifact.repo_meta.get(
             'git', False)
+        run_state['cache'] = meta.get('cache', False)
 
         if not recursion:
             run_state['script_entry_repo_to_report_errors'] = meta.get(
@@ -1125,7 +1128,7 @@ def _run(self, i):
         # Check if the output of a selected script should be cached
         cache = False if i.get(
             'skip_cache',
-            False) else meta.get(
+            False) else run_state.get(
             'cache',
             False)
         cache = cache or (
@@ -6163,6 +6166,9 @@ def update_state_from_meta(meta, env, state, const, const_state, deps, post_deps
     Internal: update env and state from meta
     """
 
+    if meta.get('cache', '') != '':
+        run_state['cache'] = meta['cache']
+
     default_env = meta.get('default_env', {})
     for key in default_env:
         env.setdefault(key, default_env[key])

script/get-generic-python-lib/detect-version.py

@@ -2,6 +2,7 @@
 import sys
 
 package_name = os.environ.get('CM_GENERIC_PYTHON_PACKAGE_NAME', '')
+package_name = package_name.split("[")[0]
 
 filename = 'tmp-ver.out'
 

script/get-huggingface-cli/_cm.yaml

@@ -0,0 +1,26 @@
+alias: get-huggingface-cli
+automation_alias: script
+automation_uid: 5b4e0237da074764
+cache: false
+can_force_cache: true
+category: DevOps automation
+clean_files: []
+deps:
+- tags: detect,os
+tags:
+- get
+- huggingface
+- hf-cli
+- huggingface-cli
+- cli
+input_mapping:
+  token: CM_HF_TOKEN
+uid: e9488a272f1d4160
+deps:
+ - tags: get,generic-python-lib,_package.huggingface_hub[cli]
+variations:
+  with-login:
+    cache: true
+    force_cache: true
+    env:
+      CM_HF_DO_LOGIN: yes

script/get-huggingface-cli/customize.py

@@ -0,0 +1,30 @@
+from cmind import utils
+import os
+
+
+def preprocess(i):
+    env = i['env']
+    if env.get('CM_HF_TOKEN', '') != '':
+        env['CM_HF_LOGIN_CMD'] = f"""git config --global credential.helper store && huggingface-cli login --token {env['CM_HF_TOKEN']} --add-to-git-credential
+"""
+    elif str(env.get('CM_HF_DO_LOGIN')).lower() in ["yes", "1", "true"]:
+        env['CM_HF_LOGIN_CMD'] = f"""git config --global credential.helper store && huggingface-cli login
+"""
+    return {'return': 0}
+
+
+def postprocess(i):
+    env = i['env']
+
+    r = i['automation'].parse_version({'match_text': r'huggingface_hub\s*version:\s*([\d.]+)',
+                                       'group_number': 1,
+                                       'env_key': 'CM_GITHUBCLI_VERSION',
+                                       'which_env': i['env']})
+    if r['return'] > 0:
+        return r
+
+    version = r['version']
+
+    print(i['recursion_spaces'] + '    Detected version: {}'.format(version))
+
+    return {'return': 0, 'version': version}

script/get-huggingface-cli/run.bat

@@ -0,0 +1,14 @@
+@echo off
+REM Check if the environment variable CM_HF_LOGIN_CMD is defined and not empty
+IF DEFINED CM_HF_LOGIN_CMD (
+    echo %CM_HF_LOGIN_CMD%
+    call %CM_HF_LOGIN_CMD%
+    IF ERRORLEVEL 1 (
+        echo Command failed with error code %ERRORLEVEL%
+        exit /b %ERRORLEVEL%
+    )
+)
+
+REM Run the Hugging Face CLI version command and save output
+huggingface-cli version > tmp-ver.out
+

script/get-huggingface-cli/run.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+if  [[ -n ${CM_HF_LOGIN_CMD} ]]; then
+  echo "${CM_HF_LOGIN_CMD}"
+  eval ${CM_HF_LOGIN_CMD}
+  test $? -eq 0 || exit $?
+fi
+huggingface-cli version > tmp-ver.out

script/get-ml-model-huggingface-zoo/_cm.yaml

@@ -35,6 +35,10 @@ uid: 53cf8252a443446a
 variations:
   clone-repo:
     deps:
+    - tags: get,hf-cli,_with-token
+      enable_if_env:
+        CM_HF_TOKEN:
+          - on
     - env:
         CM_GIT_CHECKOUT_PATH_ENV_NAME: CM_ML_MODEL_PATH
       tags: get,git,repo,_lfs