added possibility to add extra tags from ENV:

"extra_cache_tags": "ml-model,resnet50,raw,ml-model-resnet50,_<<<CM_ML_MODEL_FRAMEWORK>>>", Needed when we download files via CM script to recreate original variations ...
ctuning · Feb 3, 2024 · d707a8c · d707a8c
1 parent 957cd57
commit d707a8c
Show file tree

Hide file tree

Showing 2 changed files with 180 additions and 167 deletions.
diff --git a/cm-mlops/automation/script/module.py b/cm-mlops/automation/script/module.py
@@ -368,17 +368,6 @@ def run(self, i):
             if value != '':
                 env['CM_' + key.upper()] = value
 
-        # Check extra cache tags
-        x = env.get('CM_EXTRA_CACHE_TAGS','').strip()
-        extra_cache_tags = [] if x=='' else x.split(',')
-
-        if i.get('extra_cache_tags','')!='':
-            for x in i['extra_cache_tags'].strip().split(','):
-                if x!='' and x not in extra_cache_tags:
-                    extra_cache_tags.append(x)
-
-        if env.get('CM_NAME','')!='':
-            extra_cache_tags.append('name-'+env['CM_NAME'].strip().lower())
 
 
         ############################################################################################################
@@ -972,6 +961,29 @@ def run(self, i):
         update_env_with_values(env)
 
 
+
+        ############################################################################################################
+        # Check extra cache tags
+        x = env.get('CM_EXTRA_CACHE_TAGS','').strip()
+        extra_cache_tags = [] if x=='' else x.split(',')
+
+        if i.get('extra_cache_tags','')!='':
+            for x in i['extra_cache_tags'].strip().split(','):
+                if x!='':
+                    if '<<<' in x:
+                        import re
+                        tmp_values = re.findall(r'<<<(.*?)>>>', str(x))
+                        for tmp_value in tmp_values:
+                            xx = str(env.get(tmp_value,''))
+                            x = x.replace("<<<"+tmp_value+">>>", xx)
+                    if x not in extra_cache_tags:
+                        extra_cache_tags.append(x)
+
+        if env.get('CM_NAME','')!='':
+            extra_cache_tags.append('name-'+env['CM_NAME'].strip().lower())
+
+
+
         ############################################################################################################
         # Check if need to clean output files
         clean_output_files = meta.get('clean_output_files', [])