Cleanup of prepare-training-data-bert

Ailurus1 · Jun 12, 2023 · 26b7a5f · 26b7a5f
1 parent 30fe073
commit 26b7a5f
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 7 deletions.
diff --git a/cm-mlops/automation/script/module.py b/cm-mlops/automation/script/module.py
@@ -2360,7 +2360,6 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a
                             d['tags']+=","+t+env[key]
 
                 run_state['deps'].append(d['tags'])
-                run_state['deps'].append(d['tags'])
 
                 if not run_state['fake_deps']:
                     import copy

diff --git a/cm-mlops/script/download-file/_cm.json b/cm-mlops/script/download-file/_cm.json
@@ -3,6 +3,7 @@
   "automation_alias": "script",
   "automation_uid": "5b4e0237da074764",
   "cache": false,
+  "can_force_cache": true,
   "deps": [],
   "env": {
   },

diff --git a/cm-mlops/script/extract-file/_cm.json b/cm-mlops/script/extract-file/_cm.json
@@ -8,6 +8,7 @@
     }
   ],
   "cache": false,
+  "can_force_cache": true,
   "env": {
     "CM_DAE_REMOVE_EXTRACTED": "yes"
   },

diff --git a/cm-mlops/script/prepare-training-data-bert/_cm.json b/cm-mlops/script/prepare-training-data-bert/_cm.json
@@ -36,25 +36,31 @@
           "tags": "download,file,_gdown,_url.https://drive.google.com/uc?id=1fbGClQMi2CoMv7fwrwTC5YYPooQBdcFW",
           "env": {
             "CM_DOWNLOAD_FILENAME": "bert_config.json",
-            "CM_DOWNLOAD_PATH": "<<<CM_BERT_CONFIG_DOWNLOAD_DIR>>>"
+            "CM_DOWNLOAD_PATH": "<<<CM_BERT_CONFIG_DOWNLOAD_DIR>>>",
+            "CM_DOWNLOAD_CHECKSUM": "7f59165e21b7d566db610ff6756c926b"
           },
+          "force_cache": true,
           "extra_cache_tags": "mlperf,training,bert,config"
         },
         {
           "tags": "download,file,_gdown,_url.https://drive.google.com/uc?id=1USK108J6hMM_d27xCHi738qBL8_BT1u1",
           "env": {
             "CM_DOWNLOAD_FILENAME": "vocab.txt",
-            "CM_DOWNLOAD_PATH": "<<<CM_BERT_VOCAB_DOWNLOAD_DIR>>>"
+            "CM_DOWNLOAD_PATH": "<<<CM_BERT_VOCAB_DOWNLOAD_DIR>>>",
+            "CM_DOWNLOAD_CHECKSUM": "64800d5d8528ce344256daf115d4965e"
           },
+          "force_cache": true,
           "extra_cache_tags": "bert,vocab"
         },
         {
           "tags": "download,file,_gdown,_url.https://drive.google.com/uc?id=1tmMgLwoBvbEJEHXh77sqrXYw5RpqT8R_",
           "env": {
             "CM_DOWNLOAD_FILENAME": "bert_reference_results_text_md5.txt",
             "CM_DOWNLOAD_PATH": "<<<CM_BERT_DATA_DOWNLOAD_DIR>>>",
-            "CM_DOWNLOAD_FINAL_ENV_NAME": "CM_BERT_REFERENCE_RESULTS_TEXT_MD5_FILE_PATH"
+            "CM_DOWNLOAD_FINAL_ENV_NAME": "CM_BERT_REFERENCE_RESULTS_TEXT_MD5_FILE_PATH",
+            "CM_DOWNLOAD_CHECKSUM": "7d3a0619cb8bf7e829af99fa5c29daa8"
           },
+          "force_cache": true,
           "extra_cache_tags": "bert,data,results,md5"
         },
         {
@@ -64,32 +70,40 @@
             "CM_EXTRACT_EXTRACTED_FILENAME": "results4",
             "CM_DOWNLOAD_PATH": "<<<CM_BERT_DATA_DOWNLOAD_DIR>>>",
             "CM_EXTRACT_PATH": "<<<CM_BERT_DATA_DOWNLOAD_DIR>>>",
+            "CM_DOWNLOAD_CHECKSUM": "",
             "CM_EXTRACT_EXTRACTED_CHECKSUM_FILE": "<<<CM_BERT_REFERENCE_RESULTS_TEXT_MD5_FILE_PATH>>>"
           },
+          "force_cache": true,
           "extra_cache_tags": "bert,data,results"
         },
         {
           "tags": "download,file,_gdown,_url.https://drive.google.com/uc?id=1chiTBljF0Eh1U5pKs6ureVHgSbtU8OG_",
           "env": {
             "CM_DOWNLOAD_FILENAME": "model.ckpt-28252.data-00000-of-00001",
-            "CM_DOWNLOAD_PATH": "<<<CM_BERT_CHECKPOINT_DOWNLOAD_DIR>>>"
+            "CM_DOWNLOAD_PATH": "<<<CM_BERT_CHECKPOINT_DOWNLOAD_DIR>>>",
+            "CM_DOWNLOAD_CHECKSUM": "50797acd537880bfb5a7ade80d976129"
           },
+          "force_cache": true,
           "extra_cache_tags": "bert,checkpoint,data"
         },
         {
           "tags": "download,file,_gdown,_url.https://drive.google.com/uc?id=1Q47V3K3jFRkbJ2zGCrKkKk-n0fvMZsa0",
           "env": {
             "CM_DOWNLOAD_FILENAME": "model.ckpt-28252.index",
-            "CM_DOWNLOAD_PATH": "<<<CM_BERT_CHECKPOINT_DOWNLOAD_DIR>>>"
+            "CM_DOWNLOAD_PATH": "<<<CM_BERT_CHECKPOINT_DOWNLOAD_DIR>>>",
+            "CM_DOWNLOAD_CHECKSUM": "f97de3ae180eb8d479555c939d50d048"
           },
+          "force_cache": true,
           "extra_cache_tags": "bert,checkpoint,index"
         },
         {
           "tags": "download,file,_gdown,_url.https://drive.google.com/uc?id=1vAcVmXSLsLeQ1q7gvHnQUSth5W_f_pwv",
           "env": {
             "CM_DOWNLOAD_FILENAME": "model.ckpt-28252.meta",
-            "CM_DOWNLOAD_PATH": "<<<CM_BERT_CHECKPOINT_DOWNLOAD_DIR>>>"
+            "CM_DOWNLOAD_PATH": "<<<CM_BERT_CHECKPOINT_DOWNLOAD_DIR>>>",
+            "CM_DOWNLOAD_CHECKSUM": "dbd16c731e8a8113bc08eeed0326b8e7"
           },
+          "force_cache": true,
           "extra_cache_tags": "bert,checkpoint,meta"
         }
       ],