Tune data packaging (in Hugging Face)

OpenLLM-France · Oct 16, 2024 · e3da1de · e3da1de
1 parent 9c11c81
commit e3da1de
Show file tree

Hide file tree

Showing 4 changed files with 650 additions and 174 deletions.
diff --git a/assets/compile_stats.py b/assets/compile_stats.py
@@ -186,13 +186,15 @@ def to_name_subset(name):
 
 
 def to_language_name_subset(name, subset=None):
+    _languages = ["fr", "en", "de", "es", "it"]
     if subset is None:
         name, subset = to_name_subset(name)
-    for lan in "fr", "en", "de", "es", "it":
+    for lan in _languages:
         subset2 = subset.rstrip(":.0123456789")
         if subset.startswith(lan) and (len(subset) == len(lan) or subset[len(lan)] in ".:-"):
-            if "-" in subset and len(subset2) == 5:
-                subset = subset2
+            if len(subset2) >= 5 and subset[len(lan)] in ".:-" and subset2[3:5] in _languages:
+                # multi-lingual
+                subset = subset2[:5]
                 lan = subset
             subset = subset[len(lan) :].strip(":.")
             subset = subset.strip(":_")