Merge pull request #41 from mlcommons/master

s
flexaihq · Dec 12, 2024 · e6a88d1 · e6a88d1
2 parents dfc74a6 + 7f66e24
commit e6a88d1
Show file tree

Hide file tree

Showing 11 changed files with 701 additions and 583 deletions.
diff --git a/README.md b/README.md
@@ -13,28 +13,31 @@
 
 [Collective Knowledge (CK, CM, CM4MLOps, CM4MLPerf and CMX)](https://cKnowledge.org) 
 is an educational community project to learn how to run AI, ML and other emerging workloads 
-in the most efficient and cost-effective way across diverse models, data sets, software and hardware.
+in the most efficient and cost-effective way across diverse models, data sets, software and hardware:
+[ [white paper](https://arxiv.org/abs/2406.16791) ].
+
 It includes the following sub-projects.
 
 ### Collective Mind (CM)
 
-[Collective Mind (CM)](https://github.com/mlcommons/ck/tree/master/cm) -
-a very lightweight Python-based framework with a unified CLI, Python API and minimal dependencies
-intended to help researchers and engineers automate their repetitive, tedious and time-consuming tasks
-to build, run, benchmark and optimize AI, ML and other applications and systems 
-across diverse and continuously changing models, data, software and hardware.
+The [Collective Mind framework](https://github.com/mlcommons/ck/tree/master/cm) 
+is a lightweight, Python-based toolset featuring 
+a unified command-line interface (CLI), Python API, and minimal dependencies. 
+It is designed to assist researchers and engineers in automating repetitive, time-consuming 
+tasks such as building, running, benchmarking, and optimizing AI, machine learning, 
+and other applications across diverse and continuously changing models, data, software and hardware.
 
-Collective Mind is continuously enhanced through public and private CM4* Git repositories, 
-which serve as the unified interface for various collections of reusable automations and artifacts.
+Collective Mind is continuously enhanced through public and private Git repositories
+with CM automation recipes and artifacts accessible via unified CM interface.
 
 The CM architecture diagram is available for viewing 
 [here](https://github.com/mlcommons/ck/tree/master/docs/specs/cm-diagram-v3.5.1.png).
 
-### Collective Mind repositories
+### Notable Collective Mind repositories
 
 #### CM4MLOps
 
-[CM4MLOPS repository powered by CM](https://github.com/mlcommons/cm4mlops) - 
+[CM4MLOPS repository powered by CM](https://github.com/mlcommons/ck/tree/master/cm-mlops) - 
 a collection of portable, extensible and technology-agnostic automation recipes
 with a common CLI and Python API (CM scripts) to unify and automate 
 all the manual steps required to compose, run, benchmark and optimize complex ML/AI applications 
@@ -121,13 +124,6 @@ based on user feedback. Follow the project's progress [here]( https://github.com
 
 * [Grigori Fursin](https://cKnowledge.org/gfursin) (FlexAI, cTuning)
 
-## Maintainers
-
-* Collective Mind (CM): [Grigori Fursin](https://cKnowledge.org/gfursin)
-* CM4MLOps repository: [Arjun Suresh](https://github.com/arjunsuresh) and [Anandhu Sooraj](https://github.com/anandhu-eng)
-* CMX (the next generation of CM and CM4MLOps): [Grigori Fursin](https://cKnowledge.org/gfursin)
-
-
 ## Citing Collective Mind and Collective Knowledge
 
 If you found the CM automation framework helpful, kindly reference this article:

diff --git a/cm-mlops/README.md b/cm-mlops/README.md
@@ -1,5 +1,5 @@
 **This repository is archived. In April 2024, we have separated this CM repo from the CM framework 
-  and moved it to a [standalone mlcommons@cm4mlops repository](https://github.com/mlcommons/cm4mlops/tree/dev)
+  and moved it to a [standalone mlcommons@cm4mlops repository](https://github.com/mlcommons/cm4mlops)
   following the suggesion from our users and MLCommons members.**
 
 We keep this directory for backwards compatibility to let users reproduce past projects relying on this repository.
diff --git a/cm/CHANGES.md b/cm/CHANGES.md
@@ -1,3 +1,7 @@
+## V3.5.1.1
+   - added utils.get_memory_use
+   - formatted Python modules from the internal repository using autopep8
+
 ## V3.5.1
    - added Collective Mind architecture diagram: 
      https://github.com/mlcommons/ck/tree/master/docs/specs/cm-diagram-v3.5.1.png

diff --git a/cm/cmind/__init__.py b/cm/cmind/__init__.py
@@ -2,7 +2,7 @@
 #
 # Written by Grigori Fursin
 
-__version__ = "3.5.1"
+__version__ = "3.5.1.1"
 
 from cmind.core import access
 from cmind.core import x

diff --git a/cm/cmind/repo/automation/automation/module.py b/cm/cmind/repo/automation/automation/module.py
@@ -7,6 +7,7 @@
 from cmind.automation import Automation
 from cmind import utils
 
+
 class CAutomation(Automation):
     """
     CM "automation" automation actions
@@ -35,12 +36,12 @@ def print_input(self, i):
         """
 
         import json
-        print (json.dumps(i, indent=2))
-
-        return {'return':0}
+        print(json.dumps(i, indent=2))
 
+        return {'return': 0}
 
     ############################################################
+
     def add(self, i):
         """
         Add CM automation.
@@ -66,36 +67,40 @@ def add(self, i):
 
         console = i.get('out') == 'con'
 
-        parsed_artifact = i.get('parsed_artifact',[])
+        parsed_artifact = i.get('parsed_artifact', [])
 
-        artifact_obj = parsed_artifact[0] if len(parsed_artifact)>0 else ('','')
+        artifact_obj = parsed_artifact[0] if len(
+            parsed_artifact) > 0 else ('', '')
 
         module_name = 'module.py'
 
         tags_list = utils.convert_tags_to_list(i)
-        if 'automation' not in tags_list: tags_list.append('automation')
+        if 'automation' not in tags_list:
+            tags_list.append('automation')
 
         # Add placeholder (use common action)
-        i['out']='con'
-        i['common']=True
+        i['out'] = 'con'
+        i['common'] = True
 
-        i['meta']={'automation_alias':self.meta['alias'],
-                   'automation_uid':self.meta['uid'],
-                   'tags':tags_list}
+        i['meta'] = {'automation_alias': self.meta['alias'],
+                     'automation_uid': self.meta['uid'],
+                     'tags': tags_list}
 
-        if 'tags' in i: del(i['tags'])
+        if 'tags' in i:
+            del (i['tags'])
 
         automation = i['automation']
-        if automation!='.' and ',' not in automation: 
+        if automation != '.' and ',' not in automation:
             i['automation'] = automation + ',' + self.meta['uid']
 
-        r_obj=self.cmind.access(i)
-        if r_obj['return']>0: return r_obj
+        r_obj = self.cmind.access(i)
+        if r_obj['return'] > 0:
+            return r_obj
 
         new_automation_path = r_obj['path']
 
         if console:
-            print ('Created automation in {}'.format(new_automation_path))
+            print('Created automation in {}'.format(new_automation_path))
 
         # Create Python module holder
         module_holder_path = new_automation_path
@@ -106,12 +111,12 @@ def add(self, i):
         # Copy module files
         for f in ['module_dummy.py']:
             f1 = os.path.join(self.path, f)
-            f2 = os.path.join(new_automation_path, f.replace('_dummy',''))
+            f2 = os.path.join(new_automation_path, f.replace('_dummy', ''))
 
             if console:
-                print ('  * Copying {} to {}'.format(f1, f2))
+                print('  * Copying {} to {}'.format(f1, f2))
 
-            shutil.copyfile(f1,f2)
+            shutil.copyfile(f1, f2)
 
         return r_obj
 
@@ -137,33 +142,37 @@ def add_cmx(self, i):
 
         # Prepare to call common function
         r = utils.process_input(i)
-        if r['return']>0: return r
+        if r['return'] > 0:
+            return r
 
         # Take only out from original control
-        i['control']={'out':i['control']['out'],
-                      'common':True}
+        i['control'] = {'out': i['control']['out'],
+                        'common': True}
 
         tags_list = utils.convert_tags_to_list(i)
-        if 'automation' not in tags_list: tags_list.append('automation')
+        if 'automation' not in tags_list:
+            tags_list.append('automation')
 
-        i['meta']={'automation_alias':self.meta['alias'],
-                   'automation_uid':self.meta['uid'],
-                   'tags':tags_list}
+        i['meta'] = {'automation_alias': self.meta['alias'],
+                     'automation_uid': self.meta['uid'],
+                     'tags': tags_list}
 
-        if 'tags' in i: del(i['tags'])
+        if 'tags' in i:
+            del (i['tags'])
 
         # Use yaml by default
         if 'yaml' not in i:
             i['yaml'] = True
 
         # Pass to common action
         r_obj = self.cmind.x(i)
-        if r_obj['return']>0: return r_obj
+        if r_obj['return'] > 0:
+            return r_obj
 
         new_automation_path = r_obj['path']
 
         if console:
-            print ('Created automation in {}'.format(new_automation_path))
+            print('Created automation in {}'.format(new_automation_path))
 
         module_name = 'modulex.py'
 
@@ -176,18 +185,17 @@ def add_cmx(self, i):
         # Copy module files
         for f in ['modulex_dummy.py']:
             f1 = os.path.join(self.path, f)
-            f2 = os.path.join(new_automation_path, f.replace('_dummy',''))
+            f2 = os.path.join(new_automation_path, f.replace('_dummy', ''))
 
             if console:
-                print ('  * Copying {} to {}'.format(f1, f2))
+                print('  * Copying {} to {}'.format(f1, f2))
 
-            shutil.copyfile(f1,f2)
+            shutil.copyfile(f1, f2)
 
         return r_obj
 
-
-
     ############################################################
+
     def doc(self, i):
         """
         Add CM automation.