Skip to content

Commit

Permalink
* generating documentation for scripts
Browse files Browse the repository at this point in the history 
* adding simple function to call sub-modules from automation (to break monolithic modules)
  • Loading branch information
@gfursin
gfursin committed Dec 19, 2022
1 parent 7d80b74 commit fd660c4
Show file tree
Hide file tree
Showing 197 changed files with 6,872 additions and 3,034 deletions.
194 changes: 1 addition & 193 deletions cm-mlops/automation/script/module.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2799,199 +2799,7 @@ def doc(self, i):
"""

template_file = 'list_of_scripts.md'

console = i.get('out') == 'con'

repos = i.get('repos','')
if repos == '': repos='internal,a4705959af8e447a'

parsed_artifact = i.get('parsed_artifact',[])

if len(parsed_artifact)<1:
parsed_artifact = [('',''), ('','')]
elif len(parsed_artifact)<2:
parsed_artifact.append(('',''))
else:
repos = parsed_artifact[1][0]

list_of_repos = repos.split(',') if ',' in repos else [repos]

ii = utils.sub_input(i, self.cmind.cfg['artifact_keys'])

ii['out'] = None

# Search for automations in repos
lst = []
for repo in list_of_repos:
parsed_artifact[1] = ('',repo) if utils.is_cm_uid(repo) else (repo,'')
ii['parsed_artifact'] = parsed_artifact
r = self.search(ii)
if r['return']>0: return r
lst += r['list']

md = []

toc = []

toc_category = {}
toc_category_sort = {}
script_meta = {}

for artifact in sorted(lst, key = lambda x: x.meta.get('alias','')):
path = artifact.path
meta = artifact.meta
original_meta = artifact.original_meta

print ('Documenting {}'.format(path))

alias = meta.get('alias','')
uid = meta.get('uid','')

script_meta[alias]=meta

name = meta.get('name','')
developers = meta.get('developers','')

tags = meta.get('tags',[])

variation_keys = sorted(list(meta.get('variations',{}).keys()))
version_keys = sorted(list(meta.get('versions',{}).keys()))

default_variation = meta.get('default_variation','')
default_version = meta.get('default_version','')



category = meta.get('category', '').strip()
category_sort = meta.get('category_sort', 0)
if category != '':
if category not in toc_category:
toc_category[category]=[]

if category not in toc_category_sort or category_sort>0:
toc_category_sort[category]=category_sort

if alias not in toc_category[category]:
toc_category[category].append(alias)

repo_path = artifact.repo_path
repo_meta = artifact.repo_meta

repo_alias = repo_meta.get('alias','')
repo_uid = repo_meta.get('uid','')


# Check URL
url = ''
url_repo = ''
if repo_alias == 'internal':
url_repo = 'https://github.com/mlcommons/ck/tree/master/cm/cmind/repo'
url = url_repo+'/script/'
elif '@' in repo_alias:
url_repo = 'https://github.com/'+repo_alias.replace('@','/')+'/tree/master'
if repo_meta.get('prefix','')!='': url_repo+='/'+repo_meta['prefix']
url = url_repo+ '/script/'

if url!='':
url+=alias

x = '* [{}](#{})'.format(alias, alias)
if name !='': x+=' *('+name+')*'

toc.append(x)

md.append('## '+alias)
md.append('\n')

if name!='':
md.append('*'+name+'.*')
md.append('\n')

# if developers!='':
# md.append('Developers: '+developers)
# md.append('\n')

md.append('* CM script GitHub repository: *[{}]({})*'.format(repo_alias, url_repo))
md.append('* CM script artifact (interoperability module, native scripts and meta): *[GitHub]({})*'.format(url))

# Check meta
meta_file = self.cmind.cfg['file_cmeta']
meta_path = os.path.join(path, meta_file)

meta_file += '.yaml' if os.path.isfile(meta_path+'.yaml') else '.json'

meta_url = url+'/'+meta_file

md.append('* CM script meta description: *[GitHub]({})*'.format(meta_url))
md.append('* CM automation "script": *[Docs]({})*'.format('https://github.com/octoml/ck/blob/master/docs/list_of_automations.md#script'))
md.append('* CM script tags: *cm run script --tags="{}"*'.format(','.join(tags)))

if len(variation_keys)>0:
x=''
for variation in variation_keys:
if x!='': x+=';&nbsp; '
x+='_'+variation
md.append('* CM script variations: *{}*'.format(x))

if default_variation!='':
md.append('* CM script default variation: *{}*'.format(default_variation))

if len(version_keys)>0:
md.append('* CM script versions: *{}*'.format(';&nbsp; '.join(version_keys)))

if default_version!='':
md.append('* CM script default version: *{}*'.format(default_version))




md.append('\n')




# Recreate TOC with categories
toc2 = []

for category in sorted(toc_category, key = lambda x: -toc_category_sort[x]):
toc2.append('### '+category)
toc2.append('\n')

for script in sorted(toc_category[category]):

meta = script_meta[script]

name = meta.get('name','')

x = '* [{}](#{})'.format(script, script)
if name !='': x+=' *('+name+')*'

toc2.append(x)

toc2.append('\n')

# Load template
r = utils.load_txt(os.path.join(self.path, template_file))
if r['return']>0: return r

s = r['string']

s = s.replace('{{CM_TOC2}}', '\n'.join(toc2))
s = s.replace('{{CM_TOC}}', '\n'.join(toc))
s = s.replace('{{CM_MAIN}}', '\n'.join(md))

# Output
output_dir = i.get('output_dir','')

if output_dir == '': output_dir = '..'

output_file = os.path.join(output_dir, template_file)

r = utils.save_txt(output_file, s)
if r['return']>0: return r

return {'return':0}
return utils.call_internal_module(self, __file__, 'module_misc', 'doc', i)


##############################################################################
Expand Down
Loading

0 comments on commit fd660c4

Please sign in to comment.