Code documentation support

.gitignore

@@ -18,4 +18,6 @@ dist/
 /rust/
 target/
 Cargo.lock
-.vscode/
+.vscode/
+.docs/
+.venv/

glad/__main__.py

@@ -19,7 +19,7 @@
 from glad.sink import LoggingSink
 from glad.opener import URLOpener
 from glad.parse import FeatureSet
-from glad.plugin import find_specifications, find_generators
+from glad.plugin import find_specification_docs, find_specifications, find_generators
 from glad.util import parse_apis
 
 
@@ -71,6 +71,11 @@ class GlobalConfig(Config):
         description='Makes the build reproducible by not fetching the latest '
                     'specification from Khronos.'
     )
+    DOCS = ConfigOption(
+        converter=bool,
+        default=False,
+        description='Include inline documentation in the generated files.'
+    )
 
 
 def load_specifications(specification_names, opener, specification_classes=None):
@@ -95,6 +100,32 @@ def load_specifications(specification_names, opener, specification_classes=None)
     return specifications
 
 
+def load_documentations(specification_names, opener, spec_docs_classes=None):
+    specifications_docs = dict()
+
+    if spec_docs_classes is None:
+        spec_docs_classes = find_specification_docs()
+
+    for name in set(specification_names):
+        if name not in spec_docs_classes:
+            logger.warning('no documentation available for %r', name)
+            continue
+
+        SpecificationDocs = spec_docs_classes[name]
+        spec_docs_file = SpecificationDocs.default_out_dir()
+
+        if os.path.isfile(spec_docs_file):
+            logger.info("using local documentation: '%s'", spec_docs_file)
+            spec_docs = SpecificationDocs.from_file(spec_docs_file, opener=opener)
+        else:
+            logger.info('getting %r documentation from remote location', name)
+            spec_docs = SpecificationDocs.from_remote(opener=opener)
+
+        specifications_docs[name] = spec_docs
+
+    return specifications_docs
+
+
 def apis_by_specification(api_info, specifications):
     return groupby(api_info.items(),
                    key=lambda api_info: specifications[api_info[1].specification])
@@ -153,9 +184,13 @@ def main(args=None):
         opener = URLOpener()
         gen_info_factory = GenerationInfo.create
 
-    specifications = load_specifications(
-        [value[0] for value in global_config['API'].values()], opener=opener
-    )
+    spec_names = [value[0] for value in global_config['API'].values()]
+    specifications = load_specifications(spec_names, opener=opener)
+
+    if global_config['DOCS']:
+        documentations = load_documentations(spec_names, opener=opener)
+    else:
+        documentations = None
 
     generator = generators[ns.subparser_name](
         global_config['OUT_PATH'], opener=opener, gen_info_factory=gen_info_factory
@@ -189,8 +224,12 @@ def select(specification, api, info):
             logging_sink.info('merged into {}'.format(feature_sets[0]))
 
         for feature_set in feature_sets:
+            api = feature_set.info.apis[0]
+            spec_docs = None
+            if documentations and api in documentations:
+                spec_docs = documentations[api].select(feature_set)
             logging_sink.info('generating feature set {}'.format(feature_set))
-            generator.generate(specification, feature_set, config, sink=logging_sink)
+            generator.generate(specification, feature_set, config, sink=logging_sink, spec_docs=spec_docs)
 
 
 if __name__ == '__main__':

glad/documentation.py

@@ -0,0 +1,226 @@
+import re
+import zipfile
+import glad.util
+from pathlib import Path
+from glad.parse import DocumentationSet, SpecificationDocs, CommandDocs, xml_fromstring
+from glad.util import resolve_entities, suffix, raw_text
+
+class OpenGLRefpages(SpecificationDocs):
+    DOCS_NAME = 'opengl_refpages'
+
+    URL = 'https://github.com/KhronosGroup/OpenGL-Refpages/archive/refs/heads/main.zip'
+    SPEC = 'gl'
+
+    def select(self, feature_set):
+        current_major = max(info.version.major for info in feature_set.info)
+        commands = dict()
+
+        # At the time of writing Khronos hosts documentations for gl4 and gl2.1.
+        available_versions = ['gl2.1']
+        if current_major >= 4:
+            available_versions.insert(0, 'gl4')
+
+        zf = zipfile.ZipFile(self.docs_file, 'r')
+        xml_files_in_version = {
+            version: [Path(name) for name in zf.namelist() if name.endswith('.xml') and version in name]
+            for version in available_versions
+        }
+
+        for version_dir in available_versions:
+            for xml_file in xml_files_in_version[version_dir]:
+                with zf.open(str(xml_file)) as f:
+                    xml_text = f.read().decode('utf-8')
+                parsed_docs = OpenGLRefpages.docs_from_xml(
+                    xml_text,
+                    version=version_dir,
+                    filename=xml_file.stem,
+                )
+                for command, docs in parsed_docs.items():
+                    commands.setdefault(command, docs)
+
+        zf.close()
+        return DocumentationSet(commands=commands)
+
+    @classmethod
+    def docs_from_xml(cls, xml_text, version=None, filename=None):
+        commands_parsed = dict()
+
+        # Some files don't have the proper namespace declaration for MathML.
+        xml_text = xml_text.replace('<mml:math>', '<mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML">')
+        # Entities need to be resolved before parsing.
+        xml_text = resolve_entities(xml_text)
+
+        tree = xml_fromstring(xml_text.encode('utf-8'))
+
+        # gl4 files contain large namespace declarations that pollute the tags. So we remove them.
+        for elem in tree.iter():
+            try:
+                if elem.tag.startswith('{'):
+                    elem.tag = elem.tag.split('}')[-1]
+                if elem.tag.contains(':'):
+                    elem.tag = elem.tag.split(':')[-1]
+                for key in elem.attrib:
+                    if key.startswith('{'):
+                        elem.attrib[key.split('}')[-1]] = elem.attrib.pop(key)
+            except:
+                pass
+
+        # Each refsect1 block contains a section of the documentation.
+        # Sections groups Description, Parameters, Notes, etc.
+        sections = tree.findall('.//refsect1')
+
+        # Brief parsing
+        # Command brief description appears in the first 'refnamediv' block
+        brief_block = tree.find('.//refnamediv//refpurpose')
+
+        if brief_block is None:
+            # No brief means file doesn't contain any command definitions.
+            return dict()
+        brief = suffix(".", cls.xml_text(brief_block))
+
+        if version == 'gl2.1':
+            docs_url = f'https://registry.khronos.org/OpenGL-Refpages/{version}/xhtml/{filename}.xml'
+        else:
+            docs_url = f'https://registry.khronos.org/OpenGL-Refpages/{version}/html/{filename}.xhtml'
+
+        # Description parsing
+        description = []
+        description_blocks = next(
+            (s for s in sections if raw_text(s.find('title')) == 'Description'),
+            None,
+        )
+        if description_blocks is not None:
+            blocks = description_blocks.findall('./*')
+            description = list(
+                filter(
+                    bool,
+                    (re.sub(f'^{CommandDocs.BREAK}', '', cls.xml_text(p)) for p in blocks if p.tag != 'title'),
+                ),
+            )
+
+        # Notes parsing
+        notes = []
+        notes_blocks = next((s for s in sections if raw_text(s.find('title')) == 'Notes'), None)
+
+        if notes_blocks is not None:
+            blocks = notes_blocks.findall('./*')
+            notes = list(
+                filter(
+                    bool,
+                    (cls.xml_text(p) for p in blocks if p.tag != 'title'),
+                ),
+            )
+
+        # Parameters parsing
+        # Khronos specs puts all the function definitions inside funcsynopsis/funcdef blocks.
+        #
+        # However, instead of describing each function on a separate file, they group multiple
+        # related function definitions, whose parameters may be different, into a single file.
+        # This means that we have to find the correct block of parameters for each definition.
+        funcdefs = [
+            d for d in tree.findall('.//funcsynopsis/*')
+            if d.find('.//funcdef') is not None
+        ]
+
+        for func_def in funcdefs:
+            func_name = func_def.find('.//function').text
+            func_params = [raw_text(s) for s in func_def.findall('.//parameter')]
+
+
+            # Params are defined in a separate section, called 'Parameters for <func_name>'
+            # or just 'Parameters'.
+            params_block = next(
+                (s for s in sections if raw_text(s.find('title')) == f'Parameters for {func_name}'),
+                None,
+            )
+            if params_block is None:
+                for p in (s for s in sections if raw_text(s.find('title')) == 'Parameters'):
+                    block_params = [raw_text(n) for n in p.findall('.//term//parameter')]
+                    # If all our func_params are contained in this block, we choose it.
+                    if all(func_param in block_params for func_param in func_params):
+                        params_block = p
+                        break
+
+            # At this point we interpret params_block=None as a void parameter list.
+            is_void = params_block is None
+            params_entries = params_block.findall('.//varlistentry/*') if not is_void else []
+
+            params = []
+            # A description can apply for more than one param (term), so we stack them until
+            # we find a listitem, which is a description of a param.
+            terms_stack = []
+            for param_or_desc in params_entries:
+                if param_or_desc.tag == 'term':
+                    terms_stack.append(param_or_desc)
+                    continue
+                if param_or_desc.tag == 'listitem':
+                    # Now that we have a listitem,  it is considered that all the terms
+                    # on the stack have the same parameter description.
+                    param_desc = cls.xml_text(param_or_desc).replace(CommandDocs.BREAK, '')
+                    for terms in terms_stack:
+                        param_names = [
+                            p.text for p in terms.findall('.//parameter') if p.text in func_params
+                        ]
+
+                        for param_name in param_names:
+                            params.append(CommandDocs.Param(param_name, param_desc))
+                    terms_stack.clear()
+
+            commands_parsed[func_name] = CommandDocs(
+                func_name, brief, params, description, notes, None, None, docs_url,
+            )
+        return commands_parsed
+
+    @classmethod
+    def format(cls, e, parent=None, is_tail=False):
+        if is_tail:
+            if e.tag in ('term', 'mn', 'msub'):
+                return ''
+            return re.sub(r'\n+', '', e.tail)
+
+        if e.tag == 'constant':
+            return f'`{e.text}`'
+        if e.tag == 'function':
+            return f'`{e.text}`'
+        if e.tag == 'emphasis':
+            return f'*{e.text}*'
+        if e.tag == 'term':
+            return f'\n{CommandDocs.BREAK}- {e.text.strip()}'
+        if e.tag == 'listitem':
+            if parent is None:
+                return f'\n{CommandDocs.BREAK}{e.text.strip()}'
+            if parent is not None and parent.tag == 'varlistentry':
+                return f'\n{CommandDocs.BREAK}{e.text.strip()}'
+            return f'\n{CommandDocs.BREAK}- {e.text.strip()}'
+        return re.sub(r'\n+', '', e.text)
+
+    @classmethod
+    def xml_text(cls, e):
+        def paren(expr):
+            if re.match(r'^[a-zA-Z0-9_]+$', expr):
+                return expr
+            return f'({expr})'
+
+        def mfenced(e):
+            if e.attrib['close']:
+                return f'{e.attrib["open"]}{", ".join(cls.xml_text(c) for c in e)}{e.attrib["close"]}'
+            return f'{e.attrib["open"]}{" ".join(cls.xml_text(c) for c in e)}'
+
+        text = ''.join(glad.util.itertext(
+            e,
+            convert={
+                'table': lambda _: '(table omitted)',
+                'informaltable': lambda _: '(table omitted)',
+                'include': lambda _: '(table omitted)',
+                'programlisting': lambda _: '(code omitted)',
+                'mfrac': lambda e, : f'{paren(cls.xml_text(e[0]))}/{paren(cls.xml_text(e[1]))}',
+                'msup': lambda e: f'{paren(cls.xml_text(e[0]))}^{paren(cls.xml_text(e[1]))}',
+                'msub': lambda e: f'{paren(cls.xml_text(e[0]))}_{paren(cls.xml_text(e[1]))}',
+                'mtd': lambda e: f'{cls.xml_text(e[0])}; ',
+                'mfenced': mfenced,
+            },
+            format=cls.format,
+        ))
+        # \u2062, \u2062,
+        # Invisible characters used by docs.gl to separate words.
+        return re.sub(r'\n?[ \u2062\u2061\t]+', ' ', text.strip())

glad/generator/__init__.py

@@ -138,15 +138,16 @@ def modify_feature_set(self, spec, feature_set, config):
         """
         return feature_set
 
-    def get_template_arguments(self, spec, feature_set, config):
+    def get_template_arguments(self, spec, feature_set, config, spec_docs=None):
         return dict(
             spec=spec,
             feature_set=feature_set,
+            spec_docs=spec_docs,
             options=config.to_dict(transform=lambda x: x.lower()),
             gen_info=self.gen_info_factory(self, spec, feature_set, config)
         )
 
-    def generate(self, spec, feature_set, config, sink=LoggingSink(__name__)):
+    def generate(self, spec, feature_set, config, spec_docs=None, sink=LoggingSink(__name__)):
         feature_set = self.modify_feature_set(spec, feature_set, config)
         for template, output_path in self.get_templates(spec, feature_set, config):
             #try:
@@ -156,17 +157,17 @@ def generate(self, spec, feature_set, config, sink=LoggingSink(__name__)):
             #    raise ValueError('Unsupported specification/configuration')
 
             result = template.render(
-                **self.get_template_arguments(spec, feature_set, config)
+                **self.get_template_arguments(spec, feature_set, config, spec_docs=spec_docs)
             )
 
             output_path = os.path.join(self.path, output_path)
             makefiledir(output_path)
             with open(output_path, 'w') as f:
                 f.write(result)
 
-        self.post_generate(spec, feature_set, config)
+        self.post_generate(spec, feature_set, config, spec_docs=None)
 
-    def post_generate(self, spec, feature_set, config):
+    def post_generate(self, spec, feature_set, config, spec_docs=None):
         pass
 
 

glad/generator/c/__init__.py

@@ -378,8 +378,8 @@ def select(self, spec, api, version, profile, extensions, config, sink=LoggingSi
 
         return JinjaGenerator.select(self, spec, api, version, profile, extensions, config, sink=sink)
 
-    def get_template_arguments(self, spec, feature_set, config):
-        args = JinjaGenerator.get_template_arguments(self, spec, feature_set, config)
+    def get_template_arguments(self, spec, feature_set, config, spec_docs=None):
+        args = JinjaGenerator.get_template_arguments(self, spec, feature_set, config, spec_docs=spec_docs)
 
         # TODO allow MX for every specification/api
         if spec.name not in (VK.NAME, GL.NAME):
@@ -412,7 +412,7 @@ def get_templates(self, spec, feature_set, config):
 
         return templates
 
-    def post_generate(self, spec, feature_set, config):
+    def post_generate(self, spec, feature_set, config, spec_docs=None):
         self._add_additional_headers(feature_set, config)
 
     def modify_feature_set(self, spec, feature_set, config):