From 768fda8c558a6480ea6f989faa8e83331804ad71 Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Mon, 13 May 2024 17:13:19 -0500
Subject: [PATCH 01/33] add docs flag

---
 glad/generator/c/__init__.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/glad/generator/c/__init__.py b/glad/generator/c/__init__.py
index 70f27f8d..acdf6e3c 100644
--- a/glad/generator/c/__init__.py
+++ b/glad/generator/c/__init__.py
@@ -244,6 +244,11 @@ class CConfig(Config):
         default=False,
         description='On-demand function pointer loading, initialize on use (experimental)'
     )
+    WITH_DOCS = ConfigOption(
+        converter=bool,
+        default=False,
+        description='Include inline documentation in the generated files.'
+    )
 
     __constraints__ = [
         # RequirementConstraint(['MX_GLOBAL'], 'MX'),

From 67a5683610cabcc7f20a1a85191491d02ef8e42d Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Tue, 14 May 2024 13:13:37 -0500
Subject: [PATCH 02/33] add utils

---
 glad/util.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/glad/util.py b/glad/util.py
index 99f543e2..70b47ec6 100644
--- a/glad/util.py
+++ b/glad/util.py
@@ -1,4 +1,5 @@
 import functools
+from itertools import chain
 import os
 import re
 import sys
@@ -201,3 +202,20 @@ def expand_type_name(name):
         prefix = upper_name.rsplit(suffix, 1)[0]
 
     return ExpandedName(prefix, suffix)
+
+def flatten(l):
+    return list(chain.from_iterable(l))
+
+def prefix(prefix, text):
+    if not text:
+        return text
+    if text.strip().startswith(prefix):
+        return text
+    return f'{prefix}{text}'
+
+def suffix(suffix, text):
+    if not text:
+        return text
+    if text.strip().endswith(suffix):
+        return text
+    return f'{text}{suffix}'

From 283ce5b8986bc29f91449e1dc5510fd4ed36aee4 Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Tue, 14 May 2024 13:16:04 -0500
Subject: [PATCH 03/33] update .gitignore

---
 .gitignore | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index cc054834..a315df8c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,4 +18,6 @@ dist/
 /rust/
 target/
 Cargo.lock
-.vscode/
\ No newline at end of file
+.vscode/
+.cached/
+.venv/

From 148070ecc894fa3a2534724ef9cdb78bf9d2b3cf Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Tue, 14 May 2024 13:17:54 -0500
Subject: [PATCH 04/33] integrate code documentation

---
 glad/__main__.py | 11 +----------
 glad/parse.py    | 27 ++++++++++++++++++++++++---
 2 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/glad/__main__.py b/glad/__main__.py
index 5403809d..38fa923b 100644
--- a/glad/__main__.py
+++ b/glad/__main__.py
@@ -81,16 +81,7 @@ def load_specifications(specification_names, opener, specification_classes=None)
 
     for name in set(specification_names):
         Specification = specification_classes[name]
-        xml_name = name + '.xml'
-
-        if os.path.isfile(xml_name):
-            logger.info('using local specification: %s', xml_name)
-            specification = Specification.from_file(xml_name, opener=opener)
-        else:
-            logger.info('getting %r specification from remote location', name)
-            specification = Specification.from_remote(opener=opener)
-
-        specifications[name] = specification
+        specifications[name] = Specification.load(name, opener=opener)
 
     return specifications
 
diff --git a/glad/parse.py b/glad/parse.py
index 3465bca5..9487bd42 100644
--- a/glad/parse.py
+++ b/glad/parse.py
@@ -1,3 +1,4 @@
+from tkinter import N
 from glad.sink import LoggingSink
 
 try:
@@ -206,11 +207,12 @@ def __contains__(self, item):
 class Specification(object):
     API = 'https://cvs.khronos.org/svn/repos/ogl/trunk/doc/registry/public/api/'
     NAME = None
+    DOCS = None
 
     def __init__(self, root):
         self.root = root
-
         self._combined = None
+        self._docs = None
 
     def _magic_require(self, api, profile):
         """
@@ -254,6 +256,17 @@ def name(self):
 
         return self.NAME
 
+    @classmethod
+    def load(cls, name, opener=None):
+        xml_name = name + '.xml'
+
+        if os.path.isfile(xml_name):
+            logger.info('using local specification: %s', xml_name)
+            return cls.from_file(xml_name, opener=opener)
+        else:
+            logger.info('getting %r specification from remote location', name)
+            return cls.from_remote(opener=opener)
+
     @classmethod
     def from_url(cls, url, opener=None):
         if opener is None:
@@ -403,8 +416,9 @@ def commands(self):
             if len(parsed) > 0:
                 commands.setdefault(parsed[0].name, []).extend(parsed)
 
-        # fixup aliases
+        # populate docs and fixup aliases
         for command in chain.from_iterable(commands.values()):
+            command.doc_comment = self._docs.docs_for_name(command.name)
             if command.alias is not None and command.proto is None:
                 aliased_command = command
                 while aliased_command.proto is None:
@@ -732,6 +746,12 @@ def select(self, api, version, profile, extension_names, sink=LoggingSink(__name
         extensions = [self.extensions[api][name] for name in extension_names
                       if name in self.extensions[api]]
 
+        # Load documentation for the specific configuration
+        if self.DOCS:
+            self._docs = self.DOCS(api, version, profile, extensions)
+            sink.info('loading documentation for api {} version {}'.format(api, version))
+            self._docs.load()
+
         # Collect information
         result = set()
         # collect all required types, functions (=commands) and enums by API
@@ -1142,12 +1162,13 @@ def from_element(cls, element, extnumber=None, **kwargs):
 
 
 class Command(IdentifiedByName):
-    def __init__(self, name, api=None, proto=None, params=None, alias=None):
+    def __init__(self, name, api=None, proto=None, params=None, alias=None, doc_comment=None):
         self.name = name
         self.api = api
         self.proto = proto
         self.params = params
         self.alias = alias
+        self.doc_comment = doc_comment
 
         if self.alias is None and self.proto is None:
             raise ValueError("command is neither a full command nor an alias")

From 0034c226293811fc990411c4b965ea789743361f Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Tue, 14 May 2024 13:18:20 -0500
Subject: [PATCH 05/33] add support for doc comments on template

---
 glad/generator/c/templates/template_utils.h | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/glad/generator/c/templates/template_utils.h b/glad/generator/c/templates/template_utils.h
index cc5b320b..12102d47 100644
--- a/glad/generator/c/templates/template_utils.h
+++ b/glad/generator/c/templates/template_utils.h
@@ -116,6 +116,25 @@ typedef {{ command.proto.ret|type_to_c }} (GLAD_API_PTR *{{ command.name|pfn }})
 GLAD_API_CALL {{ command.name|pfn }} glad_{{ command.name }};
 {% if debug %}
 GLAD_API_CALL {{ command.name|pfn }} glad_debug_{{ command.name }};
+{% endif %}
+{% if options.with_docs and command.doc_comment %}
+/**
+ * @brief {{ command.doc_comment.brief | wordwrap(80, wrapstring='\n * ') }}
+ *
+{% for param in command.doc_comment.params %}
+ * @param {{ param.name }} {{ param.desc | wordwrap(80, wrapstring='\n * ') }}
+{% endfor %}
+ *
+{% for paragraph in command.doc_comment.description %}
+ * {{ paragraph | replace('<BREAK>', '@details ') | wordwrap(80, wrapstring='\n * ') }}
+{% endfor %}
+ *
+{% for note in command.doc_comment.notes %}
+ * {{ note | replace('<BREAK>', '@note ') | wordwrap(80, wrapstring='\n * ') }}
+{% endfor %}
+*/
+{% endif %}
+{% if debug %}
 #define {{ command.name }} glad_debug_{{ command.name }}
 {% else %}
 #define {{ command.name }} glad_{{ command.name }}

From 4129015aa807fd283f2d8013e23a79c4eaaf9a0b Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Fri, 17 May 2024 17:07:13 -0500
Subject: [PATCH 06/33] add docsgl documentation

---
 glad/parse.py         | 47 ++++++++++++++++++++++++++++++++++++++++---
 glad/specification.py |  2 ++
 2 files changed, 46 insertions(+), 3 deletions(-)

diff --git a/glad/parse.py b/glad/parse.py
index 9487bd42..c69c69eb 100644
--- a/glad/parse.py
+++ b/glad/parse.py
@@ -30,7 +30,7 @@ def xml_parse(path):
 from itertools import chain
 
 from glad.opener import URLOpener
-from glad.util import Version, topological_sort, memoize
+from glad.util import Version, raw_text, topological_sort, memoize
 import glad.util
 
 logger = logging.getLogger(__name__)
@@ -418,7 +418,7 @@ def commands(self):
 
         # populate docs and fixup aliases
         for command in chain.from_iterable(commands.values()):
-            command.doc_comment = self._docs.docs_for_name(command.name)
+            command.doc_comment = self._docs.docs_for_command_name(command.name)
             if command.alias is not None and command.proto is None:
                 aliased_command = command
                 while aliased_command.proto is None:
@@ -862,7 +862,7 @@ def from_element(element):
             # not so great workaround to get APIENTRY included in the raw output
             apientry.text = 'APIENTRY'
 
-        raw = ''.join(element.itertext())
+        raw = raw_text(element)
         api = element.get('api')
         category = element.get('category')
         name = element.get('name') or element.find('name').text
@@ -1474,3 +1474,44 @@ def from_element(cls, element, api=None):
     def __str__(self):
         return '{self.name}@{self.version!r}'.format(self=self)
     __repr__ = __str__
+
+
+class ApiDocumentation(object):
+    URL = None
+
+    def __init__(self, api, version, profile, extensions):
+        self.api = api
+        self.version = version
+        self.profile = profile
+        self.extensions = extensions
+
+    def load(self):
+        raise NotImplementedError
+
+    def docs_for_command_name(self, name):
+        """
+        Returns the CommandDocs for the given command name or None if not found.
+        """
+        raise NotImplementedError
+
+
+class CommandDocs(object):
+    class Param(namedtuple('Param', ['name', 'desc'])):
+        pass
+
+    def __init__(self, brief, params, description, notes, errors, see_also):
+        self.brief = brief
+        self.params = params
+        self.description = description
+        self.notes = notes
+        self.errors = errors
+        self.see_also = see_also
+
+    def __str__(self):
+        return 'DocComment(brief={!r}, ' \
+            'params={!r}, description={!r}, notes={!r}, errors={!r}, see_also={!r})' \
+            .format(
+                self.brief, self.params, self.description, self.notes, self.errors, self.see_also,
+            )
+
+    __repr__ = __str__
diff --git a/glad/specification.py b/glad/specification.py
index 8030ee1e..cb4606c4 100644
--- a/glad/specification.py
+++ b/glad/specification.py
@@ -1,4 +1,5 @@
 from glad.parse import Specification, Require
+from glad.documentation import DocsGL
 
 
 class EGL(Specification):
@@ -13,6 +14,7 @@ def protections(self, symbol, api=None, profile=None, feature_set=None):
 
 class GL(Specification):
     DISPLAY_NAME = 'OpenGL'
+    DOCS = DocsGL
 
     API = 'https://raw.githubusercontent.com/KhronosGroup/OpenGL-Registry/main/xml/'
     NAME = 'gl'

From 7fca80ccdf707da85771a748a2d7633796213ba0 Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Fri, 17 May 2024 17:07:30 -0500
Subject: [PATCH 07/33] improve templating

---
 glad/generator/c/templates/template_utils.h | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/glad/generator/c/templates/template_utils.h b/glad/generator/c/templates/template_utils.h
index 12102d47..01dbcb10 100644
--- a/glad/generator/c/templates/template_utils.h
+++ b/glad/generator/c/templates/template_utils.h
@@ -119,8 +119,10 @@ GLAD_API_CALL {{ command.name|pfn }} glad_debug_{{ command.name }};
 {% endif %}
 {% if options.with_docs and command.doc_comment %}
 /**
+{% if command.doc_comment.brief %}
  * @brief {{ command.doc_comment.brief | wordwrap(80, wrapstring='\n * ') }}
  *
+{% endif %}
 {% for param in command.doc_comment.params %}
  * @param {{ param.name }} {{ param.desc | wordwrap(80, wrapstring='\n * ') }}
 {% endfor %}
@@ -132,6 +134,9 @@ GLAD_API_CALL {{ command.name|pfn }} glad_debug_{{ command.name }};
 {% for note in command.doc_comment.notes %}
  * {{ note | replace('<BREAK>', '@note ') | wordwrap(80, wrapstring='\n * ') }}
 {% endfor %}
+{% if command.doc_comment.see_also %}
+ * @see {{ command.doc_comment.see_also }}
+{% endif %}
 */
 {% endif %}
 {% if debug %}

From c1f7e86c568e145b14daa302a6f8127131a55c3b Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Fri, 17 May 2024 20:15:18 -0500
Subject: [PATCH 08/33] make text parsing utilities more versatile

---
 glad/util.py | 26 +++++++++++++++++++-------
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/glad/util.py b/glad/util.py
index 70b47ec6..b96c7c9f 100644
--- a/glad/util.py
+++ b/glad/util.py
@@ -171,18 +171,30 @@ def memoized(*args, **kwargs):
     return memoize_decorator
 
 
-def itertext(element, ignore=()):
+def raw_text(e):
+    if e is None:
+        return ''
+    return ''.join(e.itertext())
+
+
+def _format_none(e, is_tail=False):
+    return e.tail if is_tail else e.text
+
+
+def itertext(element, ignore=(), format=_format_none):
     tag = element.tag
+    if tag in ignore:
+        return
+
     if not isinstance(tag, basestring) and tag is not None:
         return
     if element.text:
-        yield element.text
+        yield format(element)
     for e in element:
-        if not e.tag in ignore:
-            for s in itertext(e, ignore=ignore):
-                yield s
-            if e.tail:
-                yield e.tail
+        for s in itertext(e, ignore=ignore, format=format):
+            yield s
+        if e.tail:
+            yield format(e, is_tail=True)
 
 
 def expand_type_name(name):

From a8fc0330a7be9d38aebb197efb8134de9f91a3c4 Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Fri, 17 May 2024 21:18:05 -0500
Subject: [PATCH 09/33] support custom break logic for templates

---
 glad/generator/c/templates/template_utils.h | 4 ++--
 glad/parse.py                               | 8 ++++++++
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/glad/generator/c/templates/template_utils.h b/glad/generator/c/templates/template_utils.h
index 01dbcb10..60bf8391 100644
--- a/glad/generator/c/templates/template_utils.h
+++ b/glad/generator/c/templates/template_utils.h
@@ -128,11 +128,11 @@ GLAD_API_CALL {{ command.name|pfn }} glad_debug_{{ command.name }};
 {% endfor %}
  *
 {% for paragraph in command.doc_comment.description %}
- * {{ paragraph | replace('<BREAK>', '@details ') | wordwrap(80, wrapstring='\n * ') }}
+ * {{ paragraph | replace(command.doc_comment.BREAK, '@details ') | wordwrap(80, wrapstring='\n * ') }}
 {% endfor %}
  *
 {% for note in command.doc_comment.notes %}
- * {{ note | replace('<BREAK>', '@note ') | wordwrap(80, wrapstring='\n * ') }}
+ * {{ note | replace(command.doc_comment.BREAK, '@note ') | wordwrap(80, wrapstring='\n * ') }}
 {% endfor %}
 {% if command.doc_comment.see_also %}
  * @see {{ command.doc_comment.see_also }}
diff --git a/glad/parse.py b/glad/parse.py
index c69c69eb..d31a4d80 100644
--- a/glad/parse.py
+++ b/glad/parse.py
@@ -1496,6 +1496,14 @@ def docs_for_command_name(self, name):
 
 
 class CommandDocs(object):
+    """
+    Inline code documentation for a command/function.
+    """
+
+    # Template rendering will interpret this as a custom paragraph break.
+    # If no special break logic is needed, just use '\n'.
+    BREAK = '__BREAK__'
+
     class Param(namedtuple('Param', ['name', 'desc'])):
         pass
 

From ec699c1345d1b12eff7b2bb8b81d09738bf5a0e8 Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Sat, 18 May 2024 02:11:05 -0500
Subject: [PATCH 10/33] implement DocsGL documentation

---
 glad/documentation.py | 170 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 170 insertions(+)
 create mode 100644 glad/documentation.py

diff --git a/glad/documentation.py b/glad/documentation.py
new file mode 100644
index 00000000..e97768bb
--- /dev/null
+++ b/glad/documentation.py
@@ -0,0 +1,170 @@
+from glad.parse import ApiDocumentation, CommandDocs, xml_parse
+from glad.util import prefix, memoize, raw_text
+from shutil import rmtree
+import glad.util
+import subprocess
+import re
+
+
+class DocsGL(ApiDocumentation):
+    URL = 'https://github.com/BSVino/docs.gl.git'
+    CACHED = True # Only clones the DocsGL repo once
+    API = 'gl'
+    docs = dict()
+
+    @property
+    @memoize(method=True)
+    def out_dir(self):
+        from pathlib import Path
+        from tempfile import gettempdir
+        if self.CACHED:
+            return Path('.cached') / 'docs.gl'
+        return Path(gettempdir()) / 'docs.gl'
+
+    def load(self):
+        if self.out_dir.exists() and not self.CACHED:
+            rmtree(str(self.out_dir))
+        if not self.out_dir.exists():
+            subprocess.run(['git', 'clone', '--depth=1', self.URL, str(self.out_dir)])
+
+        current_version = self.version.major
+
+        # As the time of writing DocsGL offers documentation from gl4 to gl2.
+        # If say we are targeting gl3, we will try to get the command documentation from gl3,
+        # otherwise we'll try from gl2 and so on. If more than one version is available only the
+        # most recent one will be used.
+        for version in range(current_version, 1, -1):
+            docs_dir = self.out_dir / f'{self.API}{version}'
+            if not docs_dir.exists():
+                break
+
+            for html_file in docs_dir.glob('*.xhtml'):
+                for func, docs in DocsGL.docs_from_html_file(html_file).items():
+                    self.docs.setdefault(func, docs)
+
+    def docs_for_command_name(self, name):
+        return self.docs.get(name, None)
+
+    @classmethod
+    def docs_from_html_file(cls, path):
+        commands_parsed = dict()
+
+        tree = xml_parse(path)
+        sections = tree.findall('.//*[@class="refsect1"]')
+
+        # Some pages are just a redirect script
+        if tree.tag == 'script':
+            try:
+                redirect = tree.text.split('window.location.replace("')[1].split('")')[0]
+                path = path.parent / f'{redirect}.xhtml'
+                tree = xml_parse(path)
+            except:
+                return dict()
+
+        # Brief parsing
+        # Command brief description appears in the first 'refnamediv' block
+        brief_block = cls.xml_text(tree.find('.//div[@class="refnamediv"]/p'))
+        brief = f'[{path.stem}](https://docs.gl/{path.parent.name}/{path.stem}) — ' \
+            f'{brief_block.split("—")[1]}'
+
+        # Description parsing
+        description = []
+        description_blocks = next(
+            (s for s in sections if raw_text(s.find('h2')) == 'Description'),
+            None,
+        )
+        if description_blocks:
+            blocks = description_blocks.findall('./*')
+            description = list(
+                filter(
+                    bool,
+                    (prefix(CommandDocs.BREAK, cls.xml_text(p)) for p in blocks if p.tag != 'h2'),
+                ),
+            )
+
+        # Notes parsing
+        notes = []
+        notes_blocks = next((s for s in sections if raw_text(s.find('h2')) == 'Notes'), None)
+        if notes_blocks:
+            blocks = notes_blocks.findall('./*')
+            notes = list(
+                filter(
+                    bool,
+                    (prefix(CommandDocs.BREAK, cls.xml_text(p)) for p in blocks if p.tag != 'h2'),
+                ),
+            )
+
+        # Parameters parsing
+        # DocsGL puts all the function definitions inside .funcsynopsis.funcdef blocks.
+        #
+        # However, instead of describing each function on a separate file, DocsGL combines
+        # multiple related function definitions, whose parameters may be different, into a single
+        # file. This means that we have to find the correct block of parameters for each definition.
+        funcdefs = [
+            d for d in tree.findall('.//*[@class="funcsynopsis"]/*')
+            if d.find('.//*[@class="funcdef"]')
+        ]
+        for func_def in funcdefs:
+            func_name = func_def.find('.//*[@class="fsfunc"]').text
+            func_params = [raw_text(s) for s in func_def.findall('.//var[@class="pdparam"]')]
+
+            # Params are defined in a separate section, often called 'Parameters for <func_name>'
+            # or just 'Parameters'.
+            params_block = next(
+                (s for s in sections if raw_text(s.find('h2')) == f'Parameters for {func_name}'),
+                None,
+            )
+            if not params_block:
+                for p in list(s for s in sections if raw_text(s.find('h2')) == 'Parameters'):
+                    block_params = [raw_text(n) for n in p.findall('.//dt//code')]
+                    if all(p in block_params for p in func_params):
+                        params_block = p
+                        break
+
+            params = []
+            if params_block is not None:
+                for name, desc in zip(
+                    params_block.findall('.//dl//dt//code'),
+                    params_block.findall('.//dl//dd/p'),
+                ):
+                    param_name = raw_text(name)
+                    if param_name in func_params:
+                        params.append(CommandDocs.Param(param_name, cls.xml_text(desc)))
+            # We interpret params_block=None as a void parameter list.
+
+            commands_parsed[func_name] = CommandDocs(
+                brief, params, description, notes, None, None,
+            )
+        return commands_parsed
+
+    @staticmethod
+    def format(e, is_tail=False):
+        if is_tail:
+            if e.tag == 'mfenced':
+                # closing mathjax fences
+                return f'{e.attrib["close"]}'
+            if e.tag == 'dt':
+                # closing a definition term
+                return '\n'
+            return e.tail
+
+        if e.tag == 'a':
+            return f'![{e.text}]({e.attrib["href"]})'
+        if e.tag == 'code':
+            return f'`{e.text}`'
+        if e.tag == 'mfenced':
+            return f'{e.attrib["open"]}{e.text}'
+        if e.tag == 'dt':
+            return f'\n{CommandDocs.BREAK}-'
+        if e.tag == 'li':
+            return f'\n{CommandDocs.BREAK}-{e.text}'
+        return e.text
+
+    @staticmethod
+    def xml_text(e):
+        text = ''.join(glad.util.itertext(
+            e,
+            ignore=('table', 'pre'), # tables and code blocks are not supported yet
+            format=DocsGL.format,
+        ))
+        return re.sub(r'\s+', ' ', text.strip())

From 5977606728eacc3b688423ab3143a155a8afe0eb Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Sat, 18 May 2024 03:39:04 -0500
Subject: [PATCH 11/33] fix parsing for descriptions with multiple parameters

---
 glad/documentation.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/glad/documentation.py b/glad/documentation.py
index e97768bb..c7dae0ec 100644
--- a/glad/documentation.py
+++ b/glad/documentation.py
@@ -123,13 +123,14 @@ def docs_from_html_file(cls, path):
 
             params = []
             if params_block is not None:
-                for name, desc in zip(
-                    params_block.findall('.//dl//dt//code'),
+                for names, desc in zip(
+                    params_block.findall('.//dl//dt'),
                     params_block.findall('.//dl//dd/p'),
                 ):
-                    param_name = raw_text(name)
-                    if param_name in func_params:
-                        params.append(CommandDocs.Param(param_name, cls.xml_text(desc)))
+                    for name in names.findall('.//code'):
+                        param_name = raw_text(name)
+                        if param_name in func_params:
+                            params.append(CommandDocs.Param(param_name, cls.xml_text(desc)))
             # We interpret params_block=None as a void parameter list.
 
             commands_parsed[func_name] = CommandDocs(

From f2c4fbb5f4a917a943eed145a108054b578f1917 Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Sat, 18 May 2024 03:48:10 -0500
Subject: [PATCH 12/33] fully parse doc params description

---
 glad/documentation.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/glad/documentation.py b/glad/documentation.py
index c7dae0ec..6f0f2e59 100644
--- a/glad/documentation.py
+++ b/glad/documentation.py
@@ -125,7 +125,7 @@ def docs_from_html_file(cls, path):
             if params_block is not None:
                 for names, desc in zip(
                     params_block.findall('.//dl//dt'),
-                    params_block.findall('.//dl//dd/p'),
+                    params_block.findall('.//dl//dd'),
                 ):
                     for name in names.findall('.//code'):
                         param_name = raw_text(name)

From 0e267f72a06a9ed4612e171338bf1c52edeae367 Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Sat, 18 May 2024 04:30:07 -0500
Subject: [PATCH 13/33] fix crash when no --with-docs is present

---
 glad/parse.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/glad/parse.py b/glad/parse.py
index d31a4d80..ee407e5a 100644
--- a/glad/parse.py
+++ b/glad/parse.py
@@ -418,7 +418,8 @@ def commands(self):
 
         # populate docs and fixup aliases
         for command in chain.from_iterable(commands.values()):
-            command.doc_comment = self._docs.docs_for_command_name(command.name)
+            if self._docs:
+                command.doc_comment = self._docs.docs_for_command_name(command.name)
             if command.alias is not None and command.proto is None:
                 aliased_command = command
                 while aliased_command.proto is None:

From 947a8f0c8002e8e89f1309d7d777f2f892a0f032 Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Sat, 18 May 2024 04:38:38 -0500
Subject: [PATCH 14/33] fix pipe format for templates

---
 glad/generator/c/templates/template_utils.h | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/glad/generator/c/templates/template_utils.h b/glad/generator/c/templates/template_utils.h
index 60bf8391..7f06dcce 100644
--- a/glad/generator/c/templates/template_utils.h
+++ b/glad/generator/c/templates/template_utils.h
@@ -120,19 +120,19 @@ GLAD_API_CALL {{ command.name|pfn }} glad_debug_{{ command.name }};
 {% if options.with_docs and command.doc_comment %}
 /**
 {% if command.doc_comment.brief %}
- * @brief {{ command.doc_comment.brief | wordwrap(80, wrapstring='\n * ') }}
+ * @brief {{ command.doc_comment.brief|wordwrap(80, wrapstring='\n * ') }}
  *
 {% endif %}
 {% for param in command.doc_comment.params %}
- * @param {{ param.name }} {{ param.desc | wordwrap(80, wrapstring='\n * ') }}
+ * @param {{ param.name }} {{ param.desc|wordwrap(80, wrapstring='\n * ') }}
 {% endfor %}
  *
 {% for paragraph in command.doc_comment.description %}
- * {{ paragraph | replace(command.doc_comment.BREAK, '@details ') | wordwrap(80, wrapstring='\n * ') }}
+ * {{ paragraph|replace(command.doc_comment.BREAK, '@details ')|wordwrap(80, wrapstring='\n * ') }}
 {% endfor %}
  *
 {% for note in command.doc_comment.notes %}
- * {{ note | replace(command.doc_comment.BREAK, '@note ') | wordwrap(80, wrapstring='\n * ') }}
+ * {{ note|replace(command.doc_comment.BREAK, '@note ')|wordwrap(80, wrapstring='\n * ') }}
 {% endfor %}
 {% if command.doc_comment.see_also %}
  * @see {{ command.doc_comment.see_also }}

From 48d8babedd9e5ad12d1a920333ee01e9ac4e1806 Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Sat, 18 May 2024 05:08:10 -0500
Subject: [PATCH 15/33] rename ApiDocumentation to SpecificationDocs

---
 glad/documentation.py | 4 ++--
 glad/parse.py         | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/glad/documentation.py b/glad/documentation.py
index 6f0f2e59..3da3546a 100644
--- a/glad/documentation.py
+++ b/glad/documentation.py
@@ -1,4 +1,4 @@
-from glad.parse import ApiDocumentation, CommandDocs, xml_parse
+from glad.parse import SpecificationDocs, CommandDocs, xml_parse
 from glad.util import prefix, memoize, raw_text
 from shutil import rmtree
 import glad.util
@@ -6,7 +6,7 @@
 import re
 
 
-class DocsGL(ApiDocumentation):
+class DocsGL(SpecificationDocs):
     URL = 'https://github.com/BSVino/docs.gl.git'
     CACHED = True # Only clones the DocsGL repo once
     API = 'gl'
diff --git a/glad/parse.py b/glad/parse.py
index ee407e5a..33a99c01 100644
--- a/glad/parse.py
+++ b/glad/parse.py
@@ -1477,7 +1477,7 @@ def __str__(self):
     __repr__ = __str__
 
 
-class ApiDocumentation(object):
+class SpecificationDocs(object):
     URL = None
 
     def __init__(self, api, version, profile, extensions):

From 3a1514684f40969d2d2174cb894ea23757f140f5 Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Sat, 18 May 2024 05:22:21 -0500
Subject: [PATCH 16/33] only load documentation if --with-docs is present

---
 glad/generator/__init__.py |  2 +-
 glad/parse.py              | 13 ++++++++-----
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/glad/generator/__init__.py b/glad/generator/__init__.py
index 75e017cf..14b4fb4d 100644
--- a/glad/generator/__init__.py
+++ b/glad/generator/__init__.py
@@ -57,7 +57,7 @@ def select(self, spec, api, version, profile, extensions, config, sink=LoggingSi
         :param sink: sink used to collect non fatal errors and information
         :return: FeatureSet with the required types, enums, commands/functions
         """
-        return spec.select(api, version, profile, extensions, sink=sink)
+        return spec.select(api, version, profile, extensions, config, sink=sink)
 
     def generate(self, spec, feature_set, config, sink=LoggingSink(__name__)):
         """
diff --git a/glad/parse.py b/glad/parse.py
index 33a99c01..736dc133 100644
--- a/glad/parse.py
+++ b/glad/parse.py
@@ -669,7 +669,7 @@ def split_types(iterable, types):
 
         return result
 
-    def select(self, api, version, profile, extension_names, sink=LoggingSink(__name__)):
+    def select(self, api, version, profile, extension_names, config, sink=LoggingSink(__name__)):
         """
         Select a specific configuration from the specification.
 
@@ -748,10 +748,13 @@ def select(self, api, version, profile, extension_names, sink=LoggingSink(__name
                       if name in self.extensions[api]]
 
         # Load documentation for the specific configuration
-        if self.DOCS:
-            self._docs = self.DOCS(api, version, profile, extensions)
-            sink.info('loading documentation for api {} version {}'.format(api, version))
-            self._docs.load()
+        if config['WITH_DOCS']:
+            if self.DOCS:
+                self._docs = self.DOCS(api, version, profile, extensions)
+                sink.info('loading documentation for api {} version {}'.format(api, version))
+                self._docs.load()
+            else:
+                sink.warning("documentation not available for specification '{}'".format(self.name))
 
         # Collect information
         result = set()

From 3cd03ec6c34da09300aef54aa2f83c63d8cc6621 Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Sat, 18 May 2024 19:46:48 -0500
Subject: [PATCH 17/33] fix <dt> listing format

Forgot to add spaces when listing ('-') <dt> elements.

Also, the regex was improperly removing contiguous line breaks.
---
 glad/documentation.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/glad/documentation.py b/glad/documentation.py
index 3da3546a..691294da 100644
--- a/glad/documentation.py
+++ b/glad/documentation.py
@@ -156,7 +156,7 @@ def format(e, is_tail=False):
         if e.tag == 'mfenced':
             return f'{e.attrib["open"]}{e.text}'
         if e.tag == 'dt':
-            return f'\n{CommandDocs.BREAK}-'
+            return f'\n{CommandDocs.BREAK}- '
         if e.tag == 'li':
             return f'\n{CommandDocs.BREAK}-{e.text}'
         return e.text
@@ -168,4 +168,4 @@ def xml_text(e):
             ignore=('table', 'pre'), # tables and code blocks are not supported yet
             format=DocsGL.format,
         ))
-        return re.sub(r'\s+', ' ', text.strip())
+        return re.sub(r'\n? +', ' ', text.strip())

From d87dfc42fc27b261eee8d887ec60c2f58ab755ef Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Wed, 22 May 2024 11:09:37 -0500
Subject: [PATCH 18/33] improve parsing and add basic support for equations

---
 glad/documentation.py | 51 ++++++++++++++++++++++++++++++-------------
 glad/parse.py         |  1 -
 glad/util.py          |  7 ++++--
 3 files changed, 41 insertions(+), 18 deletions(-)

diff --git a/glad/documentation.py b/glad/documentation.py
index 691294da..b9f15985 100644
--- a/glad/documentation.py
+++ b/glad/documentation.py
@@ -1,5 +1,5 @@
 from glad.parse import SpecificationDocs, CommandDocs, xml_parse
-from glad.util import prefix, memoize, raw_text
+from glad.util import prefix, suffix, memoize, raw_text
 from shutil import rmtree
 import glad.util
 import subprocess
@@ -65,7 +65,7 @@ def docs_from_html_file(cls, path):
         # Command brief description appears in the first 'refnamediv' block
         brief_block = cls.xml_text(tree.find('.//div[@class="refnamediv"]/p'))
         brief = f'[{path.stem}](https://docs.gl/{path.parent.name}/{path.stem}) — ' \
-            f'{brief_block.split("—")[1]}'
+            f'{suffix(".", brief_block.split("—")[1])}'
 
         # Description parsing
         description = []
@@ -73,7 +73,7 @@ def docs_from_html_file(cls, path):
             (s for s in sections if raw_text(s.find('h2')) == 'Description'),
             None,
         )
-        if description_blocks:
+        if description_blocks is not None:
             blocks = description_blocks.findall('./*')
             description = list(
                 filter(
@@ -85,7 +85,7 @@ def docs_from_html_file(cls, path):
         # Notes parsing
         notes = []
         notes_blocks = next((s for s in sections if raw_text(s.find('h2')) == 'Notes'), None)
-        if notes_blocks:
+        if notes_blocks is not None:
             blocks = notes_blocks.findall('./*')
             notes = list(
                 filter(
@@ -102,7 +102,7 @@ def docs_from_html_file(cls, path):
         # file. This means that we have to find the correct block of parameters for each definition.
         funcdefs = [
             d for d in tree.findall('.//*[@class="funcsynopsis"]/*')
-            if d.find('.//*[@class="funcdef"]')
+            if d.find('.//*[@class="funcdef"]') is not None
         ]
         for func_def in funcdefs:
             func_name = func_def.find('.//*[@class="fsfunc"]').text
@@ -114,7 +114,7 @@ def docs_from_html_file(cls, path):
                 (s for s in sections if raw_text(s.find('h2')) == f'Parameters for {func_name}'),
                 None,
             )
-            if not params_block:
+            if not params_block is not None:
                 for p in list(s for s in sections if raw_text(s.find('h2')) == 'Parameters'):
                     block_params = [raw_text(n) for n in p.findall('.//dt//code')]
                     if all(p in block_params for p in func_params):
@@ -141,31 +141,52 @@ def docs_from_html_file(cls, path):
     @staticmethod
     def format(e, is_tail=False):
         if is_tail:
-            if e.tag == 'mfenced':
-                # closing mathjax fences
-                return f'{e.attrib["close"]}'
             if e.tag == 'dt':
                 # closing a definition term
                 return '\n'
-            return e.tail
+            if e.tag == 'mtr':
+                # closing a mathjax row
+                return '\n'
+            r = re.sub(r'\n+', '', e.tail)
+            if e.tag in ('mn', 'msub'):
+                return ''
+            return re.sub(r'\n+', '', e.tail)
 
         if e.tag == 'a':
             return f'![{e.text}]({e.attrib["href"]})'
         if e.tag == 'code':
             return f'`{e.text}`'
-        if e.tag == 'mfenced':
-            return f'{e.attrib["open"]}{e.text}'
         if e.tag == 'dt':
             return f'\n{CommandDocs.BREAK}- '
         if e.tag == 'li':
             return f'\n{CommandDocs.BREAK}-{e.text}'
-        return e.text
+        return re.sub(r'\n+', '', e.text)
 
     @staticmethod
     def xml_text(e):
+        def paren(expr):
+            if re.match(r'^[a-zA-Z0-9_]+$', expr):
+                return expr
+            return f'({expr})'
+
+        def mfenced(e):
+            if e.attrib['close']:
+                return f'{e.attrib["open"]}{", ".join(DocsGL.xml_text(c) for c in e)}{e.attrib["close"]}'
+            return f'{e.attrib["open"]}{" ".join(DocsGL.xml_text(c) for c in e)}'
+
         text = ''.join(glad.util.itertext(
             e,
-            ignore=('table', 'pre'), # tables and code blocks are not supported yet
+            convert={
+                'table': lambda _: f'(table omitted)',
+                'pre': lambda _: f'(code omitted)',
+                'mfrac': lambda e, : f'{paren(DocsGL.xml_text(e[0]))}/{paren(DocsGL.xml_text(e[1]))}',
+                'msup': lambda e: f'{paren(DocsGL.xml_text(e[0]))}^{paren(DocsGL.xml_text(e[1]))}',
+                'msub': lambda e: f'{paren(DocsGL.xml_text(e[0]))}_{paren(DocsGL.xml_text(e[1]))}',
+                'mtd': lambda e: f'{DocsGL.xml_text(e[0])}; ',
+                'mfenced': mfenced,
+            },
             format=DocsGL.format,
         ))
-        return re.sub(r'\n? +', ' ', text.strip())
+        # \u00a0, \u2062, \u2062,
+        # are invisible characters used by docs.gl to separate words.
+        return re.sub(r'\n?[ \u00a0\u2062\u2061]+', ' ', text.strip())
diff --git a/glad/parse.py b/glad/parse.py
index 736dc133..f7a95003 100644
--- a/glad/parse.py
+++ b/glad/parse.py
@@ -1,4 +1,3 @@
-from tkinter import N
 from glad.sink import LoggingSink
 
 try:
diff --git a/glad/util.py b/glad/util.py
index b96c7c9f..25f8ac5e 100644
--- a/glad/util.py
+++ b/glad/util.py
@@ -181,17 +181,20 @@ def _format_none(e, is_tail=False):
     return e.tail if is_tail else e.text
 
 
-def itertext(element, ignore=(), format=_format_none):
+def itertext(element, ignore=(), convert=dict(), format=_format_none):
     tag = element.tag
     if tag in ignore:
         return
+    if tag in convert:
+        yield convert[tag](element)
+        return
 
     if not isinstance(tag, basestring) and tag is not None:
         return
     if element.text:
         yield format(element)
     for e in element:
-        for s in itertext(e, ignore=ignore, format=format):
+        for s in itertext(e, ignore=ignore, convert=convert, format=format):
             yield s
         if e.tail:
             yield format(e, is_tail=True)

From 300b6fe7e2493844892b6e50a0f1d1373b97d300 Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Fri, 24 May 2024 22:42:13 -0500
Subject: [PATCH 19/33] revert specification load change

---
 glad/__main__.py | 11 ++++++++++-
 glad/parse.py    | 14 ++------------
 2 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/glad/__main__.py b/glad/__main__.py
index 38fa923b..5403809d 100644
--- a/glad/__main__.py
+++ b/glad/__main__.py
@@ -81,7 +81,16 @@ def load_specifications(specification_names, opener, specification_classes=None)
 
     for name in set(specification_names):
         Specification = specification_classes[name]
-        specifications[name] = Specification.load(name, opener=opener)
+        xml_name = name + '.xml'
+
+        if os.path.isfile(xml_name):
+            logger.info('using local specification: %s', xml_name)
+            specification = Specification.from_file(xml_name, opener=opener)
+        else:
+            logger.info('getting %r specification from remote location', name)
+            specification = Specification.from_remote(opener=opener)
+
+        specifications[name] = specification
 
     return specifications
 
diff --git a/glad/parse.py b/glad/parse.py
index f7a95003..e8d15339 100644
--- a/glad/parse.py
+++ b/glad/parse.py
@@ -255,17 +255,6 @@ def name(self):
 
         return self.NAME
 
-    @classmethod
-    def load(cls, name, opener=None):
-        xml_name = name + '.xml'
-
-        if os.path.isfile(xml_name):
-            logger.info('using local specification: %s', xml_name)
-            return cls.from_file(xml_name, opener=opener)
-        else:
-            logger.info('getting %r specification from remote location', name)
-            return cls.from_remote(opener=opener)
-
     @classmethod
     def from_url(cls, url, opener=None):
         if opener is None:
@@ -1487,6 +1476,7 @@ def __init__(self, api, version, profile, extensions):
         self.version = version
         self.profile = profile
         self.extensions = extensions
+        self.docs = dict()
 
     def load(self):
         raise NotImplementedError
@@ -1519,7 +1509,7 @@ def __init__(self, brief, params, description, notes, errors, see_also):
         self.see_also = see_also
 
     def __str__(self):
-        return 'DocComment(brief={!r}, ' \
+        return 'CommandDocs(brief={!r}, ' \
             'params={!r}, description={!r}, notes={!r}, errors={!r}, see_also={!r})' \
             .format(
                 self.brief, self.params, self.description, self.notes, self.errors, self.see_also,

From 5b5798cacf33d5e906d62fbf4abe2b2848c1a716 Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Fri, 24 May 2024 23:27:52 -0500
Subject: [PATCH 20/33] remove specification-docs dependency

---
 glad/parse.py         | 13 ++-----------
 glad/specification.py |  2 --
 2 files changed, 2 insertions(+), 13 deletions(-)

diff --git a/glad/parse.py b/glad/parse.py
index e8d15339..698af524 100644
--- a/glad/parse.py
+++ b/glad/parse.py
@@ -206,7 +206,6 @@ def __contains__(self, item):
 class Specification(object):
     API = 'https://cvs.khronos.org/svn/repos/ogl/trunk/doc/registry/public/api/'
     NAME = None
-    DOCS = None
 
     def __init__(self, root):
         self.root = root
@@ -657,7 +656,7 @@ def split_types(iterable, types):
 
         return result
 
-    def select(self, api, version, profile, extension_names, config, sink=LoggingSink(__name__)):
+    def select(self, api, version, profile, extension_names, sink=LoggingSink(__name__)):
         """
         Select a specific configuration from the specification.
 
@@ -735,15 +734,6 @@ def select(self, api, version, profile, extension_names, config, sink=LoggingSin
         extensions = [self.extensions[api][name] for name in extension_names
                       if name in self.extensions[api]]
 
-        # Load documentation for the specific configuration
-        if config['WITH_DOCS']:
-            if self.DOCS:
-                self._docs = self.DOCS(api, version, profile, extensions)
-                sink.info('loading documentation for api {} version {}'.format(api, version))
-                self._docs.load()
-            else:
-                sink.warning("documentation not available for specification '{}'".format(self.name))
-
         # Collect information
         result = set()
         # collect all required types, functions (=commands) and enums by API
@@ -1469,6 +1459,7 @@ def __str__(self):
 
 
 class SpecificationDocs(object):
+    SPEC = None
     URL = None
 
     def __init__(self, api, version, profile, extensions):
diff --git a/glad/specification.py b/glad/specification.py
index cb4606c4..8030ee1e 100644
--- a/glad/specification.py
+++ b/glad/specification.py
@@ -1,5 +1,4 @@
 from glad.parse import Specification, Require
-from glad.documentation import DocsGL
 
 
 class EGL(Specification):
@@ -14,7 +13,6 @@ def protections(self, symbol, api=None, profile=None, feature_set=None):
 
 class GL(Specification):
     DISPLAY_NAME = 'OpenGL'
-    DOCS = DocsGL
 
     API = 'https://raw.githubusercontent.com/KhronosGroup/OpenGL-Registry/main/xml/'
     NAME = 'gl'

From c71a78979ea2bf3125143d780b20aec6f0d369d0 Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Sat, 25 May 2024 19:49:50 -0500
Subject: [PATCH 21/33] refactor: pass docs to generator.generate instead of
 the specification

---
 glad/__main__.py                            | 40 +++++++++++--
 glad/documentation.py                       | 52 ++++++-----------
 glad/generator/__init__.py                  | 13 +++--
 glad/generator/c/__init__.py                |  6 +-
 glad/generator/c/templates/template_utils.h | 21 +++----
 glad/generator/rust/__init__.py             |  4 +-
 glad/parse.py                               | 64 ++++++++++++++++-----
 glad/plugin.py                              | 17 +++++-
 8 files changed, 142 insertions(+), 75 deletions(-)

diff --git a/glad/__main__.py b/glad/__main__.py
index 5403809d..1a1b05f9 100644
--- a/glad/__main__.py
+++ b/glad/__main__.py
@@ -19,7 +19,7 @@
 from glad.sink import LoggingSink
 from glad.opener import URLOpener
 from glad.parse import FeatureSet
-from glad.plugin import find_specifications, find_generators
+from glad.plugin import find_specification_docs, find_specifications, find_generators
 from glad.util import parse_apis
 
 
@@ -95,6 +95,28 @@ def load_specifications(specification_names, opener, specification_classes=None)
     return specifications
 
 
+def load_documentations(specification_names, opener, spec_docs_classes=None):
+    specifications_docs = dict()
+
+    if spec_docs_classes is None:
+        spec_docs_classes = find_specification_docs()
+
+    for name in set(specification_names):
+        SpecificationDocs = spec_docs_classes[name]
+        spec_docs_dir = SpecificationDocs.default_out_dir()
+
+        if os.path.isdir(spec_docs_dir):
+            logger.info('using local documentation: %s', spec_docs_dir)
+            spec_docs = SpecificationDocs.from_dir(spec_docs_dir, opener=opener)
+        else:
+            logger.info('getting %r documentation from remote location', name)
+            spec_docs = SpecificationDocs.from_remote(opener=opener)
+
+        specifications_docs[name] = spec_docs
+
+    return specifications_docs
+
+
 def apis_by_specification(api_info, specifications):
     return groupby(api_info.items(),
                    key=lambda api_info: specifications[api_info[1].specification])
@@ -153,9 +175,13 @@ def main(args=None):
         opener = URLOpener()
         gen_info_factory = GenerationInfo.create
 
-    specifications = load_specifications(
-        [value[0] for value in global_config['API'].values()], opener=opener
-    )
+    spec_names = [value[0] for value in global_config['API'].values()]
+    specifications = load_specifications(spec_names, opener=opener)
+
+    if config.get('WITH_DOCS'):
+        documentations = load_documentations(spec_names, opener=opener)
+    else:
+        documentations = None
 
     generator = generators[ns.subparser_name](
         global_config['OUT_PATH'], opener=opener, gen_info_factory=gen_info_factory
@@ -189,8 +215,12 @@ def select(specification, api, info):
             logging_sink.info('merged into {}'.format(feature_sets[0]))
 
         for feature_set in feature_sets:
+            api = feature_set.info.apis[0]
+            spec_docs = None
+            if documentations and api in documentations:
+                spec_docs = documentations[api].select(feature_set)
             logging_sink.info('generating feature set {}'.format(feature_set))
-            generator.generate(specification, feature_set, config, sink=logging_sink)
+            generator.generate(specification, feature_set, config, sink=logging_sink, spec_docs=spec_docs)
 
 
 if __name__ == '__main__':
diff --git a/glad/documentation.py b/glad/documentation.py
index b9f15985..d6448950 100644
--- a/glad/documentation.py
+++ b/glad/documentation.py
@@ -1,49 +1,33 @@
-from glad.parse import SpecificationDocs, CommandDocs, xml_parse
-from glad.util import prefix, suffix, memoize, raw_text
-from shutil import rmtree
-import glad.util
-import subprocess
 import re
+import glad.util
+from glad.parse import DocumentationSet, SpecificationDocs, CommandDocs, xml_parse
+from glad.util import prefix, suffix, raw_text
 
 
 class DocsGL(SpecificationDocs):
-    URL = 'https://github.com/BSVino/docs.gl.git'
-    CACHED = True # Only clones the DocsGL repo once
-    API = 'gl'
-    docs = dict()
-
-    @property
-    @memoize(method=True)
-    def out_dir(self):
-        from pathlib import Path
-        from tempfile import gettempdir
-        if self.CACHED:
-            return Path('.cached') / 'docs.gl'
-        return Path(gettempdir()) / 'docs.gl'
-
-    def load(self):
-        if self.out_dir.exists() and not self.CACHED:
-            rmtree(str(self.out_dir))
-        if not self.out_dir.exists():
-            subprocess.run(['git', 'clone', '--depth=1', self.URL, str(self.out_dir)])
-
-        current_version = self.version.major
+    DOCS_NAME = 'docs.gl'
+
+    URL = 'https://github.com/BSVino/docs.gl/archive/refs/heads/mainline.zip'
+    SPEC = 'gl'
+
+    def select(self, feature_set):
+        current_major = list(feature_set.info)[0].version.major
+        commands = dict()
 
         # As the time of writing DocsGL offers documentation from gl4 to gl2.
         # If say we are targeting gl3, we will try to get the command documentation from gl3,
         # otherwise we'll try from gl2 and so on. If more than one version is available only the
         # most recent one will be used.
-        for version in range(current_version, 1, -1):
-            docs_dir = self.out_dir / f'{self.API}{version}'
-            if not docs_dir.exists():
+        for version in range(current_major, 1, -1):
+            version_dir = self.docs_dir / f'{self.SPEC}{version}'
+            if not version_dir.exists():
                 break
 
-            for html_file in docs_dir.glob('*.xhtml'):
-                for func, docs in DocsGL.docs_from_html_file(html_file).items():
-                    self.docs.setdefault(func, docs)
+            for html_file in version_dir.glob('*.xhtml'):
+                for command, docs in DocsGL.docs_from_html_file(html_file).items():
+                    commands.setdefault(command, docs)
 
-    def docs_for_command_name(self, name):
-        return self.docs.get(name, None)
+        return DocumentationSet(commands=commands)
 
     @classmethod
     def docs_from_html_file(cls, path):
diff --git a/glad/generator/__init__.py b/glad/generator/__init__.py
index 14b4fb4d..5479a4ee 100644
--- a/glad/generator/__init__.py
+++ b/glad/generator/__init__.py
@@ -57,7 +57,7 @@ def select(self, spec, api, version, profile, extensions, config, sink=LoggingSi
         :param sink: sink used to collect non fatal errors and information
         :return: FeatureSet with the required types, enums, commands/functions
         """
-        return spec.select(api, version, profile, extensions, config, sink=sink)
+        return spec.select(api, version, profile, extensions, sink=sink)
 
     def generate(self, spec, feature_set, config, sink=LoggingSink(__name__)):
         """
@@ -138,15 +138,16 @@ def modify_feature_set(self, spec, feature_set, config):
         """
         return feature_set
 
-    def get_template_arguments(self, spec, feature_set, config):
+    def get_template_arguments(self, spec, feature_set, config, spec_docs=None):
         return dict(
             spec=spec,
             feature_set=feature_set,
+            spec_docs=spec_docs,
             options=config.to_dict(transform=lambda x: x.lower()),
             gen_info=self.gen_info_factory(self, spec, feature_set, config)
         )
 
-    def generate(self, spec, feature_set, config, sink=LoggingSink(__name__)):
+    def generate(self, spec, feature_set, config, spec_docs=None, sink=LoggingSink(__name__)):
         feature_set = self.modify_feature_set(spec, feature_set, config)
         for template, output_path in self.get_templates(spec, feature_set, config):
             #try:
@@ -156,7 +157,7 @@ def generate(self, spec, feature_set, config, sink=LoggingSink(__name__)):
             #    raise ValueError('Unsupported specification/configuration')
 
             result = template.render(
-                **self.get_template_arguments(spec, feature_set, config)
+                **self.get_template_arguments(spec, feature_set, config, spec_docs=spec_docs)
             )
 
             output_path = os.path.join(self.path, output_path)
@@ -164,9 +165,9 @@ def generate(self, spec, feature_set, config, sink=LoggingSink(__name__)):
             with open(output_path, 'w') as f:
                 f.write(result)
 
-        self.post_generate(spec, feature_set, config)
+        self.post_generate(spec, feature_set, config, spec_docs=None)
 
-    def post_generate(self, spec, feature_set, config):
+    def post_generate(self, spec, feature_set, config, spec_docs=None):
         pass
 
 
diff --git a/glad/generator/c/__init__.py b/glad/generator/c/__init__.py
index acdf6e3c..c9cbe33b 100644
--- a/glad/generator/c/__init__.py
+++ b/glad/generator/c/__init__.py
@@ -383,8 +383,8 @@ def select(self, spec, api, version, profile, extensions, config, sink=LoggingSi
 
         return JinjaGenerator.select(self, spec, api, version, profile, extensions, config, sink=sink)
 
-    def get_template_arguments(self, spec, feature_set, config):
-        args = JinjaGenerator.get_template_arguments(self, spec, feature_set, config)
+    def get_template_arguments(self, spec, feature_set, config, spec_docs=None):
+        args = JinjaGenerator.get_template_arguments(self, spec, feature_set, config, spec_docs=spec_docs)
 
         # TODO allow MX for every specification/api
         if spec.name not in (VK.NAME, GL.NAME):
@@ -417,7 +417,7 @@ def get_templates(self, spec, feature_set, config):
 
         return templates
 
-    def post_generate(self, spec, feature_set, config):
+    def post_generate(self, spec, feature_set, config, spec_docs=None):
         self._add_additional_headers(feature_set, config)
 
     def modify_feature_set(self, spec, feature_set, config):
diff --git a/glad/generator/c/templates/template_utils.h b/glad/generator/c/templates/template_utils.h
index 7f06dcce..d8c3e461 100644
--- a/glad/generator/c/templates/template_utils.h
+++ b/glad/generator/c/templates/template_utils.h
@@ -117,25 +117,26 @@ GLAD_API_CALL {{ command.name|pfn }} glad_{{ command.name }};
 {% if debug %}
 GLAD_API_CALL {{ command.name|pfn }} glad_debug_{{ command.name }};
 {% endif %}
-{% if options.with_docs and command.doc_comment %}
+{% if options.with_docs and spec_docs %}
+{% set command_docs = spec_docs.docs_for_command_name(command.name) %}
 /**
-{% if command.doc_comment.brief %}
- * @brief {{ command.doc_comment.brief|wordwrap(80, wrapstring='\n * ') }}
+{% if command_docs.brief %}
+ * @brief {{ command_docs.brief|wordwrap(80, wrapstring='\n * ') }}
  *
 {% endif %}
-{% for param in command.doc_comment.params %}
+{% for param in command_docs.params %}
  * @param {{ param.name }} {{ param.desc|wordwrap(80, wrapstring='\n * ') }}
 {% endfor %}
  *
-{% for paragraph in command.doc_comment.description %}
- * {{ paragraph|replace(command.doc_comment.BREAK, '@details ')|wordwrap(80, wrapstring='\n * ') }}
+{% for paragraph in command_docs.description %}
+ * {{ paragraph|replace(command_docs.BREAK, '@details ')|wordwrap(80, wrapstring='\n * ') }}
 {% endfor %}
  *
-{% for note in command.doc_comment.notes %}
- * {{ note|replace(command.doc_comment.BREAK, '@note ')|wordwrap(80, wrapstring='\n * ') }}
+{% for note in command_docs.notes %}
+ * {{ note|replace(command_docs.BREAK, '@note ')|wordwrap(80, wrapstring='\n * ') }}
 {% endfor %}
-{% if command.doc_comment.see_also %}
- * @see {{ command.doc_comment.see_also }}
+{% if command_docs.see_also %}
+ * @see {{ command_docs.see_also }}
 {% endif %}
 */
 {% endif %}
diff --git a/glad/generator/rust/__init__.py b/glad/generator/rust/__init__.py
index a75efa47..1759841a 100644
--- a/glad/generator/rust/__init__.py
+++ b/glad/generator/rust/__init__.py
@@ -208,8 +208,8 @@ def select(self, spec, api, version, profile, extensions, config, sink=LoggingSi
 
         return JinjaGenerator.select(self, spec, api, version, profile, extensions, config, sink=sink)
 
-    def get_template_arguments(self, spec, feature_set, config):
-        args = JinjaGenerator.get_template_arguments(self, spec, feature_set, config)
+    def get_template_arguments(self, spec, feature_set, config, spec_docs=None):
+        args = JinjaGenerator.get_template_arguments(self, spec, feature_set, config, spec_docs=spec_docs)
 
         args.update(
             version=glad.__version__,
diff --git a/glad/parse.py b/glad/parse.py
index 698af524..936bf902 100644
--- a/glad/parse.py
+++ b/glad/parse.py
@@ -24,6 +24,9 @@ def xml_parse(path):
 import logging
 import os.path
 import warnings
+import zipfile
+from io import BytesIO
+from pathlib import Path
 from collections import defaultdict, OrderedDict, namedtuple, deque
 from contextlib import closing
 from itertools import chain
@@ -210,7 +213,6 @@ class Specification(object):
     def __init__(self, root):
         self.root = root
         self._combined = None
-        self._docs = None
 
     def _magic_require(self, api, profile):
         """
@@ -405,8 +407,6 @@ def commands(self):
 
         # populate docs and fixup aliases
         for command in chain.from_iterable(commands.values()):
-            if self._docs:
-                command.doc_comment = self._docs.docs_for_command_name(command.name)
             if command.alias is not None and command.proto is None:
                 aliased_command = command
                 while aliased_command.proto is None:
@@ -1459,24 +1459,60 @@ def __str__(self):
 
 
 class SpecificationDocs(object):
+    DOCS_NAME = None
+
     SPEC = None
     URL = None
 
-    def __init__(self, api, version, profile, extensions):
-        self.api = api
-        self.version = version
-        self.profile = profile
-        self.extensions = extensions
-        self.docs = dict()
+    def __init__(self, docs_dir):
+        self.docs_dir = docs_dir
 
-    def load(self):
+    def select(self, feature_set):
         raise NotImplementedError
 
+    @classmethod
+    def default_out_dir(cls):
+        if cls.DOCS_NAME is None:
+            raise ValueError('DOCS_NAME not set')
+        return Path('.docs') / cls.DOCS_NAME
+
+    @classmethod
+    def from_url(cls, url, opener=None):
+        if opener is None:
+            opener = URLOpener.default()
+        docs_dir = cls.default_out_dir()
+        zip_out_dir = docs_dir.parent
+
+        with closing(opener.urlopen(url)) as f:
+            raw = f.read()
+            f = BytesIO(raw)
+
+            with zipfile.ZipFile(f) as zf:
+                zf.extractall(zip_out_dir)
+                zip_dir = zip_out_dir / zf.namelist()[0]
+                os.rename(zip_dir, docs_dir)
+
+        return cls(docs_dir)
+
+    @classmethod
+    def from_remote(cls, opener=None):
+        return cls.from_url(cls.URL, opener=opener)
+
+    @classmethod
+    def from_dir(cls, dir_path, opener=None):
+        return cls(dir_path)
+
+
+class DocumentationSet(object):
+    def __init__(self, commands):
+        self.commands = commands
+
+    def __str__(self):
+        return 'DocumentationSet(commands={})'.format(len(self.commands))
+    __repr__ = __str__
+
     def docs_for_command_name(self, name):
-        """
-        Returns the CommandDocs for the given command name or None if not found.
-        """
-        raise NotImplementedError
+        return self.commands.get(name, None)
 
 
 class CommandDocs(object):
diff --git a/glad/plugin.py b/glad/plugin.py
index ec5afe1e..6e3ddb87 100644
--- a/glad/plugin.py
+++ b/glad/plugin.py
@@ -13,9 +13,10 @@ def entry_points(group=None):
     from pkg_resources import iter_entry_points as entry_points
 
 import glad.specification
+import glad.documentation
 from glad.generator.c import CGenerator
 from glad.generator.rust import RustGenerator
-from glad.parse import Specification
+from glad.parse import Specification, SpecificationDocs
 
 
 logger = logging.getLogger(__name__)
@@ -23,6 +24,7 @@ def entry_points(group=None):
 
 GENERATOR_ENTRY_POINT = 'glad.generator'
 SPECIFICATION_ENTRY_POINT = 'glad.specification'
+DOCUMENTATION_ENTRY_POINT = 'glad.documentation'
 
 
 DEFAULT_GENERATORS = dict(
@@ -30,11 +32,15 @@ def entry_points(group=None):
     rust=RustGenerator
 )
 DEFAULT_SPECIFICATIONS = dict()
+DEFAULT_SPECIFICATION_DOCS = dict()
 
 for name, cls in inspect.getmembers(glad.specification, inspect.isclass):
     if issubclass(cls, Specification) and cls is not Specification:
         DEFAULT_SPECIFICATIONS[cls.NAME] = cls
 
+for name, cls in inspect.getmembers(glad.documentation, inspect.isclass):
+    if issubclass(cls, SpecificationDocs) and cls is not SpecificationDocs:
+        DEFAULT_SPECIFICATION_DOCS[cls.SPEC] = cls
 
 def find_generators(default=None, entry_point=GENERATOR_ENTRY_POINT):
     generators = dict(DEFAULT_GENERATORS if default is None else default)
@@ -54,3 +60,12 @@ def find_specifications(default=None, entry_point=SPECIFICATION_ENTRY_POINT):
         logger.debug('loaded specification %s: %s', entry_point.name, specifications[entry_point.name])
 
     return specifications
+
+def find_specification_docs(default=None, entry_point=DOCUMENTATION_ENTRY_POINT):
+    documentations = dict(DEFAULT_SPECIFICATION_DOCS if default is None else default)
+
+    for entry_point in entry_points(group=entry_point):
+        documentations[entry_point.name] = entry_point.load()
+        logger.debug('loaded documentation %s: %s', entry_point.name, documentations[entry_point.name])
+
+    return documentations

From 0c956e22799c815b7d957d54751927b64758500e Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Sat, 25 May 2024 19:51:47 -0500
Subject: [PATCH 22/33] change docs out dir from .cached to .docs

---
 .gitignore | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index a315df8c..0efa3758 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,5 +19,5 @@ dist/
 target/
 Cargo.lock
 .vscode/
-.cached/
+.docs/
 .venv/

From 29a8ee757daabaa9edc1c9476f9d6f2eb4e1e1b0 Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Sat, 25 May 2024 19:57:01 -0500
Subject: [PATCH 23/33] make --with-docs a global option since it's language
 agnostic

---
 glad/__main__.py             | 7 ++++++-
 glad/generator/c/__init__.py | 5 -----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/glad/__main__.py b/glad/__main__.py
index 1a1b05f9..cfea12d4 100644
--- a/glad/__main__.py
+++ b/glad/__main__.py
@@ -71,6 +71,11 @@ class GlobalConfig(Config):
         description='Makes the build reproducible by not fetching the latest '
                     'specification from Khronos.'
     )
+    WITH_DOCS = ConfigOption(
+        converter=bool,
+        default=False,
+        description='Include inline documentation in the generated files.'
+    )
 
 
 def load_specifications(specification_names, opener, specification_classes=None):
@@ -178,7 +183,7 @@ def main(args=None):
     spec_names = [value[0] for value in global_config['API'].values()]
     specifications = load_specifications(spec_names, opener=opener)
 
-    if config.get('WITH_DOCS'):
+    if global_config['WITH_DOCS']:
         documentations = load_documentations(spec_names, opener=opener)
     else:
         documentations = None
diff --git a/glad/generator/c/__init__.py b/glad/generator/c/__init__.py
index c9cbe33b..8ca2e9ce 100644
--- a/glad/generator/c/__init__.py
+++ b/glad/generator/c/__init__.py
@@ -244,11 +244,6 @@ class CConfig(Config):
         default=False,
         description='On-demand function pointer loading, initialize on use (experimental)'
     )
-    WITH_DOCS = ConfigOption(
-        converter=bool,
-        default=False,
-        description='Include inline documentation in the generated files.'
-    )
 
     __constraints__ = [
         # RequirementConstraint(['MX_GLOBAL'], 'MX'),

From 9aa3dbdbb3b74a3867646415e062c6cde4eaed12 Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Sat, 25 May 2024 19:58:43 -0500
Subject: [PATCH 24/33] fix: log warning when no documentation is found for the
 spec

---
 glad/__main__.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/glad/__main__.py b/glad/__main__.py
index cfea12d4..58f61f64 100644
--- a/glad/__main__.py
+++ b/glad/__main__.py
@@ -107,6 +107,10 @@ def load_documentations(specification_names, opener, spec_docs_classes=None):
         spec_docs_classes = find_specification_docs()
 
     for name in set(specification_names):
+        if name not in spec_docs_classes:
+            logger.warning('no documentation available for %r', name)
+            continue
+
         SpecificationDocs = spec_docs_classes[name]
         spec_docs_dir = SpecificationDocs.default_out_dir()
 

From b7289727ef10aa44687511b4f9a21d6de41be719 Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Sat, 25 May 2024 20:39:52 -0500
Subject: [PATCH 25/33] fix docs rendering

---
 glad/generator/c/templates/template_utils.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/glad/generator/c/templates/template_utils.h b/glad/generator/c/templates/template_utils.h
index d8c3e461..07be1ac6 100644
--- a/glad/generator/c/templates/template_utils.h
+++ b/glad/generator/c/templates/template_utils.h
@@ -117,7 +117,7 @@ GLAD_API_CALL {{ command.name|pfn }} glad_{{ command.name }};
 {% if debug %}
 GLAD_API_CALL {{ command.name|pfn }} glad_debug_{{ command.name }};
 {% endif %}
-{% if options.with_docs and spec_docs %}
+{% if spec_docs %}
 {% set command_docs = spec_docs.docs_for_command_name(command.name) %}
 /**
 {% if command_docs.brief %}

From 5b73a9fc6de03b964f7d204fd68e335f7be67927 Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Sun, 26 May 2024 03:36:55 -0500
Subject: [PATCH 26/33] prefer lxml instead of xml package

I needed the 'recover' kwarg, since some xml files were broken.

See <https://lxml.de/api/lxml.etree.XMLParser-class.html>.
---
 glad/parse.py    | 25 +++++++------------------
 requirements.txt |  1 +
 2 files changed, 8 insertions(+), 18 deletions(-)

diff --git a/glad/parse.py b/glad/parse.py
index 936bf902..e7f3e998 100644
--- a/glad/parse.py
+++ b/glad/parse.py
@@ -1,23 +1,12 @@
 from glad.sink import LoggingSink
 
-try:
-    from lxml import etree
-    from lxml.etree import ETCompatXMLParser as parser
-
-    def xml_fromstring(argument):
-        return etree.fromstring(argument, parser=parser())
-    def xml_parse(path):
-        return etree.parse(path, parser=parser()).getroot()
-except ImportError:
-    try:
-        import xml.etree.cElementTree as etree
-    except ImportError:
-        import xml.etree.ElementTree as etree
-
-    def xml_fromstring(argument):
-        return etree.fromstring(argument)
-    def xml_parse(path):
-        return etree.parse(path).getroot()
+from lxml import etree
+from lxml.etree import ETCompatXMLParser as parser
+
+def xml_fromstring(argument):
+    return etree.fromstring(argument, parser=parser())
+def xml_parse(path, recover=False):
+    return etree.parse(path, parser=parser(recover=recover)).getroot()
 
 import re
 import copy
diff --git a/requirements.txt b/requirements.txt
index c9ae643a..d27a32af 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1 +1,2 @@
 Jinja2>=2.7,<4.0
+lxml==5.2.2

From 13e26fdd13cdb83db99b5f364a3290423d6ff685 Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Sun, 26 May 2024 03:38:47 -0500
Subject: [PATCH 27/33] replace DocsGL with Khronos refpages

---
 glad/documentation.py | 140 +++++++++++++++++++++++-------------------
 1 file changed, 77 insertions(+), 63 deletions(-)

diff --git a/glad/documentation.py b/glad/documentation.py
index d6448950..cce3f759 100644
--- a/glad/documentation.py
+++ b/glad/documentation.py
@@ -1,30 +1,31 @@
 import re
 import glad.util
+from lxml import etree
 from glad.parse import DocumentationSet, SpecificationDocs, CommandDocs, xml_parse
 from glad.util import prefix, suffix, raw_text
 
+class OpenGLRefpages(SpecificationDocs):
+    DOCS_NAME = 'opengl_refpages'
 
-class DocsGL(SpecificationDocs):
-    DOCS_NAME = 'docs.gl'
-
-    URL = 'https://github.com/BSVino/docs.gl/archive/refs/heads/mainline.zip'
+    URL = 'https://github.com/KhronosGroup/OpenGL-Refpages/archive/refs/heads/main.zip'
     SPEC = 'gl'
 
     def select(self, feature_set):
         current_major = list(feature_set.info)[0].version.major
         commands = dict()
 
-        # As the time of writing DocsGL offers documentation from gl4 to gl2.
-        # If say we are targeting gl3, we will try to get the command documentation from gl3,
-        # otherwise we'll try from gl2 and so on. If more than one version is available only the
-        # most recent one will be used.
-        for version in range(current_major, 1, -1):
-            version_dir = self.docs_dir / f'{self.SPEC}{version}'
+        # At the time of writing Khronos hosts documentations for gl4 and gl2.1.
+        available_versions = ['gl2.1']
+        if current_major >= 4:
+            available_versions.append('gl4')
+
+        for version_dir in available_versions:
+            version_dir = self.docs_dir / f'{version_dir}'
             if not version_dir.exists():
                 break
 
-            for html_file in version_dir.glob('*.xhtml'):
-                for command, docs in DocsGL.docs_from_html_file(html_file).items():
+            for html_file in version_dir.glob('*.xml'):
+                for command, docs in OpenGLRefpages.docs_from_html_file(html_file).items():
                     commands.setdefault(command, docs)
 
         return DocumentationSet(commands=commands)
@@ -32,29 +33,37 @@ def select(self, feature_set):
     @classmethod
     def docs_from_html_file(cls, path):
         commands_parsed = dict()
+        version = path.parent.name
+        tree = xml_parse(path, recover=True)
 
-        tree = xml_parse(path)
-        sections = tree.findall('.//*[@class="refsect1"]')
-
-        # Some pages are just a redirect script
-        if tree.tag == 'script':
+        # gl4 files contain a namespace that polutes the tags, so we clean it up.
+        for elem in tree.getiterator():
             try:
-                redirect = tree.text.split('window.location.replace("')[1].split('")')[0]
-                path = path.parent / f'{redirect}.xhtml'
-                tree = xml_parse(path)
+                if elem.tag.startswith('{'):
+                    elem.tag = etree.QName(elem).localname
             except:
-                return dict()
+                pass
+        etree.cleanup_namespaces(tree)
+
+        sections = tree.findall('.//refsect1')
 
         # Brief parsing
         # Command brief description appears in the first 'refnamediv' block
-        brief_block = cls.xml_text(tree.find('.//div[@class="refnamediv"]/p'))
-        brief = f'[{path.stem}](https://docs.gl/{path.parent.name}/{path.stem}) — ' \
-            f'{suffix(".", brief_block.split("—")[1])}'
+        brief_block = tree.find('.//refnamediv//refpurpose')
+
+        if brief_block is None:
+            return dict()
+
+        if version == 'gl2.1':
+            url = f'https://registry.khronos.org/OpenGL-Refpages/{version}/xhtml/{path.stem}.xml'
+        else:
+            url = f'https://registry.khronos.org/OpenGL-Refpages/{version}/html/{path.stem}.xhtml'
+        brief = f'[{path.stem}]({url}) — {suffix(".", cls.xml_text(brief_block))}'
 
         # Description parsing
         description = []
         description_blocks = next(
-            (s for s in sections if raw_text(s.find('h2')) == 'Description'),
+            (s for s in sections if raw_text(s.find('title')) == 'Description'),
             None,
         )
         if description_blocks is not None:
@@ -68,7 +77,7 @@ def docs_from_html_file(cls, path):
 
         # Notes parsing
         notes = []
-        notes_blocks = next((s for s in sections if raw_text(s.find('h2')) == 'Notes'), None)
+        notes_blocks = next((s for s in sections if raw_text(s.find('title')) == 'Notes'), None)
         if notes_blocks is not None:
             blocks = notes_blocks.findall('./*')
             notes = list(
@@ -79,43 +88,49 @@ def docs_from_html_file(cls, path):
             )
 
         # Parameters parsing
-        # DocsGL puts all the function definitions inside .funcsynopsis.funcdef blocks.
+        # Khronos specs puts all the function definitions inside funcsynopsis/funcdef blocks.
         #
-        # However, instead of describing each function on a separate file, DocsGL combines
-        # multiple related function definitions, whose parameters may be different, into a single
-        # file. This means that we have to find the correct block of parameters for each definition.
+        # However, instead of describing each function on a separate file, they group multiple
+        # related function definitions, whose parameters may be different, into a single file.
+        # This means that we have to find the correct block of parameters for each definition.
         funcdefs = [
-            d for d in tree.findall('.//*[@class="funcsynopsis"]/*')
-            if d.find('.//*[@class="funcdef"]') is not None
+            d for d in tree.findall('.//funcsynopsis/*')
+            if d.find('.//funcdef') is not None
         ]
+
         for func_def in funcdefs:
-            func_name = func_def.find('.//*[@class="fsfunc"]').text
-            func_params = [raw_text(s) for s in func_def.findall('.//var[@class="pdparam"]')]
+            func_name = func_def.find('.//function').text
+            func_params = [raw_text(s) for s in func_def.findall('.//parameter')]
 
-            # Params are defined in a separate section, often called 'Parameters for <func_name>'
+            # Params are defined in a separate section, called 'Parameters for <func_name>'
             # or just 'Parameters'.
             params_block = next(
-                (s for s in sections if raw_text(s.find('h2')) == f'Parameters for {func_name}'),
+                (s for s in sections if raw_text(s.find('title')) == f'Parameters for {func_name}'),
                 None,
             )
-            if not params_block is not None:
-                for p in list(s for s in sections if raw_text(s.find('h2')) == 'Parameters'):
-                    block_params = [raw_text(n) for n in p.findall('.//dt//code')]
-                    if all(p in block_params for p in func_params):
+            if params_block is None:
+                for p in list(s for s in sections if raw_text(s.find('title')) == 'Parameters'):
+                    block_params = [raw_text(n) for n in p.findall('.//term//parameter')]
+                    if all(func_param in block_params for func_param in func_params):
                         params_block = p
                         break
 
+            # At this point we interpret params_block=None as a void parameter list.
+
             params = []
-            if params_block is not None:
-                for names, desc in zip(
-                    params_block.findall('.//dl//dt'),
-                    params_block.findall('.//dl//dd'),
-                ):
-                    for name in names.findall('.//code'):
-                        param_name = raw_text(name)
+            # A description can apply for more than one param (term), so we stack them until
+            # we find a listitem, which is a description of a param.
+            terms_stack = []
+            for param_or_desc in params_block.findall('.//varlistentry/*') if params_block is not None else []:
+                if param_or_desc.tag == 'term':
+                    terms_stack.append(param_or_desc)
+                    continue
+                if param_or_desc.tag == 'listitem':
+                    for term in terms_stack:
+                        param_name = raw_text(term.find('.//parameter'))
                         if param_name in func_params:
-                            params.append(CommandDocs.Param(param_name, cls.xml_text(desc)))
-            # We interpret params_block=None as a void parameter list.
+                            params.append(CommandDocs.Param(param_name, cls.xml_text(param_or_desc)))
+                    terms_stack.clear()
 
             commands_parsed[func_name] = CommandDocs(
                 brief, params, description, notes, None, None,
@@ -146,8 +161,8 @@ def format(e, is_tail=False):
             return f'\n{CommandDocs.BREAK}-{e.text}'
         return re.sub(r'\n+', '', e.text)
 
-    @staticmethod
-    def xml_text(e):
+    @classmethod
+    def xml_text(cls, e):
         def paren(expr):
             if re.match(r'^[a-zA-Z0-9_]+$', expr):
                 return expr
@@ -155,22 +170,21 @@ def paren(expr):
 
         def mfenced(e):
             if e.attrib['close']:
-                return f'{e.attrib["open"]}{", ".join(DocsGL.xml_text(c) for c in e)}{e.attrib["close"]}'
-            return f'{e.attrib["open"]}{" ".join(DocsGL.xml_text(c) for c in e)}'
+                return f'{e.attrib["open"]}{", ".join(cls.xml_text(c) for c in e)}{e.attrib["close"]}'
+            return f'{e.attrib["open"]}{" ".join(cls.xml_text(c) for c in e)}'
 
         text = ''.join(glad.util.itertext(
             e,
             convert={
                 'table': lambda _: f'(table omitted)',
-                'pre': lambda _: f'(code omitted)',
-                'mfrac': lambda e, : f'{paren(DocsGL.xml_text(e[0]))}/{paren(DocsGL.xml_text(e[1]))}',
-                'msup': lambda e: f'{paren(DocsGL.xml_text(e[0]))}^{paren(DocsGL.xml_text(e[1]))}',
-                'msub': lambda e: f'{paren(DocsGL.xml_text(e[0]))}_{paren(DocsGL.xml_text(e[1]))}',
-                'mtd': lambda e: f'{DocsGL.xml_text(e[0])}; ',
-                'mfenced': mfenced,
+                'informaltable': lambda _: f'(table omitted)',
+                'programlisting': lambda _: f'(code omitted)',
+                'mml:mfrac': lambda e, : f'{paren(cls.xml_text(e[0]))}/{paren(cls.xml_text(e[1]))}', #
+                'mml:msup': lambda e: f'{paren(cls.xml_text(e[0]))}^{paren(cls.xml_text(e[1]))}', #
+                'mml:msub': lambda e: f'{paren(cls.xml_text(e[0]))}_{paren(cls.xml_text(e[1]))}', #
+                'mml:mtd': lambda e: f'{cls.xml_text(e[0])}; ', #
+                'mml:mfenced': mfenced, #
             },
-            format=DocsGL.format,
+            format=cls.format,
         ))
-        # \u00a0, \u2062, \u2062,
-        # are invisible characters used by docs.gl to separate words.
-        return re.sub(r'\n?[ \u00a0\u2062\u2061]+', ' ', text.strip())
+        return re.sub(r'\n? +', ' ', text.strip())

From 57c5ead4240f957a45642be9178b250bd30db7db Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Sun, 2 Jun 2024 10:34:28 -0500
Subject: [PATCH 28/33] fix parameters parsing and math namespacing

---
 glad/documentation.py                       | 89 +++++++++++++--------
 glad/generator/c/templates/template_utils.h | 18 +++--
 glad/parse.py                               | 11 ++-
 glad/util.py                                |  6 +-
 4 files changed, 81 insertions(+), 43 deletions(-)

diff --git a/glad/documentation.py b/glad/documentation.py
index cce3f759..665315dc 100644
--- a/glad/documentation.py
+++ b/glad/documentation.py
@@ -11,13 +11,13 @@ class OpenGLRefpages(SpecificationDocs):
     SPEC = 'gl'
 
     def select(self, feature_set):
-        current_major = list(feature_set.info)[0].version.major
+        current_major = max(info.version.major for info in feature_set.info)
         commands = dict()
 
         # At the time of writing Khronos hosts documentations for gl4 and gl2.1.
         available_versions = ['gl2.1']
         if current_major >= 4:
-            available_versions.append('gl4')
+            available_versions.insert(0, 'gl4')
 
         for version_dir in available_versions:
             version_dir = self.docs_dir / f'{version_dir}'
@@ -41,6 +41,11 @@ def docs_from_html_file(cls, path):
             try:
                 if elem.tag.startswith('{'):
                     elem.tag = etree.QName(elem).localname
+                if elem.tag.contains(':'):
+                    elem.tag = elem.tag.split(':')[-1]
+                for key in elem.attrib:
+                    if key.startswith('{'):
+                        elem.attrib[etree.QName(key).localname] = elem.attrib.pop(key)
             except:
                 pass
         etree.cleanup_namespaces(tree)
@@ -52,13 +57,14 @@ def docs_from_html_file(cls, path):
         brief_block = tree.find('.//refnamediv//refpurpose')
 
         if brief_block is None:
+            # No brief means file doesn't contain any command definitions.
             return dict()
+        brief = suffix(".", cls.xml_text(brief_block))
 
         if version == 'gl2.1':
-            url = f'https://registry.khronos.org/OpenGL-Refpages/{version}/xhtml/{path.stem}.xml'
+            docs_url = f'https://registry.khronos.org/OpenGL-Refpages/{version}/xhtml/{path.stem}.xml'
         else:
-            url = f'https://registry.khronos.org/OpenGL-Refpages/{version}/html/{path.stem}.xhtml'
-        brief = f'[{path.stem}]({url}) — {suffix(".", cls.xml_text(brief_block))}'
+            docs_url = f'https://registry.khronos.org/OpenGL-Refpages/{version}/html/{path.stem}.xhtml'
 
         # Description parsing
         description = []
@@ -71,19 +77,20 @@ def docs_from_html_file(cls, path):
             description = list(
                 filter(
                     bool,
-                    (prefix(CommandDocs.BREAK, cls.xml_text(p)) for p in blocks if p.tag != 'h2'),
+                    (cls.xml_text(p) for p in blocks if p.tag != 'title'),
                 ),
             )
 
         # Notes parsing
         notes = []
         notes_blocks = next((s for s in sections if raw_text(s.find('title')) == 'Notes'), None)
+
         if notes_blocks is not None:
             blocks = notes_blocks.findall('./*')
             notes = list(
                 filter(
                     bool,
-                    (prefix(CommandDocs.BREAK, cls.xml_text(p)) for p in blocks if p.tag != 'h2'),
+                    (cls.xml_text(p) for p in blocks if p.tag != 'title'),
                 ),
             )
 
@@ -102,6 +109,7 @@ def docs_from_html_file(cls, path):
             func_name = func_def.find('.//function').text
             func_params = [raw_text(s) for s in func_def.findall('.//parameter')]
 
+
             # Params are defined in a separate section, called 'Parameters for <func_name>'
             # or just 'Parameters'.
             params_block = next(
@@ -116,49 +124,62 @@ def docs_from_html_file(cls, path):
                         break
 
             # At this point we interpret params_block=None as a void parameter list.
+            is_void = params_block is None
+            params_entries = params_block.findall('.//varlistentry/*') if not is_void else []
 
             params = []
             # A description can apply for more than one param (term), so we stack them until
             # we find a listitem, which is a description of a param.
             terms_stack = []
-            for param_or_desc in params_block.findall('.//varlistentry/*') if params_block is not None else []:
+            for param_or_desc in params_entries:
                 if param_or_desc.tag == 'term':
                     terms_stack.append(param_or_desc)
                     continue
                 if param_or_desc.tag == 'listitem':
-                    for term in terms_stack:
-                        param_name = raw_text(term.find('.//parameter'))
-                        if param_name in func_params:
-                            params.append(CommandDocs.Param(param_name, cls.xml_text(param_or_desc)))
+                    for terms in terms_stack:
+                        param_names = [
+                            p.text for p in terms.findall('.//parameter') if p.text in func_params
+                        ]
+
+                        for param_name in param_names:
+                            params.append(CommandDocs.Param(
+                                param_name,
+                                cls.xml_text(param_or_desc).replace(CommandDocs.BREAK, ''),
+                            ))
                     terms_stack.clear()
 
             commands_parsed[func_name] = CommandDocs(
-                brief, params, description, notes, None, None,
+                func_name, brief, params, description, notes, None, None, docs_url,
             )
         return commands_parsed
 
-    @staticmethod
-    def format(e, is_tail=False):
+    @classmethod
+    def format(cls, e, is_tail=False):
         if is_tail:
-            if e.tag == 'dt':
-                # closing a definition term
-                return '\n'
+            # closing a definition term
+            if e.tag == 'term':
+                return ''
+            # closing a mathjax row
             if e.tag == 'mtr':
-                # closing a mathjax row
                 return '\n'
-            r = re.sub(r'\n+', '', e.tail)
             if e.tag in ('mn', 'msub'):
                 return ''
             return re.sub(r'\n+', '', e.tail)
 
-        if e.tag == 'a':
-            return f'![{e.text}]({e.attrib["href"]})'
-        if e.tag == 'code':
+        if e.tag == 'link':
+            if e.attrib.get('href'):
+                return f'[{e.text}]({e.attrib["href"]})'
+            return e.text
+        if e.tag == 'constant':
+            return f'`{e.text}`'
+        if e.tag == 'function':
             return f'`{e.text}`'
-        if e.tag == 'dt':
-            return f'\n{CommandDocs.BREAK}- '
-        if e.tag == 'li':
-            return f'\n{CommandDocs.BREAK}-{e.text}'
+        if e.tag == 'term':
+            return f'\n{CommandDocs.BREAK}- {e.text.strip()}'
+        if e.tag == 'listitem':
+            if e.getparent().tag == 'varlistentry':
+                return f'\n{CommandDocs.BREAK}{e.text}'
+            return f'\n{CommandDocs.BREAK}- {e.text.strip()}'
         return re.sub(r'\n+', '', e.text)
 
     @classmethod
@@ -179,12 +200,14 @@ def mfenced(e):
                 'table': lambda _: f'(table omitted)',
                 'informaltable': lambda _: f'(table omitted)',
                 'programlisting': lambda _: f'(code omitted)',
-                'mml:mfrac': lambda e, : f'{paren(cls.xml_text(e[0]))}/{paren(cls.xml_text(e[1]))}', #
-                'mml:msup': lambda e: f'{paren(cls.xml_text(e[0]))}^{paren(cls.xml_text(e[1]))}', #
-                'mml:msub': lambda e: f'{paren(cls.xml_text(e[0]))}_{paren(cls.xml_text(e[1]))}', #
-                'mml:mtd': lambda e: f'{cls.xml_text(e[0])}; ', #
-                'mml:mfenced': mfenced, #
+                'mfrac': lambda e, : f'{paren(cls.xml_text(e[0]))}/{paren(cls.xml_text(e[1]))}',
+                'msup': lambda e: f'{paren(cls.xml_text(e[0]))}^{paren(cls.xml_text(e[1]))}',
+                'msub': lambda e: f'{paren(cls.xml_text(e[0]))}_{paren(cls.xml_text(e[1]))}',
+                'mtd': lambda e: f'{cls.xml_text(e[0])}; ',
+                'mfenced': mfenced,
             },
             format=cls.format,
         ))
-        return re.sub(r'\n? +', ' ', text.strip())
+        # \u2062, \u2062,
+        # Invisible characters used by docs.gl to separate words.
+        return re.sub(r'\n?[ \u2062\u2061]+', ' ', text.strip())
diff --git a/glad/generator/c/templates/template_utils.h b/glad/generator/c/templates/template_utils.h
index 07be1ac6..10c5eb7b 100644
--- a/glad/generator/c/templates/template_utils.h
+++ b/glad/generator/c/templates/template_utils.h
@@ -119,27 +119,35 @@ GLAD_API_CALL {{ command.name|pfn }} glad_debug_{{ command.name }};
 {% endif %}
 {% if spec_docs %}
 {% set command_docs = spec_docs.docs_for_command_name(command.name) %}
+{% if command_docs %}
 /**
 {% if command_docs.brief %}
- * @brief {{ command_docs.brief|wordwrap(80, wrapstring='\n * ') }}
+ * @brief [{{ command_docs.name }}]({{ command_docs.docs_url }})
+ * — {{ command_docs.brief|wordwrap(80, wrapstring='\n * ') }}
  *
 {% endif %}
+{% if command_docs.params|length > 0 %}
 {% for param in command_docs.params %}
- * @param {{ param.name }} {{ param.desc|wordwrap(80, wrapstring='\n * ') }}
+ * @param {{ param.name }}{{ param.desc|wordwrap(80, wrapstring='\n * ') }}
 {% endfor %}
  *
+{% endif %}
 {% for paragraph in command_docs.description %}
- * {{ paragraph|replace(command_docs.BREAK, '@details ')|wordwrap(80, wrapstring='\n * ') }}
+ * @details {{ paragraph|replace(command_docs.BREAK, '@details ')|wordwrap(80, wrapstring='\n * ') }}
 {% endfor %}
+{% if command_docs.notes|length > 0 %}
  *
 {% for note in command_docs.notes %}
- * {{ note|replace(command_docs.BREAK, '@note ')|wordwrap(80, wrapstring='\n * ') }}
+ * @note {{ note|replace(command_docs.BREAK, '@note ')|wordwrap(80, wrapstring='\n * ') }}
 {% endfor %}
+{% endif %}
 {% if command_docs.see_also %}
- * @see {{ command_docs.see_also }}
+ *
+ * @see {{ command_docs.see_also|wordwrap(80, wrapstring='\n * ') }}
 {% endif %}
 */
 {% endif %}
+{% endif %}
 {% if debug %}
 #define {{ command.name }} glad_debug_{{ command.name }}
 {% else %}
diff --git a/glad/parse.py b/glad/parse.py
index e7f3e998..995d8579 100644
--- a/glad/parse.py
+++ b/glad/parse.py
@@ -5,8 +5,11 @@
 
 def xml_fromstring(argument):
     return etree.fromstring(argument, parser=parser())
-def xml_parse(path, recover=False):
-    return etree.parse(path, parser=parser(recover=recover)).getroot()
+def xml_parse(path, recover=False, xinclude=True):
+    tree = etree.parse(path, parser=parser(recover=recover))
+    if xinclude:
+        tree.xinclude()
+    return tree.getroot()
 
 import re
 import copy
@@ -1516,13 +1519,15 @@ class CommandDocs(object):
     class Param(namedtuple('Param', ['name', 'desc'])):
         pass
 
-    def __init__(self, brief, params, description, notes, errors, see_also):
+    def __init__(self, name, brief, params, description, notes, errors, see_also, docs_url):
+        self.name = name
         self.brief = brief
         self.params = params
         self.description = description
         self.notes = notes
         self.errors = errors
         self.see_also = see_also
+        self.docs_url = docs_url
 
     def __str__(self):
         return 'CommandDocs(brief={!r}, ' \
diff --git a/glad/util.py b/glad/util.py
index 25f8ac5e..28577a55 100644
--- a/glad/util.py
+++ b/glad/util.py
@@ -191,8 +191,10 @@ def itertext(element, ignore=(), convert=dict(), format=_format_none):
 
     if not isinstance(tag, basestring) and tag is not None:
         return
-    if element.text:
-        yield format(element)
+    if element.text is None:
+        element.text = ''
+    yield format(element)
+
     for e in element:
         for s in itertext(e, ignore=ignore, convert=convert, format=format):
             yield s

From cbdb5a69f380889ba6c28814e63faa3daeb6692b Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Sun, 2 Jun 2024 10:35:01 -0500
Subject: [PATCH 29/33] rename --with-docs to --docs

---
 glad/__main__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/glad/__main__.py b/glad/__main__.py
index 58f61f64..10b947e6 100644
--- a/glad/__main__.py
+++ b/glad/__main__.py
@@ -71,7 +71,7 @@ class GlobalConfig(Config):
         description='Makes the build reproducible by not fetching the latest '
                     'specification from Khronos.'
     )
-    WITH_DOCS = ConfigOption(
+    DOCS = ConfigOption(
         converter=bool,
         default=False,
         description='Include inline documentation in the generated files.'
@@ -187,7 +187,7 @@ def main(args=None):
     spec_names = [value[0] for value in global_config['API'].values()]
     specifications = load_specifications(spec_names, opener=opener)
 
-    if global_config['WITH_DOCS']:
+    if global_config['DOCS']:
         documentations = load_documentations(spec_names, opener=opener)
     else:
         documentations = None

From 70740de659a131871ba890eb02af12fc77d1e4a5 Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Sun, 2 Jun 2024 14:11:13 -0500
Subject: [PATCH 30/33] don't xinclude by default when parsing xml from file

---
 glad/documentation.py | 4 ++--
 glad/parse.py         | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/glad/documentation.py b/glad/documentation.py
index 665315dc..386dd229 100644
--- a/glad/documentation.py
+++ b/glad/documentation.py
@@ -2,7 +2,7 @@
 import glad.util
 from lxml import etree
 from glad.parse import DocumentationSet, SpecificationDocs, CommandDocs, xml_parse
-from glad.util import prefix, suffix, raw_text
+from glad.util import suffix, raw_text
 
 class OpenGLRefpages(SpecificationDocs):
     DOCS_NAME = 'opengl_refpages'
@@ -34,7 +34,7 @@ def select(self, feature_set):
     def docs_from_html_file(cls, path):
         commands_parsed = dict()
         version = path.parent.name
-        tree = xml_parse(path, recover=True)
+        tree = xml_parse(path, recover=True, xinclude=True)
 
         # gl4 files contain a namespace that polutes the tags, so we clean it up.
         for elem in tree.getiterator():
diff --git a/glad/parse.py b/glad/parse.py
index 995d8579..02077f57 100644
--- a/glad/parse.py
+++ b/glad/parse.py
@@ -3,9 +3,9 @@
 from lxml import etree
 from lxml.etree import ETCompatXMLParser as parser
 
-def xml_fromstring(argument):
-    return etree.fromstring(argument, parser=parser())
-def xml_parse(path, recover=False, xinclude=True):
+def xml_fromstring(argument, recover=False):
+    return etree.fromstring(argument, parser=parser(recover=recover))
+def xml_parse(path, recover=False, xinclude=False):
     tree = etree.parse(path, parser=parser(recover=recover))
     if xinclude:
         tree.xinclude()

From 37409b44f18650e17e297498790ff84f8683eb88 Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Sun, 2 Jun 2024 18:47:30 -0500
Subject: [PATCH 31/33] drop dependency lxml

---
 requirements.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index d27a32af..c9ae643a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1 @@
 Jinja2>=2.7,<4.0
-lxml==5.2.2

From 4a24403040db8eb14064c933fd59cacf26036e78 Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Sun, 2 Jun 2024 18:51:29 -0500
Subject: [PATCH 32/33] implement requested changes

- load lxml module first, and fallback to xml module
- don't extract zip file to parse the documents
- fix other formatting issues that I encountered
---
 glad/__main__.py      |  8 ++---
 glad/documentation.py | 84 +++++++++++++++++++++++--------------------
 glad/parse.py         | 56 ++++++++++++++++-------------
 glad/util.py          | 47 +++++++++++++++++++++---
 4 files changed, 123 insertions(+), 72 deletions(-)

diff --git a/glad/__main__.py b/glad/__main__.py
index 10b947e6..19534dcf 100644
--- a/glad/__main__.py
+++ b/glad/__main__.py
@@ -112,11 +112,11 @@ def load_documentations(specification_names, opener, spec_docs_classes=None):
             continue
 
         SpecificationDocs = spec_docs_classes[name]
-        spec_docs_dir = SpecificationDocs.default_out_dir()
+        spec_docs_file = SpecificationDocs.default_out_dir()
 
-        if os.path.isdir(spec_docs_dir):
-            logger.info('using local documentation: %s', spec_docs_dir)
-            spec_docs = SpecificationDocs.from_dir(spec_docs_dir, opener=opener)
+        if os.path.isfile(spec_docs_file):
+            logger.info("using local documentation: '%s'", spec_docs_file)
+            spec_docs = SpecificationDocs.from_file(spec_docs_file, opener=opener)
         else:
             logger.info('getting %r documentation from remote location', name)
             spec_docs = SpecificationDocs.from_remote(opener=opener)
diff --git a/glad/documentation.py b/glad/documentation.py
index 386dd229..06dde3c9 100644
--- a/glad/documentation.py
+++ b/glad/documentation.py
@@ -1,8 +1,9 @@
 import re
+import zipfile
 import glad.util
-from lxml import etree
-from glad.parse import DocumentationSet, SpecificationDocs, CommandDocs, xml_parse
-from glad.util import suffix, raw_text
+from pathlib import Path
+from glad.parse import DocumentationSet, SpecificationDocs, CommandDocs, xml_fromstring
+from glad.util import resolve_symbols, suffix, raw_text
 
 class OpenGLRefpages(SpecificationDocs):
     DOCS_NAME = 'opengl_refpages'
@@ -19,36 +20,48 @@ def select(self, feature_set):
         if current_major >= 4:
             available_versions.insert(0, 'gl4')
 
-        for version_dir in available_versions:
-            version_dir = self.docs_dir / f'{version_dir}'
-            if not version_dir.exists():
-                break
+        zf = zipfile.ZipFile(self.docs_file, 'r')
+        xml_files_in_version = {
+            version: [Path(name) for name in zf.namelist() if name.endswith('.xml') and version in name]
+            for version in available_versions
+        }
 
-            for html_file in version_dir.glob('*.xml'):
-                for command, docs in OpenGLRefpages.docs_from_html_file(html_file).items():
+        for version_dir in available_versions:
+            for xml_file in xml_files_in_version[version_dir]:
+                with zf.open(str(xml_file)) as f:
+                    xml_text = f.read().decode('utf-8')
+                parsed_docs = OpenGLRefpages.docs_from_xml(
+                    xml_text,
+                    version=version_dir,
+                    filename=xml_file.stem,
+                )
+                for command, docs in parsed_docs.items():
                     commands.setdefault(command, docs)
 
+        zf.close()
         return DocumentationSet(commands=commands)
 
     @classmethod
-    def docs_from_html_file(cls, path):
+    def docs_from_xml(cls, xml_text, version=None, filename=None):
         commands_parsed = dict()
-        version = path.parent.name
-        tree = xml_parse(path, recover=True, xinclude=True)
 
-        # gl4 files contain a namespace that polutes the tags, so we clean it up.
-        for elem in tree.getiterator():
+        xml_text = xml_text.replace('<mml:math>', '<mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML">')
+        xml_text = resolve_symbols(xml_text)
+
+        tree = xml_fromstring(xml_text.encode('utf-8'))
+
+        # gl4 files contain a namespace that polutes the tags, so we clean them up.
+        for elem in tree.iter():
             try:
                 if elem.tag.startswith('{'):
-                    elem.tag = etree.QName(elem).localname
+                    elem.tag = elem.tag.split('}')[-1]
                 if elem.tag.contains(':'):
                     elem.tag = elem.tag.split(':')[-1]
                 for key in elem.attrib:
                     if key.startswith('{'):
-                        elem.attrib[etree.QName(key).localname] = elem.attrib.pop(key)
+                        elem.attrib[key.split('}')[-1]] = elem.attrib.pop(key)
             except:
                 pass
-        etree.cleanup_namespaces(tree)
 
         sections = tree.findall('.//refsect1')
 
@@ -62,9 +75,9 @@ def docs_from_html_file(cls, path):
         brief = suffix(".", cls.xml_text(brief_block))
 
         if version == 'gl2.1':
-            docs_url = f'https://registry.khronos.org/OpenGL-Refpages/{version}/xhtml/{path.stem}.xml'
+            docs_url = f'https://registry.khronos.org/OpenGL-Refpages/{version}/xhtml/{filename}.xml'
         else:
-            docs_url = f'https://registry.khronos.org/OpenGL-Refpages/{version}/html/{path.stem}.xhtml'
+            docs_url = f'https://registry.khronos.org/OpenGL-Refpages/{version}/html/{filename}.xhtml'
 
         # Description parsing
         description = []
@@ -77,7 +90,7 @@ def docs_from_html_file(cls, path):
             description = list(
                 filter(
                     bool,
-                    (cls.xml_text(p) for p in blocks if p.tag != 'title'),
+                    (re.sub(f'^{CommandDocs.BREAK}', '', cls.xml_text(p)) for p in blocks if p.tag != 'title'),
                 ),
             )
 
@@ -154,31 +167,25 @@ def docs_from_html_file(cls, path):
         return commands_parsed
 
     @classmethod
-    def format(cls, e, is_tail=False):
+    def format(cls, e, parent=None, is_tail=False):
         if is_tail:
-            # closing a definition term
-            if e.tag == 'term':
-                return ''
-            # closing a mathjax row
-            if e.tag == 'mtr':
-                return '\n'
-            if e.tag in ('mn', 'msub'):
+            if e.tag in ('term', 'mn', 'msub'):
                 return ''
             return re.sub(r'\n+', '', e.tail)
 
-        if e.tag == 'link':
-            if e.attrib.get('href'):
-                return f'[{e.text}]({e.attrib["href"]})'
-            return e.text
         if e.tag == 'constant':
             return f'`{e.text}`'
         if e.tag == 'function':
             return f'`{e.text}`'
+        if e.tag == 'emphasis':
+            return f'*{e.text}*'
         if e.tag == 'term':
             return f'\n{CommandDocs.BREAK}- {e.text.strip()}'
         if e.tag == 'listitem':
-            if e.getparent().tag == 'varlistentry':
-                return f'\n{CommandDocs.BREAK}{e.text}'
+            if parent is None:
+                return f'\n{CommandDocs.BREAK}{e.text.strip()}'
+            if parent is not None and parent.tag == 'varlistentry':
+                return f'\n{CommandDocs.BREAK}{e.text.strip()}'
             return f'\n{CommandDocs.BREAK}- {e.text.strip()}'
         return re.sub(r'\n+', '', e.text)
 
@@ -197,9 +204,10 @@ def mfenced(e):
         text = ''.join(glad.util.itertext(
             e,
             convert={
-                'table': lambda _: f'(table omitted)',
-                'informaltable': lambda _: f'(table omitted)',
-                'programlisting': lambda _: f'(code omitted)',
+                'table': lambda _: '(table omitted)',
+                'informaltable': lambda _: '(table omitted)',
+                'include': lambda _: '(table omitted)',
+                'programlisting': lambda _: '(code omitted)',
                 'mfrac': lambda e, : f'{paren(cls.xml_text(e[0]))}/{paren(cls.xml_text(e[1]))}',
                 'msup': lambda e: f'{paren(cls.xml_text(e[0]))}^{paren(cls.xml_text(e[1]))}',
                 'msub': lambda e: f'{paren(cls.xml_text(e[0]))}_{paren(cls.xml_text(e[1]))}',
@@ -210,4 +218,4 @@ def mfenced(e):
         ))
         # \u2062, \u2062,
         # Invisible characters used by docs.gl to separate words.
-        return re.sub(r'\n?[ \u2062\u2061]+', ' ', text.strip())
+        return re.sub(r'\n?[ \u2062\u2061\t]+', ' ', text.strip())
diff --git a/glad/parse.py b/glad/parse.py
index 02077f57..313c5362 100644
--- a/glad/parse.py
+++ b/glad/parse.py
@@ -1,23 +1,31 @@
 from glad.sink import LoggingSink
 
-from lxml import etree
-from lxml.etree import ETCompatXMLParser as parser
-
-def xml_fromstring(argument, recover=False):
-    return etree.fromstring(argument, parser=parser(recover=recover))
-def xml_parse(path, recover=False, xinclude=False):
-    tree = etree.parse(path, parser=parser(recover=recover))
-    if xinclude:
-        tree.xinclude()
-    return tree.getroot()
+try:
+    from lxml import etree
+    from lxml.etree import ETCompatXMLParser as parser
+
+    def xml_fromstring(argument):
+        return etree.fromstring(argument, parser=parser())
+    def xml_parse(path):
+        return etree.parse(path, parser=parser()).getroot()
+except ImportError:
+    try:
+        import xml.etree.cElementTree as etree
+        from xml.etree.cElementTree import XMLParser as parser
+    except ImportError:
+        import xml.etree.ElementTree as etree
+        from xml.etree.ElementTree import XMLParser as parser
+
+    def xml_fromstring(argument):
+        return etree.fromstring(argument)
+    def xml_parse(path):
+        return etree.parse(path, parser=parser()).getroot()
 
 import re
 import copy
 import logging
 import os.path
 import warnings
-import zipfile
-from io import BytesIO
 from pathlib import Path
 from collections import defaultdict, OrderedDict, namedtuple, deque
 from contextlib import closing
@@ -1456,8 +1464,8 @@ class SpecificationDocs(object):
     SPEC = None
     URL = None
 
-    def __init__(self, docs_dir):
-        self.docs_dir = docs_dir
+    def __init__(self, docs_file):
+        self.docs_file = docs_file
 
     def select(self, feature_set):
         raise NotImplementedError
@@ -1466,33 +1474,31 @@ def select(self, feature_set):
     def default_out_dir(cls):
         if cls.DOCS_NAME is None:
             raise ValueError('DOCS_NAME not set')
-        return Path('.docs') / cls.DOCS_NAME
+        return Path('.docs') / f'{cls.DOCS_NAME}.zip'
 
     @classmethod
     def from_url(cls, url, opener=None):
         if opener is None:
             opener = URLOpener.default()
-        docs_dir = cls.default_out_dir()
-        zip_out_dir = docs_dir.parent
+
+        docs_file = cls.default_out_dir()
+        docs_file.parent.mkdir(parents=True, exist_ok=True)
 
         with closing(opener.urlopen(url)) as f:
             raw = f.read()
-            f = BytesIO(raw)
 
-            with zipfile.ZipFile(f) as zf:
-                zf.extractall(zip_out_dir)
-                zip_dir = zip_out_dir / zf.namelist()[0]
-                os.rename(zip_dir, docs_dir)
+            with open(docs_file, 'wb') as f:
+                f.write(raw)
 
-        return cls(docs_dir)
+        return cls(docs_file)
 
     @classmethod
     def from_remote(cls, opener=None):
         return cls.from_url(cls.URL, opener=opener)
 
     @classmethod
-    def from_dir(cls, dir_path, opener=None):
-        return cls(dir_path)
+    def from_file(cls, docs_file, opener=None):
+        return cls(docs_file)
 
 
 class DocumentationSet(object):
diff --git a/glad/util.py b/glad/util.py
index 28577a55..3f29f660 100644
--- a/glad/util.py
+++ b/glad/util.py
@@ -177,11 +177,11 @@ def raw_text(e):
     return ''.join(e.itertext())
 
 
-def _format_none(e, is_tail=False):
+def _format_none(e, parent=None, is_tail=False):
     return e.tail if is_tail else e.text
 
 
-def itertext(element, ignore=(), convert=dict(), format=_format_none):
+def itertext(element, parent=None, ignore=(), convert=dict(), format=_format_none):
     tag = element.tag
     if tag in ignore:
         return
@@ -193,13 +193,13 @@ def itertext(element, ignore=(), convert=dict(), format=_format_none):
         return
     if element.text is None:
         element.text = ''
-    yield format(element)
+    yield format(element, parent=parent)
 
     for e in element:
-        for s in itertext(e, ignore=ignore, convert=convert, format=format):
+        for s in itertext(e, ignore=ignore, parent=element, convert=convert, format=format):
             yield s
         if e.tail:
-            yield format(e, is_tail=True)
+            yield format(e, parent=element, is_tail=True)
 
 
 def expand_type_name(name):
@@ -236,3 +236,40 @@ def suffix(suffix, text):
     if text.strip().endswith(suffix):
         return text
     return f'{text}{suffix}'
+
+math_symbols_map = {
+    '&times;': '×',
+    '&minus;': '-',
+    '&it;': ' ',
+    '&af;': '',
+    '&nbsp;': ' ',
+    '&ne;': '≠',
+    '&le;': '≤',
+    '&ge;': '≥',
+    '&delta;': 'Δ',
+    '&Delta;': 'Δ',
+    '&PartialD;': '∂',
+    '&Prime;': '′',
+    '&infin;': '∞',
+    '&plus;': '+',
+    '&sdot;': '⋅',
+    '&lambda;': 'λ',
+    '&Hat;': '^',
+    '&Sigma;': 'Σ',
+    '&CenterDot': '·',
+    '&lceil;': '⌈',
+    '&rceil;': '⌉',
+    '&lfloor;': '⌊',
+    '&rfloor;': '⌋',
+    '&LeftFloor;': '⌊',
+    '&RightFloor;': '⌋',
+    '&LeftCeiling;': '⌈',
+    '&RightCeiling;': '⌉',
+    '&DoubleVerticalBar;': '∥',
+    '&VerticalBar;': '|',
+}
+
+def resolve_symbols(xml_text, symbols_map=math_symbols_map):
+    for symbol, rep in symbols_map.items():
+        xml_text = xml_text.replace(symbol, rep)
+    return xml_text

From 4ac233b1043173681300768878a00693fcec4b60 Mon Sep 17 00:00:00 2001
From: Paolo Flores <pflores.fisi22@gmail.com>
Date: Sun, 2 Jun 2024 19:26:44 -0500
Subject: [PATCH 33/33] improve readability and add comments for reference

---
 glad/documentation.py | 21 +++++++++++++--------
 glad/util.py          |  2 +-
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/glad/documentation.py b/glad/documentation.py
index 06dde3c9..420eb08b 100644
--- a/glad/documentation.py
+++ b/glad/documentation.py
@@ -3,7 +3,7 @@
 import glad.util
 from pathlib import Path
 from glad.parse import DocumentationSet, SpecificationDocs, CommandDocs, xml_fromstring
-from glad.util import resolve_symbols, suffix, raw_text
+from glad.util import resolve_entities, suffix, raw_text
 
 class OpenGLRefpages(SpecificationDocs):
     DOCS_NAME = 'opengl_refpages'
@@ -45,12 +45,14 @@ def select(self, feature_set):
     def docs_from_xml(cls, xml_text, version=None, filename=None):
         commands_parsed = dict()
 
+        # Some files don't have the proper namespace declaration for MathML.
         xml_text = xml_text.replace('<mml:math>', '<mml:math xmlns:mml="http://www.w3.org/1998/Math/MathML">')
-        xml_text = resolve_symbols(xml_text)
+        # Entities need to be resolved before parsing.
+        xml_text = resolve_entities(xml_text)
 
         tree = xml_fromstring(xml_text.encode('utf-8'))
 
-        # gl4 files contain a namespace that polutes the tags, so we clean them up.
+        # gl4 files contain large namespace declarations that pollute the tags. So we remove them.
         for elem in tree.iter():
             try:
                 if elem.tag.startswith('{'):
@@ -63,6 +65,8 @@ def docs_from_xml(cls, xml_text, version=None, filename=None):
             except:
                 pass
 
+        # Each refsect1 block contains a section of the documentation.
+        # Sections groups Description, Parameters, Notes, etc.
         sections = tree.findall('.//refsect1')
 
         # Brief parsing
@@ -130,8 +134,9 @@ def docs_from_xml(cls, xml_text, version=None, filename=None):
                 None,
             )
             if params_block is None:
-                for p in list(s for s in sections if raw_text(s.find('title')) == 'Parameters'):
+                for p in (s for s in sections if raw_text(s.find('title')) == 'Parameters'):
                     block_params = [raw_text(n) for n in p.findall('.//term//parameter')]
+                    # If all our func_params are contained in this block, we choose it.
                     if all(func_param in block_params for func_param in func_params):
                         params_block = p
                         break
@@ -149,16 +154,16 @@ def docs_from_xml(cls, xml_text, version=None, filename=None):
                     terms_stack.append(param_or_desc)
                     continue
                 if param_or_desc.tag == 'listitem':
+                    # Now that we have a listitem,  it is considered that all the terms
+                    # on the stack have the same parameter description.
+                    param_desc = cls.xml_text(param_or_desc).replace(CommandDocs.BREAK, '')
                     for terms in terms_stack:
                         param_names = [
                             p.text for p in terms.findall('.//parameter') if p.text in func_params
                         ]
 
                         for param_name in param_names:
-                            params.append(CommandDocs.Param(
-                                param_name,
-                                cls.xml_text(param_or_desc).replace(CommandDocs.BREAK, ''),
-                            ))
+                            params.append(CommandDocs.Param(param_name, param_desc))
                     terms_stack.clear()
 
             commands_parsed[func_name] = CommandDocs(
diff --git a/glad/util.py b/glad/util.py
index 3f29f660..bc5ca543 100644
--- a/glad/util.py
+++ b/glad/util.py
@@ -269,7 +269,7 @@ def suffix(suffix, text):
     '&VerticalBar;': '|',
 }
 
-def resolve_symbols(xml_text, symbols_map=math_symbols_map):
+def resolve_entities(xml_text, symbols_map=math_symbols_map):
     for symbol, rep in symbols_map.items():
         xml_text = xml_text.replace(symbol, rep)
     return xml_text