From e6f63e6f5ae6f22a130a20e3843f8f143e001b47 Mon Sep 17 00:00:00 2001 From: Brian Mesick Date: Thu, 19 Dec 2024 10:14:34 -0500 Subject: [PATCH 1/5] fix: Fix report rendering, improve navigation - Fixes broken links - Includes default templates in built package - Adds navigation for current page and all pages - Indicates installed packages vs. local code - Make generation timestamp more readable --- MANIFEST.in | 2 +- code_annotations/base.py | 15 +++++-- code_annotations/cli.py | 2 +- code_annotations/generate_docs.py | 30 ++++++++++---- .../report_templates/annotation.tpl | 5 --- .../report_templates/annotation_group.tpl | 3 -- code_annotations/report_templates/base.tpl | 14 ------- .../report_templates/rst/annotation.tpl | 13 ++++++ .../{ => rst}/annotation_data.tpl | 0 .../report_templates/rst/annotation_group.tpl | 2 + .../{ => rst}/annotation_list.tpl | 11 +++-- .../report_templates/rst/base.tpl | 41 +++++++++++++++++++ 12 files changed, 99 insertions(+), 39 deletions(-) delete mode 100644 code_annotations/report_templates/annotation.tpl delete mode 100644 code_annotations/report_templates/annotation_group.tpl delete mode 100644 code_annotations/report_templates/base.tpl create mode 100644 code_annotations/report_templates/rst/annotation.tpl rename code_annotations/report_templates/{ => rst}/annotation_data.tpl (100%) create mode 100644 code_annotations/report_templates/rst/annotation_group.tpl rename code_annotations/report_templates/{ => rst}/annotation_list.tpl (72%) create mode 100644 code_annotations/report_templates/rst/base.tpl diff --git a/MANIFEST.in b/MANIFEST.in index 08f31b1..7061606 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,4 +4,4 @@ include CONTRIBUTING.rst include LICENSE.txt include README.rst include requirements/base.in -recursive-include code_annotations *.html *.png *.gif *js *.css *jpg *jpeg *svg *py *.yaml *.yml +recursive-include code_annotations *.tpl *.html *.png *.gif *js *.css *jpg *jpeg *svg *py *.yaml *.yml diff --git a/code_annotations/base.py b/code_annotations/base.py index dddfcb7..d0b5b57 100644 --- a/code_annotations/base.py +++ b/code_annotations/base.py @@ -14,6 +14,8 @@ from code_annotations.exceptions import ConfigurationException from code_annotations.helpers import VerboseEcho +PACKAGE_DIR = os.path.dirname(os.path.realpath(__file__)) +DEFAULT_TEMPLATE_DIR = os.path.join(PACKAGE_DIR, "report_templates") class AnnotationConfig: """ @@ -58,10 +60,17 @@ def __init__(self, config_file_path, report_path_override=None, verbosity=1, sou self.echo(f"Configured for source path: {self.source_path}") self._configure_coverage(raw_config.get('coverage_target', None)) - self.report_template_dir = raw_config.get('report_template_dir') - self.rendered_report_dir = raw_config.get('rendered_report_dir') - self.rendered_report_file_extension = raw_config.get('rendered_report_file_extension') + + self.rendered_report_format = raw_config.get('rendered_report_format', 'rst') + self.report_template_dir = raw_config.get( + 'report_template_dir', + os.path.join(DEFAULT_TEMPLATE_DIR, self.rendered_report_format) + ) + self.rendered_report_dir = raw_config.get('rendered_report_dir', 'annotation_reports') self.rendered_report_source_link_prefix = raw_config.get('rendered_report_source_link_prefix') + self.trim_filename_prefixes = raw_config.get('trim_filename_prefixes', []) + self.third_party_package_location = raw_config.get('third_party_package_location', None) + self.rendered_report_file_extension = f".{self.rendered_report_format}" self._configure_annotations(raw_config) self._configure_extensions() diff --git a/code_annotations/cli.py b/code_annotations/cli.py index 853da13..2fecaac 100644 --- a/code_annotations/cli.py +++ b/code_annotations/cli.py @@ -244,7 +244,7 @@ def generate_docs(config_file, verbosity, report_files): for key in ( "report_template_dir", "rendered_report_dir", - "rendered_report_file_extension", + "rendered_report_format", "rendered_report_source_link_prefix", ): if not getattr(config, key): diff --git a/code_annotations/generate_docs.py b/code_annotations/generate_docs.py index abf8319..3d34627 100644 --- a/code_annotations/generate_docs.py +++ b/code_annotations/generate_docs.py @@ -27,7 +27,9 @@ def __init__(self, config, report_files): self.config = config self.echo = self.config.echo self.report_files = report_files - self.create_time = datetime.datetime.utcnow().isoformat() + self.create_time = datetime.datetime.now(tz=datetime.timezone.utc) + + print(self.create_time) self.full_report = self._aggregate_reports() @@ -62,6 +64,12 @@ def _add_report_file_to_full_report(self, report_file, report): loaded_report = yaml.safe_load(report_file) for filename in loaded_report: + trimmed_filename = filename + for prefix in self.config.trim_filename_prefixes: + if filename.startswith(prefix): + trimmed_filename = filename[len(prefix):] + break + if filename in report: for loaded_annotation in loaded_report[filename]: found = False @@ -74,9 +82,10 @@ def _add_report_file_to_full_report(self, report_file, report): break if not found: - report[filename].append(loaded_annotation) + report[trimmed_filename].append(loaded_annotation) else: - report[filename] = loaded_report[filename] + report[trimmed_filename] = loaded_report[filename] + def _aggregate_reports(self): """ @@ -91,7 +100,7 @@ def _aggregate_reports(self): return report - def _write_doc_file(self, doc_filename, doc_data): + def _write_doc_file(self, doc_title, doc_filename, doc_data): """ Write out a single report file with the given data. This is rendered using the configured top level template. @@ -110,14 +119,17 @@ def _write_doc_file(self, doc_filename, doc_data): with open(full_doc_filename, 'w') as output: output.write(self.top_level_template.render( + doc_title=doc_title, create_time=self.create_time, report=doc_data, all_choices=self.all_choices, all_annotations=self.config.annotation_tokens, group_mapping=self.group_mapping, slugify=slugify, - source_link_prefix=self.config.rendered_report_source_link_prefix) - ) + source_link_prefix=self.config.rendered_report_source_link_prefix, + third_party_package_location=self.config.third_party_package_location, + ), + ) def _generate_per_choice_docs(self): """ @@ -130,7 +142,7 @@ def _generate_per_choice_docs(self): if isinstance(annotation['annotation_data'], list) and choice in annotation['annotation_data']: choice_report[filename].append(annotation) - self._write_doc_file(f'choice_{choice}', choice_report) + self._write_doc_file(f"All References to Choice '{choice}'", f'choice_{choice}', choice_report) def _generate_per_annotation_docs(self): """ @@ -143,13 +155,13 @@ def _generate_per_annotation_docs(self): if report_annotation['annotation_token'] == annotation: annotation_report[filename].append(report_annotation) - self._write_doc_file(f'annotation_{annotation}', annotation_report) + self._write_doc_file(f"All References to Annotation '{annotation}'", f'annotation_{annotation}', annotation_report) def render(self): """ Perform the rendering of all documentation using the configured Jinja2 templates. """ # Generate the top level list of all annotations - self._write_doc_file('index', self.full_report) + self._write_doc_file("Complete Annotation List", 'index', self.full_report) self._generate_per_choice_docs() self._generate_per_annotation_docs() diff --git a/code_annotations/report_templates/annotation.tpl b/code_annotations/report_templates/annotation.tpl deleted file mode 100644 index d0ce6c8..0000000 --- a/code_annotations/report_templates/annotation.tpl +++ /dev/null @@ -1,5 +0,0 @@ -{% if annotation.extra and annotation.extra.object_id %} -`<{{ annotation.extra.object_id }}> line {{ annotation.line_number }} <{{ source_link_prefix }}{{ filename }}#L{{ annotation.line_number }}>`_: {{ annotation.annotation_token }} {% include "annotation_data.tpl" %} -{% else %} -`{{ filename }}:{{ annotation.line_number }} <{{ source_link_prefix }}{{ filename }}#L{{ annotation.line_number }}>`_: {{ annotation.annotation_token }} {% include "annotation_data.tpl" %} -{% endif %} diff --git a/code_annotations/report_templates/annotation_group.tpl b/code_annotations/report_templates/annotation_group.tpl deleted file mode 100644 index ed719b0..0000000 --- a/code_annotations/report_templates/annotation_group.tpl +++ /dev/null @@ -1,3 +0,0 @@ -.. _index.rst#{{ slugify(filename + '-' + annotation.report_group_id |string) }}: -.. admonition:: Group "{{ group_mapping[annotation.annotation_token] }}" - diff --git a/code_annotations/report_templates/base.tpl b/code_annotations/report_templates/base.tpl deleted file mode 100644 index b460430..0000000 --- a/code_annotations/report_templates/base.tpl +++ /dev/null @@ -1,14 +0,0 @@ -{% block content %}{% endblock %} - -{% for choice in all_choices %} -.. _choice_{{ slugify(choice) }}: {{ slugify('choice_' + choice) + '.rst' }} -{% endfor %} - -{% for annotation in all_annotations %} -.. _annotation_{{ slugify(annotation) }}: {{ slugify('annotation_' + annotation) + '.rst' }} -{% endfor %} - - -{% block footer %} -Built at {{ create_time }} -{% endblock %} diff --git a/code_annotations/report_templates/rst/annotation.tpl b/code_annotations/report_templates/rst/annotation.tpl new file mode 100644 index 0000000..2acccb7 --- /dev/null +++ b/code_annotations/report_templates/rst/annotation.tpl @@ -0,0 +1,13 @@ +{% if is_third_party %} +{# no links for third party code since we don't know where to link to #} +{% if annotation.extra and annotation.extra.object_id %} +{{ annotation.extra.object_id }} {% if annotation.line_number > 0 %}line {{ annotation.line_number }}{% endif %}: {{ annotation.annotation_token }} {% include "annotation_data.tpl" %} +{% else %} +{{ filename }}:{{ annotation.line_number }}: {{ annotation.annotation_token }} {% include "annotation_data.tpl" %} +{% endif %} + +{% elif annotation.extra and annotation.extra.object_id %} +`{{ annotation.extra.object_id }} {% if annotation.line_number > 0 %}line {{ annotation.line_number }}{% endif %} <{{ source_link_prefix }}{{ filename }}#L{{ annotation.line_number }}>`_: {{ annotation.annotation_token }} {% include "annotation_data.tpl" %} +{% else %} +`{{ filename }}:{{ annotation.line_number }} <{{ source_link_prefix }}{{ filename }}#L{{ annotation.line_number }}>`_: {{ annotation.annotation_token }} {% include "annotation_data.tpl" %} +{% endif %} diff --git a/code_annotations/report_templates/annotation_data.tpl b/code_annotations/report_templates/rst/annotation_data.tpl similarity index 100% rename from code_annotations/report_templates/annotation_data.tpl rename to code_annotations/report_templates/rst/annotation_data.tpl diff --git a/code_annotations/report_templates/rst/annotation_group.tpl b/code_annotations/report_templates/rst/annotation_group.tpl new file mode 100644 index 0000000..495a7b8 --- /dev/null +++ b/code_annotations/report_templates/rst/annotation_group.tpl @@ -0,0 +1,2 @@ +.. _index.rst#{{ slugify(filename + '-' + annotation.report_group_id |string) }}: +.. admonition:: {{ group_mapping[annotation.annotation_token] or annotation.annotation_token }} diff --git a/code_annotations/report_templates/annotation_list.tpl b/code_annotations/report_templates/rst/annotation_list.tpl similarity index 72% rename from code_annotations/report_templates/annotation_list.tpl rename to code_annotations/report_templates/rst/annotation_list.tpl index 624fd77..20226ab 100644 --- a/code_annotations/report_templates/annotation_list.tpl +++ b/code_annotations/report_templates/rst/annotation_list.tpl @@ -1,17 +1,22 @@ {% extends "base.tpl" %} {% block content %} -Complete Annotation List ------------------------- Annotations found in {{ report|length }} files. {% for filename in report %} -{{ filename }} has {{ report[filename]|length }} annotations. +{% set is_third_party = third_party_package_location in filename %} + +{{ filename }} +{{ "-" * filename|length }} + + .. note:: {{ report[filename]|length }} annotations {% if is_third_party %}(installed package){% endif %} + {% for annotation in report[filename] %} {% if annotation.report_group_id %} {% if loop.changed(annotation.report_group_id) %} {% include 'annotation_group.tpl' %} + {% endif %} * {% include 'annotation.tpl' %} diff --git a/code_annotations/report_templates/rst/base.tpl b/code_annotations/report_templates/rst/base.tpl new file mode 100644 index 0000000..77499d4 --- /dev/null +++ b/code_annotations/report_templates/rst/base.tpl @@ -0,0 +1,41 @@ +{{ "#" * doc_title|length }} +{{ doc_title }} +{{ "#" * doc_title|length }} + +.. sidebar:: Table of Contents + + `Home `_ + + Annotations + + {% for a in all_annotations %} + * annotation_{{ slugify(a) }}_ + {% endfor %} + + Choices + + {% for choice in all_choices %} + * choice_{{ slugify(choice) }}_ + {% endfor %} + + +.. contents:: + +{% block content %}{% endblock %} + + +{# backlinks for all choices #} +{% for choice in all_choices %} +.. _choice_{{ slugify(choice) }}: {{ slugify('choice_' + choice) + '.rst' }} +{% endfor %} + + +{# backlinks for all annotations #} +{% for annotation in all_annotations %} +.. _annotation_{{ slugify(annotation) }}: {{ slugify('annotation_' + annotation) + '.rst' }} +{% endfor %} + + +{% block footer %} +Built at {{ create_time.strftime('%Y-%m-%d %H:%M:%S %Z') }} +{% endblock %} From d082c46fb9937b265516c17115dfa97ebe9c2635 Mon Sep 17 00:00:00 2001 From: Brian Mesick Date: Thu, 19 Dec 2024 15:00:26 -0500 Subject: [PATCH 2/5] feat: Add HTML report output, report docs --- README.rst | 28 ------ code_annotations/base.py | 5 +- code_annotations/generate_docs.py | 9 +- .../report_templates/html/annotation.tpl | 14 +++ .../report_templates/html/annotation_data.tpl | 8 ++ .../report_templates/html/annotation_list.tpl | 27 ++++++ .../report_templates/html/base.tpl | 92 +++++++++++++++++++ docs/configuration.rst | 27 ++++++ docs/getting_started.rst | 14 +++ ....annotations_test_success_with_report_docs | 5 +- ...tations_test_success_with_report_docs_html | 25 +++++ tests/test_generate_docs.py | 36 +++++++- 12 files changed, 253 insertions(+), 37 deletions(-) create mode 100644 code_annotations/report_templates/html/annotation.tpl create mode 100644 code_annotations/report_templates/html/annotation_data.tpl create mode 100644 code_annotations/report_templates/html/annotation_list.tpl create mode 100644 code_annotations/report_templates/html/base.tpl create mode 100644 tests/test_configurations/.annotations_test_success_with_report_docs_html diff --git a/README.rst b/README.rst index dd91285..37e6f32 100644 --- a/README.rst +++ b/README.rst @@ -1,9 +1,6 @@ code-annotations ============================= -|pypi-badge| |CI| |codecov-badge| |doc-badge| |pyversions-badge| -|license-badge| - Extensible tools for parsing annotations in codebases Overview @@ -55,28 +52,3 @@ Have a question about this repository, or about Open edX in general? Please refer to this `list of resources`_ if you need any assistance. .. _list of resources: https://open.edx.org/getting-help - - -.. |pypi-badge| image:: https://img.shields.io/pypi/v/code-annotations.svg - :target: https://pypi.python.org/pypi/code-annotations/ - :alt: PyPI - -.. |CI| image:: https://github.com/openedx/code-annotations/workflows/Python%20CI/badge.svg?branch=master - :target: https://github.com/openedx/code-annotations/actions?query=workflow%3A%22Python+CI%22 - :alt: CI - -.. |codecov-badge| image:: http://codecov.io/github/edx/code-annotations/coverage.svg?branch=master - :target: http://codecov.io/github/edx/code-annotations?branch=master - :alt: Codecov - -.. |doc-badge| image:: https://readthedocs.org/projects/code-annotations/badge/?version=latest - :target: http://code-annotations.readthedocs.io/en/latest/ - :alt: Documentation - -.. |pyversions-badge| image:: https://img.shields.io/pypi/pyversions/code-annotations.svg - :target: https://pypi.python.org/pypi/code-annotations/ - :alt: Supported Python versions - -.. |license-badge| image:: https://img.shields.io/github/license/edx/code-annotations.svg - :target: https://github.com/openedx/code-annotations/blob/master/LICENSE.txt - :alt: License diff --git a/code_annotations/base.py b/code_annotations/base.py index d0b5b57..6f7964d 100644 --- a/code_annotations/base.py +++ b/code_annotations/base.py @@ -17,6 +17,7 @@ PACKAGE_DIR = os.path.dirname(os.path.realpath(__file__)) DEFAULT_TEMPLATE_DIR = os.path.join(PACKAGE_DIR, "report_templates") + class AnnotationConfig: """ Configuration shared among all Code Annotations commands. @@ -67,9 +68,9 @@ def __init__(self, config_file_path, report_path_override=None, verbosity=1, sou os.path.join(DEFAULT_TEMPLATE_DIR, self.rendered_report_format) ) self.rendered_report_dir = raw_config.get('rendered_report_dir', 'annotation_reports') - self.rendered_report_source_link_prefix = raw_config.get('rendered_report_source_link_prefix') + self.rendered_report_source_link_prefix = raw_config.get('rendered_report_source_link_prefix', None) self.trim_filename_prefixes = raw_config.get('trim_filename_prefixes', []) - self.third_party_package_location = raw_config.get('third_party_package_location', None) + self.third_party_package_location = raw_config.get('third_party_package_location', "site-packages") self.rendered_report_file_extension = f".{self.rendered_report_format}" self._configure_annotations(raw_config) diff --git a/code_annotations/generate_docs.py b/code_annotations/generate_docs.py index 3d34627..b13a1ca 100644 --- a/code_annotations/generate_docs.py +++ b/code_annotations/generate_docs.py @@ -86,7 +86,6 @@ def _add_report_file_to_full_report(self, report_file, report): else: report[trimmed_filename] = loaded_report[filename] - def _aggregate_reports(self): """ Combine all of the given report files into a single report object. @@ -105,6 +104,7 @@ def _write_doc_file(self, doc_title, doc_filename, doc_data): Write out a single report file with the given data. This is rendered using the configured top level template. Args: + doc_title: Title to use for the document. doc_filename: Filename to write to. doc_data: Dict of reporting data to use, in the {'file name': [list, of, annotations,]} style. """ @@ -128,8 +128,7 @@ def _write_doc_file(self, doc_title, doc_filename, doc_data): slugify=slugify, source_link_prefix=self.config.rendered_report_source_link_prefix, third_party_package_location=self.config.third_party_package_location, - ), - ) + )) def _generate_per_choice_docs(self): """ @@ -155,7 +154,9 @@ def _generate_per_annotation_docs(self): if report_annotation['annotation_token'] == annotation: annotation_report[filename].append(report_annotation) - self._write_doc_file(f"All References to Annotation '{annotation}'", f'annotation_{annotation}', annotation_report) + self._write_doc_file( + f"All References to Annotation '{annotation}'", f'annotation_{annotation}', annotation_report + ) def render(self): """ diff --git a/code_annotations/report_templates/html/annotation.tpl b/code_annotations/report_templates/html/annotation.tpl new file mode 100644 index 0000000..8f4055b --- /dev/null +++ b/code_annotations/report_templates/html/annotation.tpl @@ -0,0 +1,14 @@ +{% if is_third_party %} +{# no links for third party code since we don't know where to link to #} +{% if annotation.extra and annotation.extra.object_id %} +{{ annotation.extra.object_id }} {% if annotation.line_number > 0 %}line {{ annotation.line_number }}{% endif %}: {{ annotation.annotation_token }} {% include "annotation_data.tpl" %} +{% else %} + {% if loop.changed(annotation.line_number)%}{{ filename }}:{{ annotation.line_number }}
{% endif %}: + {{ annotation.annotation_token }} {% include "annotation_data.tpl" %} +{% endif %} + +{% elif annotation.extra and annotation.extra.object_id %} +{{ annotation.extra.object_id }} {% if annotation.line_number > 0 %}line {{ annotation.line_number }}{% endif %}: {{ annotation.annotation_token }} {% include "annotation_data.tpl" %} +{% else %} +`{{ filename }}:{{ annotation.line_number }}: {{ annotation.annotation_token }} {% include "annotation_data.tpl" %} +{% endif %} diff --git a/code_annotations/report_templates/html/annotation_data.tpl b/code_annotations/report_templates/html/annotation_data.tpl new file mode 100644 index 0000000..a95251c --- /dev/null +++ b/code_annotations/report_templates/html/annotation_data.tpl @@ -0,0 +1,8 @@ +{% if annotation.annotation_data is sequence and annotation.annotation_data is not string %} +{% for a in annotation.annotation_data %} +{{ a }}{% if not loop.last %}, {% endif %} +{% endfor %} + +{% else %} +{{ annotation.annotation_data }} +{% endif %} diff --git a/code_annotations/report_templates/html/annotation_list.tpl b/code_annotations/report_templates/html/annotation_list.tpl new file mode 100644 index 0000000..83434f2 --- /dev/null +++ b/code_annotations/report_templates/html/annotation_list.tpl @@ -0,0 +1,27 @@ +{% extends "base.tpl" %} +{% block content %} +Annotations found in {{ report|length }} files. + +{% for filename in report %} +{% set is_third_party = third_party_package_location in filename %} + +

{{ filename }}

+
+ {{ report[filename]|length }} annotations {% if is_third_party %}(installed package){% endif %}
+
+ + {% for annotation in report[filename] %} + {% if loop.changed(annotation.report_group_id) %} + {% if not loop.first %}{% endif %} +
    + {% endif %} +
  • {% include 'annotation.tpl' %}
    • + {% if loop.last %} +
+ {% endif %} + {% endfor %} + + +{% endfor %} + +{% endblock %} diff --git a/code_annotations/report_templates/html/base.tpl b/code_annotations/report_templates/html/base.tpl new file mode 100644 index 0000000..b292122 --- /dev/null +++ b/code_annotations/report_templates/html/base.tpl @@ -0,0 +1,92 @@ + + + {{ doc_title }} + + + + +

{{ doc_title }}

+ +
+
+
+

Home

+ +

Annotations

+ + + +

Choices

+ + +
+
+

Files in this page

+ + + {% block content %}{% endblock %} +
+
+
+{% block footer %} +
+

+
+Built at {{ create_time.strftime('%Y-%m-%d %H:%M:%S %Z') }} +
+{% endblock %} + + diff --git a/docs/configuration.rst b/docs/configuration.rst index 83cc836..6469bb6 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -26,6 +26,15 @@ Configuring Code Annotations is a pretty simple affair. Here is an example showi - py3 javascript: - js + # Options only used for human readable reports + report_template_dir: /my_custom_templates/report_templates/ + rendered_report_dir: rendered_reports + rendered_report_format: html + rendered_report_source_link_prefix: https://github.com/my_org/my_project/blob/master/ + trim_filename_prefixes: + - /my_org/my_project/ + - /my_project/venv/lib/python3.11/ + third_party_package_location: site-packages ``source_path`` The file or directory to be searched for annotations. If a directory, it will be searched recursively. This can be @@ -68,3 +77,21 @@ Configuring Code Annotations is a pretty simple affair. Here is an example showi extensions are turned on here. The key is the extension name, as given in the ``setup.py`` or ``setup.cfg`` of the package that installed the extension. The values are a list of file extensions that, when found, will be passed to the extension for annotation searching. See :doc:`extensions` for details. + +``report_template_dir`` (optional) + When running the ``generate_docs`` comman you can specify a custom template directory here to override the default templates if you would like a different look. + +``rendered_report_dir`` (optional) + When running the ``generate_docs`` command, this option specifies the directory where the rendered report will be written. The default is ``annotation_reports`` in the current working directory. + +``rendered_report_format`` (optional) + When running the ``generate_docs`` command, this option specifies the format of the rendered report. Options are ``html`` and ``rst``. The default is ``rst``. + +``rendered_report_source_link_prefix`` (optional) + When running the ``generate_docs`` command, this option specifies the URL prefix to use when linking to source files in the rendered report. When specified, "local" source files (those not found in site-packages) will be appended to this setting to create hyperlinks to the lines in source files online. For Github links this is the correct format: ``https://github.com/openedx/edx-platform/blob/master/``. The default is an None. + +``trim_filename_prefixes`` (optional) + When running the ``generate_docs`` command, this option specifies a list of prefixes to remove from the beginning of filenames in the rendered report. This is useful for making the report more readable by removing long, repetitive prefixes of the type often found in a Django search. The default is an empty list. + +``third_party_package_location`` (optional) + When running the ``generate_docs`` command, this option specifies the location of third party packages that may have been found in a Django search. This is used to determine if a file is a third party file or not. The default is ``site-packages``. diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 0fcf005..2ed3b1b 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -126,3 +126,17 @@ Add more structure to your annotations Annotations can be more than simple messages. They can enforce the use of choices from a fixed list, and can be grouped to provide more context-aware information. See :doc:`configuration` and :doc:`writing_annotations` for more information on how to use those options. + +Get a human readable report +--------------------------- +The output generated by the search commands is a YAML file. To get a human readable report generated from one of those files in either rst or html format, you can use the ``generate_docs`` command. + +There are several configuration options available for this command, including the ability to specify the output format, the output directory, and create links to the source files on sites like Github. For more information, see :doc:`configuration`. Once your configuration is set, you can run: + +.. code-block:: bash + + $ code_annotations generate_docs --config_file /path/to/your/config /path/to/your/report.yaml + +Which will generate files in the output directory you configured. From there you can open the files in your browser to see the report, if you chose HTML, or use a tool like `restview`_ to render the RST files to your browser. + +.. _restview: https://pypi.python.org/pypi/restview diff --git a/tests/test_configurations/.annotations_test_success_with_report_docs b/tests/test_configurations/.annotations_test_success_with_report_docs index 946c852..ff4858f 100644 --- a/tests/test_configurations/.annotations_test_success_with_report_docs +++ b/tests/test_configurations/.annotations_test_success_with_report_docs @@ -1,10 +1,11 @@ source_path: tests/extensions/javascript_test_files/ report_path: test_reports safelist_path: .annotation_safe_list.yml -report_template_dir: code_annotations/report_templates/ +report_template_dir: code_annotations/report_templates/rst rendered_report_dir: test_reports/ -rendered_report_file_extension: .rst +rendered_report_format: rst rendered_report_source_link_prefix: https://github.com/openedx/edx-platform/tree/master/ + coverage_target: 50.0 annotations: ".. no_pii:": diff --git a/tests/test_configurations/.annotations_test_success_with_report_docs_html b/tests/test_configurations/.annotations_test_success_with_report_docs_html new file mode 100644 index 0000000..535adad --- /dev/null +++ b/tests/test_configurations/.annotations_test_success_with_report_docs_html @@ -0,0 +1,25 @@ +source_path: tests/extensions/javascript_test_files/ +report_path: test_reports +safelist_path: .annotation_safe_list.yml +report_template_dir: code_annotations/report_templates/rst +rendered_report_dir: test_reports/ +rendered_report_format: html +rendered_report_source_link_prefix: https://github.com/openedx/edx-platform/tree/master/ +trim_filename_prefixes: + - /foo/bar +third_party_package_location: site-packages + +coverage_target: 50.0 +annotations: + ".. no_pii:": + ".. ignored:": + choices: [irrelevant, terrible, silly-silly] + "pii_group": + - ".. pii:": + - ".. pii_types:": + choices: [id, name, other] + - ".. pii_retirement:": + choices: [retained, local_api, consumer_api, third_party] +extensions: + python: + - pyt diff --git a/tests/test_generate_docs.py b/tests/test_generate_docs.py index 94c9937..f352cb3 100644 --- a/tests/test_generate_docs.py +++ b/tests/test_generate_docs.py @@ -49,6 +49,40 @@ def test_generate_report_simple(): assert os.path.exists(created_doc) +def test_generate_report_simple_html(): + find_result = call_script( + ( + 'static_find_annotations', + '--config_file', + 'tests/test_configurations/.annotations_test_python_only', + '--source_path=tests/extensions/python_test_files/simple_success.pyt', + '--no_lint', + ), + delete_test_reports=False) + + assert find_result.exit_code == EXIT_CODE_SUCCESS + assert "Writing report..." in find_result.output + report_file = get_report_filename_from_output(find_result.output) + + report_result = call_script( + ( + 'generate_docs', + report_file, + '--config_file', + 'tests/test_configurations/.annotations_test_success_with_report_docs_html', + '-vv' + ), + delete_test_docs=False + ) + + assert find_result.exit_code == EXIT_CODE_SUCCESS + assert "Report rendered in" in report_result.output + + # All file types are created + for created_doc in ('test_reports/index.html', 'test_reports/choice-id.html', 'test_reports/annotation-pii.html'): + assert os.path.exists(created_doc) + + def _do_find(source_path, new_report_path): """ Do a static annotation search with report, rename the report to a distinct name. @@ -167,4 +201,4 @@ def test_generate_report_missing_key(): )) assert report_result.exit_code == EXIT_CODE_FAILURE - assert "No report_template_dir key in tests/test_configurations/" in report_result.output + assert "No rendered_report_source_link_prefix key in" in report_result.output From b33607d9eb21a0f41994804b700f7d030fc2e5e3 Mon Sep 17 00:00:00 2001 From: Brian Mesick Date: Fri, 20 Dec 2024 09:06:26 -0500 Subject: [PATCH 3/5] test: Improve test coverage --- code_annotations/generate_docs.py | 3 --- .../.annotations_test_success_with_report_docs_html | 3 ++- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/code_annotations/generate_docs.py b/code_annotations/generate_docs.py index b13a1ca..e32dc37 100644 --- a/code_annotations/generate_docs.py +++ b/code_annotations/generate_docs.py @@ -28,9 +28,6 @@ def __init__(self, config, report_files): self.echo = self.config.echo self.report_files = report_files self.create_time = datetime.datetime.now(tz=datetime.timezone.utc) - - print(self.create_time) - self.full_report = self._aggregate_reports() self.jinja_environment = jinja2.Environment( diff --git a/tests/test_configurations/.annotations_test_success_with_report_docs_html b/tests/test_configurations/.annotations_test_success_with_report_docs_html index 535adad..27800be 100644 --- a/tests/test_configurations/.annotations_test_success_with_report_docs_html +++ b/tests/test_configurations/.annotations_test_success_with_report_docs_html @@ -6,7 +6,8 @@ rendered_report_dir: test_reports/ rendered_report_format: html rendered_report_source_link_prefix: https://github.com/openedx/edx-platform/tree/master/ trim_filename_prefixes: - - /foo/bar + - tacos + - simple third_party_package_location: site-packages coverage_target: 50.0 From f88fb570471f6e7f79b7177f32e146d772cc816a Mon Sep 17 00:00:00 2001 From: Brian Mesick Date: Thu, 16 Jan 2025 10:07:06 -0500 Subject: [PATCH 4/5] fix: Error when no report files passed in to render_reports --- code_annotations/cli.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/code_annotations/cli.py b/code_annotations/cli.py index 2fecaac..a7bd997 100644 --- a/code_annotations/cli.py +++ b/code_annotations/cli.py @@ -7,6 +7,7 @@ import traceback import click +from django.db.models.fields.related import RECURSIVE_RELATIONSHIP_CONSTANT from code_annotations.base import AnnotationConfig, ConfigurationException from code_annotations.find_django import DjangoSearch @@ -241,6 +242,11 @@ def generate_docs(config_file, verbosity, report_files): try: config = AnnotationConfig(config_file, verbosity) + if not report_files: + raise ConfigurationException( + "No report files provided. Please provide one or more report files to generate docs from." + ) + for key in ( "report_template_dir", "rendered_report_dir", From 5058d4b5a12dc12dcf0bd3fed2ea8d62808e882d Mon Sep 17 00:00:00 2001 From: Brian Mesick Date: Thu, 16 Jan 2025 13:14:41 -0500 Subject: [PATCH 5/5] style: Remove unused import --- code_annotations/cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/code_annotations/cli.py b/code_annotations/cli.py index a7bd997..9e56f02 100644 --- a/code_annotations/cli.py +++ b/code_annotations/cli.py @@ -7,7 +7,6 @@ import traceback import click -from django.db.models.fields.related import RECURSIVE_RELATIONSHIP_CONSTANT from code_annotations.base import AnnotationConfig, ConfigurationException from code_annotations.find_django import DjangoSearch