diff --git a/scanpipe/pipes/output.py b/scanpipe/pipes/output.py index 48787a9d7..45cafb07f 100644 --- a/scanpipe/pipes/output.py +++ b/scanpipe/pipes/output.py @@ -53,6 +53,7 @@ from scancodeio import SCAN_NOTICE from scancodeio import __version__ as scancodeio_version from scanpipe.pipes import docker +from scanpipe.pipes import flag from scanpipe.pipes import spdx scanpipe_app = apps.get_app_config("scanpipe") @@ -96,6 +97,30 @@ def get_queryset(project, model_name): return querysets.get(model_name) +TODO_FIELDS = [ + "path", + "status", + "size", + "name", + "extension", + "programming_language", + "mime_type", + "tag", + "detected_license_expression", + "compliance_alert", + "project__name", +] + + +def get_todos_data(project): + """Return the list of Resources that requires review.""" + return ( + project.codebaseresources.files() + .filter(status=flag.REQUIRES_REVIEW) + .values(*TODO_FIELDS) + ) + + def queryset_to_csv_file(queryset, fieldnames, output_file): """ Output csv content generated from the provided `queryset` objects to the @@ -346,8 +371,12 @@ def _add_xlsx_worksheet(workbook, worksheet_name, rows, fields): for row_index, record in enumerate(rows, start=1): row_errors = [] + record_is_dict = isinstance(record, dict) for col_index, field in enumerate(fields): - value = getattr(record, field) + if record_is_dict: + value = record.get(field) + else: + value = getattr(record, field) if not value: continue @@ -481,6 +510,9 @@ def to_xlsx(project): if layers_data := docker.get_layers_data(project): _add_xlsx_worksheet(workbook, "LAYERS", layers_data, docker.layer_fields) + if todos_data := get_todos_data(project): + _add_xlsx_worksheet(workbook, "TODOS", todos_data, TODO_FIELDS) + return output_file diff --git a/scanpipe/tests/pipes/test_output.py b/scanpipe/tests/pipes/test_output.py index ca1c0b5b8..a618b3cf3 100644 --- a/scanpipe/tests/pipes/test_output.py +++ b/scanpipe/tests/pipes/test_output.py @@ -33,6 +33,7 @@ from django.core.management import call_command from django.test import TestCase +import openpyxl import xlsxwriter from licensedcode.cache import get_licensing from lxml import etree @@ -42,10 +43,12 @@ from scanpipe.models import CodebaseResource from scanpipe.models import Project from scanpipe.models import ProjectMessage +from scanpipe.pipes import flag from scanpipe.pipes import output from scanpipe.tests import FIXTURES_REGEN from scanpipe.tests import make_dependency from scanpipe.tests import make_package +from scanpipe.tests import make_resource_file from scanpipe.tests import mocked_now from scanpipe.tests import package_data1 @@ -210,16 +213,30 @@ def test_scanpipe_pipes_outputs_to_xlsx(self): model="Model", details={}, ) + make_resource_file( + project=project, path="path/file1.ext", status=flag.REQUIRES_REVIEW + ) output_file = output.to_xlsx(project=project) self.assertIn(output_file.name, project.output_root) # Make sure the output can be generated even if the work_directory was wiped shutil.rmtree(project.work_directory) - with self.assertNumQueries(8): + with self.assertNumQueries(9): output_file = output.to_xlsx(project=project) self.assertIn(output_file.name, project.output_root) + workbook = openpyxl.load_workbook(output_file, read_only=True, data_only=True) + expected_sheet_names = [ + "PACKAGES", + "DEPENDENCIES", + "RESOURCES", + "RELATIONS", + "MESSAGES", + "TODOS", + ] + self.assertEqual(expected_sheet_names, workbook.get_sheet_names()) + def test_scanpipe_pipes_outputs_vulnerability_as_cyclonedx(self): component_bom_ref = "pkg:pypi/django@4.0.10" data = self.data / "cyclonedx/django-4.0.10-vulnerability.json" @@ -480,6 +497,34 @@ def test_scanpipe_pipes_outputs_to_attribution(self): output_file = output.to_attribution(project=project) self.assertEqual("EMPTY_TEMPLATE", output_file.read_text()) + def test_scanpipe_pipes_outputs_get_todos_data(self): + project = Project.objects.create(name="Analysis") + todos_data = output.get_todos_data(project) + self.assertEqual([], list(todos_data)) + + make_resource_file( + project=project, path="path/file1.ext", status=flag.REQUIRES_REVIEW + ) + make_resource_file(project=project, path="path/file2.ext") + + todos_data = output.get_todos_data(project) + expected = [ + { + "path": "path/file1.ext", + "status": "requires-review", + "size": None, + "name": "file1.ext", + "extension": ".ext", + "programming_language": "", + "mime_type": "", + "tag": "path", + "detected_license_expression": "", + "compliance_alert": "", + "project__name": "Analysis", + } + ] + self.assertEqual(expected, list(todos_data)) + class ScanPipeXLSXOutputPipesTest(TestCase): def test__add_xlsx_worksheet_does_truncates_long_strings_over_max_len(self):