Issue 6043, 6044 - Enhance Rust and JS bundling and add SPDX licenses…

… for both

Description: Update the generation script in 'rpm.mk' and 'bundle-rust-downstream.py'
to include SPDX license information for combined JavaScript (npm) and Cargo dependencies.

Fixes: 389ds#6043
Fixes: 389ds#6044

Reviewed by: ?

rpm.mk

@@ -12,6 +12,8 @@ JEMALLOC_URL ?= $(shell rpmspec -P $(RPMBUILD)/SPECS/389-ds-base.spec | awk '/^S
 JEMALLOC_TARBALL ?= $(shell basename "$(JEMALLOC_URL)")
 BUNDLE_JEMALLOC = 1
 NODE_MODULES_TEST = src/cockpit/389-console/package-lock.json
+NODE_MODULES_PATH = src/cockpit/389-console/
+CARGO_PATH = src/
 GIT_TAG = ${TAG}
 
 # Some sanitizers are supported only by clang
@@ -42,8 +44,8 @@ download-cargo-dependencies:
 	cargo fetch --manifest-path=./src/Cargo.toml
 	tar -czf vendor.tar.gz vendor
 
-bundle-rust:
-	python3 rpm/bundle-rust-downstream.py ./src/Cargo.lock $(DS_SPECFILE) ./vendor
+bundle-rust-npm:
+	python3 rpm/bundle-rust-npm.py $(CARGO_PATH) $(NODE_MODULES_PATH) $(DS_SPECFILE) --backup-specfile
 
 install-node-modules:
 ifeq ($(COCKPIT_ON), 1)

rpm/bundle-rust-downstream.py → rpm/bundle-rust-npm.py

@@ -1,7 +1,6 @@
 #!/usr/bin/python3
-
 # --- BEGIN COPYRIGHT BLOCK ---
-# Copyright (C) 2021 Red Hat, Inc.
+# Copyright (C) 2024 Red Hat, Inc.
 # All rights reserved.
 #
 # License: GPL (version 3 or any later version).
@@ -12,35 +11,41 @@
 
 import os
 import sys
+import subprocess
 import time
 import signal
 import argparse
 import argcomplete
 import shutil
-import toml
+import json
+import re
+from typing import List, Dict, Tuple, Set
 from lib389.cli_base import setup_script_logger
-from rust2rpm import licensing
+from lib389.utils import ensure_list_str, ensure_str
 
 SPECFILE_COMMENT_LINE = 'Bundled cargo crates list'
 START_LINE = f"##### {SPECFILE_COMMENT_LINE} - START #####\n"
 END_LINE = f"##### {SPECFILE_COMMENT_LINE} - END #####\n"
 
+IGNORED_RUST_PACKAGES: Set[str] = {"librslapd", "librnsslapd", "slapd", "slapi_r_plugin", "entryuuid", "entryuuid_syntax", "pwdchan"}
+IGNORED_NPM_PACKAGES: Set[str] = {"389-console"}
+PACKAGE_REGEX = re.compile(r"(.*)@(.*)")
 
 parser = argparse.ArgumentParser(
     formatter_class=argparse.RawDescriptionHelpFormatter,
-    description="""Add 'Provides: bundled(crate(foo)) = version' to a Fedora based specfile. 
-Additionally, add a helper comment with a comulated License metainfo which is based on Cargo.lock file content.""")
+    description="""Add 'Provides: bundled(crate(foo)) = version' 'Provides: bundled(npm(bar)) = version' to a Fedora based specfile.
+Additionally, add a helper comment with a comulated License metainfo which is based on Cargo.lock and Package-lock.json files content.""")
 
 parser.add_argument('-v', '--verbose',
                     help="Display verbose operation tracing during command execution",
                     action='store_true', default=False)
 
-parser.add_argument('cargo_lock_file',
-                    help="The path to Cargo.lock file.")
+parser.add_argument('cargo_path',
+                    help="The path to the directory with Cargo.lock file.")
+parser.add_argument('npm_path',
+                    help="The path to the directory with Package-lock.json file.")
 parser.add_argument('spec_file',
                     help="The path to spec file that will be modified.")
-parser.add_argument('vendor_dir',
-                    help="The path to the vendor directory file that will be modified.")
 parser.add_argument('--backup-specfile',
                     help="Make a backup of the downstream specfile.",
                     action='store_true', default=False)
@@ -52,59 +57,79 @@ def signal_handler(signal, frame):
     sys.exit(0)
 
 
-def get_license_list(vendor_dir):
-    license_list = list()
-    for root, _, files in os.walk(vendor_dir):
-        for file in files:
-            name = os.path.join(root, file)
-            if os.path.isfile(name) and "Cargo.toml" in name:
-                with open(name, "r") as file:
-                    contents = file.read()
-                data = toml.loads(contents)
-                license, warning = licensing.translate_license_fedora(data["package"]["license"])
-
-                # Normalise
-                license = license.replace("/", " or ").replace(" / ", " or ")
-                license = license.replace("Apache-2.0", "ASL 2.0")
-                license = license.replace("WITH LLVM-exception", "with exceptions")
-                if "or" in license or "and" in license:
-                    license = f"({license})"
-                if license == "(MIT or ASL 2.0)":
-                    license = "(ASL 2.0 or MIT)"
-
-                if license not in license_list:
-                    if warning is not None:
-                        # Ignore known warnings
-                        if not warning.endswith("LLVM-exception!") and \
-                        not warning.endswith("MIT/Apache-2.0!"):
-                            print(f"{license}: {warning}")
-                    license_list.append(license)
-    return " and ".join(license_list)
-
-
-def backup_specfile(spec_file):
+def backup_specfile(spec_file: str):
     time_now = time.strftime("%Y%m%d_%H%M%S")
     log.info(f"Backing up file {spec_file} to {spec_file}.{time_now}")
     shutil.copy2(spec_file, f"{spec_file}.{time_now}")
 
 
-def replace_license(spec_file, license_string):
+def run_cmd(cmd):
+    result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    args = ' '.join(ensure_list_str(result.args))
+    stdout = ensure_str(result.stdout)
+    stderr = ensure_str(result.stderr)
+    log.debug(f"CMD: {args} returned {result.returncode} STDOUT: {stdout} STDERR: {stderr}")
+    return stdout
+
+
+def process_rust_crates(output: str) -> Dict[str, Tuple[str, str]]:
+    crates = json.loads(output)
+    return {crate['name']: (enclose_if_contains_or(crate['license']), crate['version'])
+            for crate in crates if crate['name'] not in IGNORED_RUST_PACKAGES}
+
+
+def process_npm_packages(output: str) -> Dict[str, Tuple[str, str]]:
+    packages = json.loads(output)
+    processed_packages = {}
+    for package, data in packages.items():
+        package_name, package_version = PACKAGE_REGEX.match(package).groups()
+        if package_name not in IGNORED_NPM_PACKAGES:
+            npm_license = process_npm_license(data['licenses'])
+            processed_packages[package_name] = (npm_license, package_version)
+
+    return processed_packages
+
+
+def process_npm_license(license_data) -> str:
+    npm_license = license_data if isinstance(license_data, str) else ' OR '.join(license_data)
+    return enclose_if_contains_or(npm_license)
+
+
+def enclose_if_contains_or(license_str: str) -> str:
+    """Enclose the license string in parentheses if it contains 'OR'."""
+    return f"({license_str})" if 'OR' in license_str and not license_str.startswith('(') else license_str
+
+
+def build_provides_lines(rust_crates: Dict[str, Tuple[str, str]], npm_packages: Dict[str, Tuple[str, str]]) -> list[str]:
+    provides_lines = [f"Provides:  bundled(crate({crate})) = {version.replace('-', '_')}\n"
+                      for crate, (_, version) in rust_crates.items()]
+    provides_lines += [f"Provides:  bundled(npm({package})) = {version.replace('-', '_')}\n"
+                       for package, (_, version) in npm_packages.items()]
+    return provides_lines
+
+
+def create_license_line(rust_crates: Dict[str, Tuple[str, str]], npm_packages: Dict[str, Tuple[str, str]]) -> str:
+    licenses = {license for _, (license, _) in {**rust_crates, **npm_packages}.items() if license}
+    return " AND ".join(sorted(licenses))
+
+
+def replace_license(spec_file: str, license_string: str):
     result = []
     with open(spec_file, "r") as file:
         contents = file.readlines()
         for line in contents:
             if line.startswith("License: "):
                 result.append("# IMPORTANT - Check if it looks right. Additionally, "
                               "compare with the original line. Then, remove this comment and # FIX ME - part.\n")
-                result.append(f"# FIX ME - License:          GPLv3+ and {license_string}\n")
+                result.append(f"# FIX ME - License:          GPL-3.0-or-later AND {license_string}\n")
             else:
                 result.append(line)
     with open(spec_file, "w") as file:
         file.writelines(result)
     log.info(f"Licenses are successfully updated - {spec_file}")
 
 
-def clean_specfile(spec_file):
+def clean_specfile(spec_file: str) -> bool:
     result = []
     remove_lines = False
     cleaned = False
@@ -130,15 +155,7 @@ def clean_specfile(spec_file):
     return cleaned
 
 
-def write_provides_bundled_crate(cargo_lock_file, spec_file, cleaned):
-    # Generate 'Provides' out of cargo_lock_file
-    with open(cargo_lock_file, "r") as file:
-        contents = file.read()
-    data = toml.loads(contents)
-    provides_lines = []
-    for package in data["package"]:
-        provides_lines.append(f"Provides:  bundled(crate({package['name']})) = {package['version'].replace('-', '_')}\n")
-
+def write_provides_bundled(provides_lines: List[str], spec_file: str, cleaned: bool):
     # Find a line index where 'Provides' ends
     with open(spec_file, "r") as file:
         spec_file_lines = file.readlines()
@@ -173,24 +190,37 @@ def write_provides_bundled_crate(cargo_lock_file, spec_file, cleaned):
 
 if __name__ == '__main__':
     args = parser.parse_args()
-    log = setup_script_logger('bundle-rust-downstream', args.verbose)
+    log = setup_script_logger('bundle-rust-npm', args.verbose)
 
-    log.debug("389-ds-base Rust Crates to Bundled Downstream Specfile tool")
+    log.debug("389-ds-base Rust Crates and Node Modules to Bundled Downstream Specfile tool")
     log.debug(f"Called with: {args}")
 
-    if not os.path.exists(args.spec_file):
-        log.info(f"File doesn't exists: {args.spec_file}")
-        sys.exit(1)
-    if not os.path.exists(args.cargo_lock_file):
-        log.info(f"File doesn't exists: {args.cargo_lock_file}")
-        sys.exit(1)
-
     if args.backup_specfile:
         backup_specfile(args.spec_file)
 
+    if not os.path.isdir(args.cargo_path):
+        log.error(f"Path {args.cargo_path} does not exist or is not a directory")
+        exit(1)
+    if not os.path.isdir(args.npm_path):
+        log.error(f"Path {args.npm_path} does not exist or is not a directory")
+        exit(1)
+
     cleaned = clean_specfile(args.spec_file)
-    write_provides_bundled_crate(args.cargo_lock_file, args.spec_file, cleaned)
-    license_string = get_license_list(args.vendor_dir)
+
+    rust_output = run_cmd(["cargo", "license", "--json", "--current-dir", args.cargo_path])
+    npm_output = run_cmd(["license-checker", "--json", "--start", args.npm_path])
+
+    if rust_output is None or npm_output is None:
+        log.error("Failed to process dependencies. Ensure cargo-license and license-checker are installed and accessible.")
+        exit(1)
+
+    rust_crates = process_rust_crates(rust_output)
+    npm_packages = process_npm_packages(npm_output)
+    provides_lines = build_provides_lines(rust_crates, npm_packages)
+
+    write_provides_bundled(provides_lines, args.spec_file, cleaned)
+
+    license_string = create_license_line(rust_crates, npm_packages)
     replace_license(args.spec_file, license_string)
     log.info(f"Specfile {args.spec_file} is successfully modified! Please:\n"
               "1. Open the specfile with your editor of choice\n"