NPM CARGO LICENSES

rpm.mk

@@ -43,7 +43,7 @@ download-cargo-dependencies:
 	tar -czf vendor.tar.gz vendor
 
 bundle-rust:
-	python3 rpm/bundle-rust-downstream.py ./src/Cargo.lock $(DS_SPECFILE) ./vendor
+	python3 rpm/bundle-rust-downstream.py ./src ./src/cockpit/389-console $(DS_SPECFILE)
 
 install-node-modules:
 ifeq ($(COCKPIT_ON), 1)

rpm/bundle-rust-downstream.py

@@ -1,7 +1,6 @@
 #!/usr/bin/python3
-
 # --- BEGIN COPYRIGHT BLOCK ---
-# Copyright (C) 2021 Red Hat, Inc.
+# Copyright (C) 2024 Red Hat, Inc.
 # All rights reserved.
 #
 # License: GPL (version 3 or any later version).
@@ -12,35 +11,41 @@
 
 import os
 import sys
+import subprocess
 import time
 import signal
 import argparse
 import argcomplete
 import shutil
 import toml
+import json
+import re
+from typing import Dict, Tuple, Optional, Set
 from lib389.cli_base import setup_script_logger
-from rust2rpm import licensing
 
 SPECFILE_COMMENT_LINE = 'Bundled cargo crates list'
 START_LINE = f"##### {SPECFILE_COMMENT_LINE} - START #####\n"
 END_LINE = f"##### {SPECFILE_COMMENT_LINE} - END #####\n"
 
+IGNORED_RUST_PACKAGES: Set[str] = {"librslapd", "librnsslapd", "slapd", "slapi_r_plugin", "entryuuid", "entryuuid_syntax", "pwdchan"}
+IGNORED_NPM_PACKAGES: Set[str] = {"389-console"}
+PACKAGE_REGEX = re.compile(r"(.*)@(.*)")
 
 parser = argparse.ArgumentParser(
     formatter_class=argparse.RawDescriptionHelpFormatter,
-    description="""Add 'Provides: bundled(crate(foo)) = version' to a Fedora based specfile. 
-Additionally, add a helper comment with a comulated License metainfo which is based on Cargo.lock file content.""")
+    description="""Add 'Provides: bundled(crate(foo)) = version' 'Provides: bundled(npm(bar)) = version' to a Fedora based specfile.
+Additionally, add a helper comment with a comulated License metainfo which is based on Cargo.lock and Package-lock.json files content.""")
 
 parser.add_argument('-v', '--verbose',
                     help="Display verbose operation tracing during command execution",
                     action='store_true', default=False)
 
-parser.add_argument('cargo_lock_file',
-                    help="The path to Cargo.lock file.")
+parser.add_argument('cargo_path',
+                    help="The path to the directory with Cargo.lock file.")
+parser.add_argument('npm_path',
+                    help="The path to the directory with Package-lock.json file.")
 parser.add_argument('spec_file',
                     help="The path to spec file that will be modified.")
-parser.add_argument('vendor_dir',
-                    help="The path to the vendor directory file that will be modified.")
 parser.add_argument('--backup-specfile',
                     help="Make a backup of the downstream specfile.",
                     action='store_true', default=False)
@@ -52,43 +57,61 @@ def signal_handler(signal, frame):
     sys.exit(0)
 
 
-def get_license_list(vendor_dir):
-    license_list = list()
-    for root, _, files in os.walk(vendor_dir):
-        for file in files:
-            name = os.path.join(root, file)
-            if os.path.isfile(name) and "Cargo.toml" in name:
-                with open(name, "r") as file:
-                    contents = file.read()
-                data = toml.loads(contents)
-                license, warning = licensing.translate_license_fedora(data["package"]["license"])
-
-                # Normalise
-                license = license.replace("/", " or ").replace(" / ", " or ")
-                license = license.replace("Apache-2.0", "ASL 2.0")
-                license = license.replace("WITH LLVM-exception", "with exceptions")
-                if "or" in license or "and" in license:
-                    license = f"({license})"
-                if license == "(MIT or ASL 2.0)":
-                    license = "(ASL 2.0 or MIT)"
-
-                if license not in license_list:
-                    if warning is not None:
-                        # Ignore known warnings
-                        if not warning.endswith("LLVM-exception!") and \
-                        not warning.endswith("MIT/Apache-2.0!"):
-                            print(f"{license}: {warning}")
-                    license_list.append(license)
-    return " and ".join(license_list)
-
-
-def backup_specfile(spec_file):
+def backup_specfile(spec_file: str):
     time_now = time.strftime("%Y%m%d_%H%M%S")
     log.info(f"Backing up file {spec_file} to {spec_file}.{time_now}")
     shutil.copy2(spec_file, f"{spec_file}.{time_now}")
 
 
-def replace_license(spec_file, license_string):
+def run_cmd(cmd):
+    result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    args = ' '.join(ensure_list_str(result.args))
+    stdout = ensure_str(result.stdout)
+    stderr = ensure_str(result.stderr)
+    log.debug(f"CMD: {args} returned {result.returncode} STDOUT: {stdout} STDERR: {stderr}")
+    return stdout
+
+
+def process_rust_crates(output: str) -> Dict[str, Tuple[str, str]]:
+    crates = json.loads(output)
+    return {crate['name']: (enclose_if_contains_or(crate['license']), crate['version'])
+            for crate in crates if crate['name'] not in IGNORED_RUST_PACKAGES}
+
+
+def process_npm_packages(output: str) -> Dict[str, Tuple[str, str]]:
+    packages = json.loads(output)
+    processed_packages = {}
+    for package, data in packages.items():
+        package_name, package_version = PACKAGE_REGEX.match(package).groups()
+        if package_name not in IGNORED_NPM_PACKAGES:
+            npm_license = process_npm_license(data['licenses'])
+            processed_packages[package_name] = (npm_license, package_version)
+
+    return processed_packages
+
+
+def process_npm_license(license_data) -> str:
+    npm_license = license_data if isinstance(license_data, str) else ' OR '.join(license_data)
+    return enclose_if_contains_or(npm_license)
+
+
+def enclose_if_contains_or(license_str: str) -> str:
+    """Enclose the license string in parentheses if it contains 'OR'."""
+    return f"({license_str})" if 'OR' in license_str and not license_str.startswith('(') else license_str
+
+
+def build_provides_lines(rust_crates: Dict[str, Tuple[str, str]], npm_packages: Dict[str, Tuple[str, str]]) -> list[str]:
+    provides_lines = [f"Provides:  bundled(crate({crate}) = {version})\n" for crate, (_, version) in rust_crates.items()]
+    provides_lines += [f"Provides:  bundled(npm({package}) = {version})\n" for package, (_, version) in npm_packages.items()]
+    return provides_lines
+
+
+def create_license_line(rust_crates: Dict[str, Tuple[str, str]], npm_packages: Dict[str, Tuple[str, str]]) -> str:
+    licenses = {license for _, (license, _) in {**rust_crates, **npm_packages}.items() if license}
+    return " AND ".join(sorted(licenses))
+
+
+def replace_license(spec_file: str, license_string: str):
     result = []
     with open(spec_file, "r") as file:
         contents = file.readlines()
@@ -104,7 +127,7 @@ def replace_license(spec_file, license_string):
     log.info(f"Licenses are successfully updated - {spec_file}")
 
 
-def clean_specfile(spec_file):
+def clean_specfile(spec_file: str) -> bool:
     result = []
     remove_lines = False
     cleaned = False
@@ -130,15 +153,7 @@ def clean_specfile(spec_file):
     return cleaned
 
 
-def write_provides_bundled_crate(cargo_lock_file, spec_file, cleaned):
-    # Generate 'Provides' out of cargo_lock_file
-    with open(cargo_lock_file, "r") as file:
-        contents = file.read()
-    data = toml.loads(contents)
-    provides_lines = []
-    for package in data["package"]:
-        provides_lines.append(f"Provides:  bundled(crate({package['name']})) = {package['version'].replace('-', '_')}\n")
-
+def write_provides_bundled(provides_lines: List[str], spec_file: str, cleaned: bool):
     # Find a line index where 'Provides' ends
     with open(spec_file, "r") as file:
         spec_file_lines = file.readlines()
@@ -173,24 +188,32 @@ def write_provides_bundled_crate(cargo_lock_file, spec_file, cleaned):
 
 if __name__ == '__main__':
     args = parser.parse_args()
-    log = setup_script_logger('bundle-rust-downstream', args.verbose)
+    log = setup_script_logger('bundle-rust', args.verbose)
 
     log.debug("389-ds-base Rust Crates to Bundled Downstream Specfile tool")
     log.debug(f"Called with: {args}")
 
-    if not os.path.exists(args.spec_file):
-        log.info(f"File doesn't exists: {args.spec_file}")
-        sys.exit(1)
-    if not os.path.exists(args.cargo_lock_file):
-        log.info(f"File doesn't exists: {args.cargo_lock_file}")
-        sys.exit(1)
-
     if args.backup_specfile:
         backup_specfile(args.spec_file)
 
     cleaned = clean_specfile(args.spec_file)
-    write_provides_bundled_crate(args.cargo_lock_file, args.spec_file, cleaned)
-    license_string = get_license_list(args.vendor_dir)
+
+    # TODO: Change paths and add their validation
+    rust_output = run_cmd(["cargo", "license", "--json", "--current-dir", "./src"])
+    npm_output = run_cmd(["license-checker", "--json", "--start", "src/cockpit/389-console"])
+
+    if rust_output is None or npm_output is None:
+        print("Failed to process dependencies. Ensure cargo-license and license-checker are installed and accessible.")
+        exit(1)
+
+    rust_crates = process_rust_crates(rust_output)
+    npm_packages = process_npm_packages(npm_output)
+    provides_lines = build_provides_lines(rust_crates, npm_packages)
+
+    write_provides_bundled(provides_lines, args.spec_file, cleaned)
+
+    # Simplify?
+    license_string = create_license_line(rust_crates, npm_packages)
     replace_license(args.spec_file, license_string)
     log.info(f"Specfile {args.spec_file} is successfully modified! Please:\n"
               "1. Open the specfile with your editor of choice\n"