- {matchesInfo.matches.map((match, idx) => (
+ {matches.map((match, idx) => (
{
/>
- {matchesInfo.licenseType === LicenseTypes.DETECTION &&
- (match as LicenseDetectionMatch)?.license_expression_spdx && (
-
- | License Expression SPDX |
-
-
- |
-
- )}
+ {match.license_expression_spdx && (
+
+ | License Expression SPDX |
+
+
+ |
+
+ )}
{showLIcenseText && (
| Matched Text |
diff --git a/src/services/importedJsonTypes.ts b/src/services/importedJsonTypes.ts
index f12df517..2317c565 100644
--- a/src/services/importedJsonTypes.ts
+++ b/src/services/importedJsonTypes.ts
@@ -18,6 +18,7 @@ export interface LicenseMatch {
match_coverage: number;
matcher: string;
license_expression: string;
+ license_expression_spdx?: string;
rule_identifier: string;
rule_relevance: number;
rule_url: string;
@@ -25,11 +26,9 @@ export interface LicenseMatch {
// Parser-added fields
path?: string;
license_expression_keys?: LicenseExpressionKey[];
-}
-export interface LicenseDetectionMatch extends LicenseMatch {
- license_expression_spdx?: string;
license_expression_spdx_keys?: LicenseExpressionSpdxKey[];
}
+export type LicenseDetectionMatch = LicenseMatch;
export type LicenseClueMatch = LicenseMatch;
export interface LicenseFileRegion {
@@ -57,6 +56,7 @@ export interface LicenseClue {
fileClueIdx: number;
matches?: LicenseClueMatch[];
file_regions?: LicenseFileRegion[];
+ license_expression_spdx?: string;
}
export interface TopLevelLicenseDetection {
identifier: string;
diff --git a/src/services/models/databaseUtils.ts b/src/services/models/databaseUtils.ts
index df1e01b5..5adb5fce 100644
--- a/src/services/models/databaseUtils.ts
+++ b/src/services/models/databaseUtils.ts
@@ -37,76 +37,3 @@ export function parentPath(path: string) {
const splits = path.split("/");
return splits.length === 1 ? "#" : splits.slice(0, -1).join("/");
}
-
-export const LICENSE_EXPRESSIONS_CONJUNCTIONS = ["AND", "OR", "WITH"];
-
-// @TODO - Needs more testing
-export const parseSubExpressions = (expression: string) => {
- if (!expression || !expression.length) return [];
- const tokens = expression.split(/( |\(|\))/);
- const result = [];
- let currSubExpression = "";
- let popTokens = 0;
- for (const token of tokens) {
- if (token === "(") {
- if (popTokens) currSubExpression += "(";
- popTokens++;
- } else if (token === ")") {
- popTokens--;
- if (popTokens) {
- currSubExpression += ")";
- } else {
- result.push(currSubExpression);
- currSubExpression = "";
- }
- } else {
- if (popTokens) currSubExpression += token;
- else {
- if (token.trim().length) result.push(token);
- }
- }
- }
-
- return result.filter(
- (subExpression) =>
- subExpression.trim().length &&
- !LICENSE_EXPRESSIONS_CONJUNCTIONS.includes(subExpression.trim())
- );
-};
-
-// Test parseSubExpressions
-// [
-// 'apache-2.0 AND (mit AND json) AND (apache-2.0 AND bsd-simplified AND bsd-new AND cc0-1.0 AND cddl-1.0) AND (cddl-1.0 AND bsd-new) AND (bsd-new AND epl-2.0 AND elastic-license-v2) AND (bsd-new AND json AND lgpl-2.0 AND mit AND gpl-2.0 AND universal-foss-exception-1.0)',
-// 'apache-2.0 AND cc0-1.0 AND mit AND (lgpl-2.1 AND bsd-new AND unknown-license-reference) AND bsd-new AND (mit AND apache-2.0 AND bsd-new) AND (ofl-1.1 AND mit AND cc-by-3.0) AND (mit AND cc0-1.0) AND (mit AND apache-2.0) AND (mit AND gpl-3.0) AND ((mit OR gpl-3.0) AND mit AND gpl-3.0) AND ofl-1.1 AND (ofl-1.1 AND proprietary-license) AND isc AND (bsd-new AND bsd-simplified) AND unknown AND (apache-2.0 AND isc) AND (apache-2.0 AND mit) AND ((gpl-2.0 WITH font-exception-gpl OR ofl-1.1) AND apache-2.0) AND (apache-2.0 AND bsd-new AND bsd-simplified AND cc-by-3.0 AND cc0-1.0 AND gpl-3.0 AND isc AND lgpl-2.1 AND mit AND ofl-1.1 AND unknown-license-reference AND other-copyleft AND other-permissive AND unknown)',
-// '(gpl-2.0 WITH font-exception-gpl OR ofl-1.1) AND apache-2.0',
-// 'apache OR apache-2.0',
-// '(mit OR gpl-3.0) AND mit AND gpl-3.0',
-// 'apache-2.0 AND (mit AND json)'
-// ].map(key => {
-// console.log(key, parseSubExpressions(key), "\n");
-// })
-
-export function parseTokensFromExpression(expression: string) {
- if (!expression) expression = "";
- const tokens = expression.split(/( |\(|\))/);
- return tokens;
-}
-
-export function parseTokenKeysFromExpression(expression: string) {
- if (!expression) expression = "";
- const AVOID_KEYWORDS = new Set(["WITH", "OR", "AND", "(", ")"]);
- const tokens = parseTokensFromExpression(expression);
- return tokens.filter(
- (token) =>
- token.trim().length && token.length && !AVOID_KEYWORDS.has(token.trim())
- );
-}
-export function filterSpdxKeys(keys: string[]) {
- const ignoredPrefixes = ["License-scancode-", "LicenseRef-scancode-"];
- return keys.filter((key) => {
- for (const prefix of ignoredPrefixes) {
- if (key.includes(prefix)) return false;
- }
- return true;
- });
-}
diff --git a/src/services/workbenchDB.ts b/src/services/workbenchDB.ts
index 72a90cf4..3b04f069 100644
--- a/src/services/workbenchDB.ts
+++ b/src/services/workbenchDB.ts
@@ -32,17 +32,18 @@ import { UNKNOWN_EXPRESSION, UNKNOWN_EXPRESSION_SPDX } from "../constants/data";
import { logDependenciesOnError } from "../utils/ensureRendererDeps";
import { DebugLogger } from "../utils/logger";
import { DatabaseStructure, newDatabase } from "./models/database";
+import { parentPath } from "./models/databaseUtils";
import {
filterSpdxKeys,
- parentPath,
parseSubExpressions,
parseTokenKeysFromExpression,
-} from "./models/databaseUtils";
+} from "../utils/expressions";
import { FileAttributes } from "./models/file";
import { FlatFileAttributes, flattenFile } from "./models/flatFile";
import {
LicenseClue,
LicenseExpressionKey,
+ LicenseExpressionSpdxKey,
RawTopLevelTodo,
Resource,
ResourceLicenseDetection,
@@ -616,6 +617,74 @@ export class WorkbenchDB {
return parsedHeader;
}
+ _getLicenseExpressionKeys(
+ license_expression: string,
+ license_expression_spdx: string,
+ license_references_map: Map,
+ license_references_spdx_map: Map
+ ) {
+ const parsedLicenseKeys = parseSubExpressions(license_expression);
+ const parsedSpdxLicenseKeys = parseSubExpressions(license_expression_spdx);
+
+ const license_expression_keys: LicenseExpressionKey[] = [];
+ const license_expression_spdx_keys: LicenseExpressionSpdxKey[] = [];
+
+ parsedLicenseKeys.forEach((key) => {
+ const license_reference = license_references_map.get(key);
+
+ license_expression_keys.push({
+ key,
+ licensedb_url: license_reference?.licensedb_url || null,
+ scancode_url: license_reference?.scancode_url || null,
+ });
+ });
+ parsedSpdxLicenseKeys.forEach((key) => {
+ const license_reference = license_references_spdx_map.get(key);
+
+ license_expression_spdx_keys.push({
+ key,
+ spdx_url: license_reference?.spdx_url || null,
+ });
+ });
+ return {
+ license_expression_keys,
+ license_expression_spdx_keys,
+ };
+ }
+
+ _resolveSpdxLicenseExpression(
+ license_expression: string,
+ license_references_map: Map
+ ) {
+ const parsedLicenseKeys = parseSubExpressions(license_expression);
+
+ // Specifically useful for license clue matches, where SPDX expression is not available
+ // Find reference using license expression (if available)
+ // And set spdx expression available in reference
+ let license_expression_spdx = license_expression;
+ parsedLicenseKeys.forEach((key) => {
+ if (license_references_map.has(key)) {
+ const license_reference = license_references_map.get(key);
+ license_expression_spdx = license_expression_spdx.replace(
+ key,
+ license_reference.spdx_license_key
+ );
+ }
+ });
+
+ return license_expression_spdx;
+
+ // // Approach 2 - Tokenise & merge (Not tested for complex expressions)
+ // const licenseExpressionTokens =
+ // parseTokensFromExpression(license_expression);
+ // const licenseExpressionSpdxTokens = licenseExpressionTokens.map((key) =>
+ // license_references_map.has(key)
+ // ? license_references_map.get(key).spdx_license_key
+ // : key
+ // );
+ // return licenseExpressionSpdxTokens.join("");
+ }
+
_parseLicenseDetections(file: Resource, TopLevelData: TopLevelDataFormat) {
if (!file) return;
@@ -629,11 +698,11 @@ export class WorkbenchDB {
// upto v32.0.0rc2
const for_license_detections: string[] = file.for_license_detections || [];
- function addLicenseDetection(
+ const addLicenseDetection = (
detection: ResourceLicenseDetection,
detectionIdx: number,
from_package: string = null
- ) {
+ ) => {
const detectionIdentifier =
detection.identifier || for_license_detections[detectionIdx];
@@ -695,14 +764,6 @@ export class WorkbenchDB {
const { license_references_map, license_references_spdx_map } =
TopLevelData;
- if (!match.license_expression_keys?.length)
- match.license_expression_keys = [];
- if (
- !match.license_expression_spdx_keys ||
- !match.license_expression_spdx_keys.length
- )
- match.license_expression_spdx_keys = [];
-
// SPDX not available in matches, so find corresponding spdx license expression
match.license_expression_spdx =
detectionSpdxLicenseExpressionComponents[
@@ -727,50 +788,32 @@ export class WorkbenchDB {
match.license_expression_spdx = UNKNOWN_EXPRESSION_SPDX;
}
- // Specifically useful for license clue matches, where SPDX expression is not available
- // Find reference using license expression (if available)
- // And set spdx expression available in reference
- if (
- !match.license_expression_spdx &&
- TopLevelData.license_references_map.has(match.license_expression)
- ) {
- match.license_expression_spdx =
- TopLevelData.license_references_map.get(
- match.license_expression
- ).spdx_license_key;
+ // When SPDX expression is not available,
+ // construct it using keys in license expression
+ if (!match.license_expression_spdx) {
+ match.license_expression_spdx = this._resolveSpdxLicenseExpression(
+ match.license_expression,
+ license_references_map
+ );
}
- const parsedLicenseKeys = parseSubExpressions(
- match.license_expression
- );
- const parsedSpdxLicenseKeys = parseSubExpressions(
- match.license_expression_spdx
- );
-
- parsedLicenseKeys.forEach((key) => {
- const license_reference = license_references_map.get(key);
+ const { license_expression_keys, license_expression_spdx_keys } =
+ this._getLicenseExpressionKeys(
+ match.license_expression,
+ match.license_expression_spdx,
+ license_references_map,
+ license_references_spdx_map
+ );
- match.license_expression_keys.push({
- key,
- licensedb_url: license_reference?.licensedb_url || null,
- scancode_url: license_reference?.scancode_url || null,
- });
- });
- parsedSpdxLicenseKeys.forEach((key) => {
- const license_reference = license_references_spdx_map.get(key);
-
- match.license_expression_spdx_keys.push({
- key,
- spdx_url: license_reference?.spdx_url || null,
- });
- });
+ match.license_expression_keys = license_expression_keys;
+ match.license_expression_spdx_keys = license_expression_spdx_keys;
match.path = file.path;
targetLicenseDetection.matches.push(match);
});
}
delete detection.matches; // Not required further, hence removing to reduce sqlite size
- }
+ };
(file?.license_detections || []).forEach((detection, idx) =>
addLicenseDetection(detection, idx)
@@ -785,21 +828,24 @@ export class WorkbenchDB {
_parseLicenseClues(file: Resource, TopLevelData: TopLevelDataFormat) {
file.license_clues?.forEach((license_clue, clue_idx) => {
- const parsedLicenseKeys = parseSubExpressions(
- license_clue.license_expression
- );
-
- const license_expression_keys: LicenseExpressionKey[] = [];
- parsedLicenseKeys.forEach((key) => {
- const license_reference = TopLevelData.license_references_map.get(key);
- if (!license_reference) return [];
+ const { license_references_map, license_references_spdx_map } =
+ TopLevelData;
+
+ if (!license_clue.license_expression_spdx) {
+ license_clue.license_expression_spdx =
+ this._resolveSpdxLicenseExpression(
+ license_clue.license_expression,
+ license_references_map
+ );
+ }
- license_expression_keys.push({
- key,
- licensedb_url: license_reference.licensedb_url || null,
- scancode_url: license_reference.scancode_url || null,
- });
- });
+ const { license_expression_keys, license_expression_spdx_keys } =
+ this._getLicenseExpressionKeys(
+ license_clue.license_expression,
+ license_clue.license_expression_spdx,
+ license_references_map,
+ license_references_spdx_map
+ );
license_clue.fileId = file.id;
license_clue.filePath = file.path;
@@ -815,11 +861,13 @@ export class WorkbenchDB {
match_coverage: license_clue.match_coverage,
matcher: license_clue.matcher,
license_expression: license_clue.license_expression,
+ license_expression_keys,
+ license_expression_spdx: license_clue.license_expression_spdx,
+ license_expression_spdx_keys,
rule_identifier: license_clue.rule_identifier,
rule_relevance: license_clue.rule_relevance,
rule_url: license_clue.rule_url,
path: file.path,
- license_expression_keys,
},
];
license_clue.file_regions = [
diff --git a/src/utils/expressions.ts b/src/utils/expressions.ts
new file mode 100644
index 00000000..d45c8551
--- /dev/null
+++ b/src/utils/expressions.ts
@@ -0,0 +1,60 @@
+export const LICENSE_EXPRESSIONS_CONJUNCTIONS = ["AND", "OR", "WITH"];
+
+export const parseSubExpressions = (expression: string) => {
+ if (!expression || !expression.length) return [];
+ const tokens = expression.split(/( |\(|\))/);
+ const result = [];
+ let currSubExpression = "";
+ let popTokens = 0;
+ for (const token of tokens) {
+ if (token === "(") {
+ if (popTokens) currSubExpression += "(";
+ popTokens++;
+ } else if (token === ")") {
+ popTokens--;
+ if (popTokens) {
+ currSubExpression += ")";
+ } else {
+ result.push(currSubExpression);
+ currSubExpression = "";
+ }
+ } else {
+ if (popTokens) currSubExpression += token;
+ else {
+ if (token.trim().length) result.push(token);
+ }
+ }
+ }
+
+ return result.filter(
+ (subExpression) =>
+ subExpression.trim().length &&
+ !LICENSE_EXPRESSIONS_CONJUNCTIONS.includes(subExpression.trim())
+ );
+};
+
+export function parseTokensFromExpression(expression: string) {
+ if (!expression) expression = "";
+ const tokens = expression.split(/( |\(|\))/);
+ return tokens;
+}
+
+export function parseTokenKeysFromExpression(expression: string) {
+ if (!expression) expression = "";
+ const AVOID_KEYWORDS = new Set(["WITH", "OR", "AND", "(", ")"]);
+ const tokens = parseTokensFromExpression(expression);
+ return tokens.filter(
+ (token) =>
+ token.trim().length && token.length && !AVOID_KEYWORDS.has(token.trim())
+ );
+}
+
+export function filterSpdxKeys(keys: string[]) {
+ const ignoredPrefixes = ["License-scancode-", "LicenseRef-scancode-"];
+ return keys.filter((key) => {
+ for (const prefix of ignoredPrefixes) {
+ if (key.includes(prefix)) return false;
+ }
+ return true;
+ });
+}
diff --git a/tests/expressions.test.data.ts b/tests/expressions.test.data.ts
new file mode 100644
index 00000000..cc4f178c
--- /dev/null
+++ b/tests/expressions.test.data.ts
@@ -0,0 +1,183 @@
+export const ParseExpressionTokensAndKeysSamples = [
+ {
+ license_expression: "apache OR apache-2.0",
+ tokens: ["apache", " ", "OR", " ", "apache-2.0"],
+ keys: ["apache", "apache-2.0"],
+ },
+ {
+ license_expression: "(mit OR gpl-3.0) AND mit AND gpl-3.0",
+ tokens: [
+ "",
+ "(",
+ "mit",
+ " ",
+ "OR",
+ " ",
+ "gpl-3.0",
+ ")",
+ "",
+ " ",
+ "AND",
+ " ",
+ "mit",
+ " ",
+ "AND",
+ " ",
+ "gpl-3.0",
+ ],
+ keys: ["mit", "gpl-3.0", "mit", "gpl-3.0"],
+ },
+ {
+ license_expression:
+ "(gpl-2.0 WITH font-exception-gpl OR ofl-1.1) AND mit AND apache-2.0",
+ tokens: [
+ "",
+ "(",
+ "gpl-2.0",
+ " ",
+ "WITH",
+ " ",
+ "font-exception-gpl",
+ " ",
+ "OR",
+ " ",
+ "ofl-1.1",
+ ")",
+ "",
+ " ",
+ "AND",
+ " ",
+ "mit",
+ " ",
+ "AND",
+ " ",
+ "apache-2.0",
+ ],
+ keys: ["gpl-2.0", "font-exception-gpl", "ofl-1.1", "mit", "apache-2.0"],
+ },
+ {
+ license_expression:
+ "(gpl-2.0 WITH (font-exception-gpl OR ofl-1.1) AND bsd-new) OR (mit AND apache-2.0)",
+ tokens: [
+ "",
+ "(",
+ "gpl-2.0",
+ " ",
+ "WITH",
+ " ",
+ "",
+ "(",
+ "font-exception-gpl",
+ " ",
+ "OR",
+ " ",
+ "ofl-1.1",
+ ")",
+ "",
+ " ",
+ "AND",
+ " ",
+ "bsd-new",
+ ")",
+ "",
+ " ",
+ "OR",
+ " ",
+ "",
+ "(",
+ "mit",
+ " ",
+ "AND",
+ " ",
+ "apache-2.0",
+ ")",
+ "",
+ ],
+ keys: [
+ "gpl-2.0",
+ "font-exception-gpl",
+ "ofl-1.1",
+ "bsd-new",
+ "mit",
+ "apache-2.0",
+ ],
+ },
+];
+
+export const SubExpressionsSamples = [
+ {
+ license_expression:
+ "apache-2.0 AND (mit AND json) AND (apache-2.0 AND bsd-simplified AND bsd-new AND cc0-1.0 AND cddl-1.0) AND (cddl-1.0 AND bsd-new) AND (bsd-new AND epl-2.0 AND elastic-license-v2) AND (bsd-new AND json AND lgpl-2.0 AND mit AND gpl-2.0 AND universal-foss-exception-1.0)",
+ subExpressions: [
+ "apache-2.0",
+ "mit AND json",
+ "apache-2.0 AND bsd-simplified AND bsd-new AND cc0-1.0 AND cddl-1.0",
+ "cddl-1.0 AND bsd-new",
+ "bsd-new AND epl-2.0 AND elastic-license-v2",
+ "bsd-new AND json AND lgpl-2.0 AND mit AND gpl-2.0 AND universal-foss-exception-1.0",
+ ],
+ },
+ {
+ license_expression:
+ "apache-2.0 AND cc0-1.0 AND mit AND (lgpl-2.1 AND bsd-new AND unknown-license-reference) AND bsd-new AND (mit AND apache-2.0 AND bsd-new) AND (ofl-1.1 AND mit AND cc-by-3.0) AND (mit AND cc0-1.0) AND (mit AND apache-2.0) AND (mit AND gpl-3.0) AND ((mit OR gpl-3.0) AND mit AND gpl-3.0) AND ofl-1.1 AND (ofl-1.1 AND proprietary-license) AND isc AND (bsd-new AND bsd-simplified) AND unknown AND (apache-2.0 AND isc) AND (apache-2.0 AND mit) AND ((gpl-2.0 WITH font-exception-gpl OR ofl-1.1) AND apache-2.0) AND (apache-2.0 AND bsd-new AND bsd-simplified AND cc-by-3.0 AND cc0-1.0 AND gpl-3.0 AND isc AND lgpl-2.1 AND mit AND ofl-1.1 AND unknown-license-reference AND other-copyleft AND other-permissive AND unknown)",
+ subExpressions: [
+ "apache-2.0",
+ "cc0-1.0",
+ "mit",
+ "lgpl-2.1 AND bsd-new AND unknown-license-reference",
+ "bsd-new",
+ "mit AND apache-2.0 AND bsd-new",
+ "ofl-1.1 AND mit AND cc-by-3.0",
+ "mit AND cc0-1.0",
+ "mit AND apache-2.0",
+ "mit AND gpl-3.0",
+ "(mit OR gpl-3.0) AND mit AND gpl-3.0",
+ "ofl-1.1",
+ "ofl-1.1 AND proprietary-license",
+ "isc",
+ "bsd-new AND bsd-simplified",
+ "unknown",
+ "apache-2.0 AND isc",
+ "apache-2.0 AND mit",
+ "(gpl-2.0 WITH font-exception-gpl OR ofl-1.1) AND apache-2.0",
+ "apache-2.0 AND bsd-new AND bsd-simplified AND cc-by-3.0 AND cc0-1.0 AND gpl-3.0 AND isc AND lgpl-2.1 AND mit AND ofl-1.1 AND unknown-license-reference AND other-copyleft AND other-permissive AND unknown",
+ ],
+ },
+ {
+ license_expression:
+ "(gpl-2.0 WITH font-exception-gpl OR ofl-1.1) AND apache-2.0",
+ subExpressions: [
+ "gpl-2.0 WITH font-exception-gpl OR ofl-1.1",
+ "apache-2.0",
+ ],
+ },
+ {
+ license_expression: "apache OR apache-2.0",
+ subExpressions: ["apache", "apache-2.0"],
+ },
+ {
+ license_expression: "(mit OR gpl-3.0) AND mit AND gpl-3.0",
+ subExpressions: ["mit OR gpl-3.0", "mit", "gpl-3.0"],
+ },
+ {
+ license_expression: "apache-2.0 AND (mit AND json)",
+ subExpressions: ["apache-2.0", "mit AND json"],
+ },
+];
+
+export const FilterSpdxKeySamples = {
+ keys: [
+ "MIT",
+ "Apache-2.0 ",
+ "Apache-2.0 OR GPL-2.0-only",
+ "License-scancode-unknown",
+ "License-scancode-somekey",
+ "License-scancode-adrian",
+ "License-scancode-996-icu-1.0",
+ "LicenseRef-scancode-unknown",
+ "LicenseRef-scancode-somekey",
+ "LicenseRef-scancode-adrian",
+ "LicenseRef-scancode-996-icu-1.0",
+ ],
+ filtered: ["MIT", "Apache-2.0 ", "Apache-2.0 OR GPL-2.0-only"],
+};
diff --git a/tests/expressions.test.ts b/tests/expressions.test.ts
new file mode 100644
index 00000000..8f72924c
--- /dev/null
+++ b/tests/expressions.test.ts
@@ -0,0 +1,46 @@
+import assert from "assert";
+import {
+ filterSpdxKeys,
+ parseSubExpressions,
+ parseTokenKeysFromExpression,
+ parseTokensFromExpression,
+} from "../src/utils/expressions";
+import {
+ FilterSpdxKeySamples,
+ ParseExpressionTokensAndKeysSamples,
+ SubExpressionsSamples,
+} from "./expressions.test.data";
+
+export const LICENSE_EXPRESSIONS_CONJUNCTIONS = ["AND", "OR", "WITH"];
+
+describe("Parse sub expressions in a license expression", () => {
+ it.each(SubExpressionsSamples)(
+ "Parse sub expressions of $license_expression => $subExpressions.length sub expressions",
+ ({ license_expression, subExpressions }) =>
+ assert.deepEqual(parseSubExpressions(license_expression), subExpressions)
+ );
+});
+
+describe("Parse tokens in a license expression", () => {
+ it.each(ParseExpressionTokensAndKeysSamples)(
+ "Parse tokens of $license_expression => $tokens.length tokens",
+ ({ license_expression, tokens }) =>
+ assert.deepEqual(parseTokensFromExpression(license_expression), tokens)
+ );
+});
+
+describe("Parse keys in a license expression", () => {
+ it.each(ParseExpressionTokensAndKeysSamples)(
+ "Parse keys of $license_expression => $keys.length keys",
+ ({ license_expression, keys }) =>
+ assert.deepEqual(parseTokenKeysFromExpression(license_expression), keys)
+ );
+});
+
+describe("Filter scancode-prefixed SPDX keys", () => {
+ it("Must filter out custom scancode-prefixed SPDX keys", () =>
+ assert.deepEqual(
+ filterSpdxKeys(FilterSpdxKeySamples.keys),
+ FilterSpdxKeySamples.filtered
+ ));
+});
diff --git a/tests/test-scans/licenses/expectedLicenses.ts b/tests/test-scans/licenses/expectedLicenses.ts
index 0437e0ba..bf9877e4 100644
--- a/tests/test-scans/licenses/expectedLicenses.ts
+++ b/tests/test-scans/licenses/expectedLicenses.ts
@@ -20,14 +20,6 @@ export const LicenseSamples: {
jsonFileName: "withLicenses.json",
expectedLicenseClues: [
{
- id: 1,
- fileId: 5,
- filePath: "rx-lite/package.json",
- fileClueIdx: 0,
- score: 52,
- license_expression: "apache-2.0 OR gpl-2.0",
- rule_identifier: "apache-2.0_or_gpl-2.0_24.RULE",
- reviewed: false,
matches: [
{
score: 52,
@@ -37,11 +29,7 @@ export const LicenseSamples: {
match_coverage: 52,
matcher: "3-seq",
license_expression: "apache-2.0 OR gpl-2.0",
- rule_identifier: "apache-2.0_or_gpl-2.0_24.RULE",
- rule_relevance: 100,
- rule_url:
- "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/apache-2.0_or_gpl-2.0_24.RULE",
- path: "rx-lite/package.json",
+ license_expression_spdx: "Apache-2.0 OR GPL-2.0-only",
license_expression_keys: [
{
key: "apache-2.0",
@@ -58,11 +46,38 @@ export const LicenseSamples: {
"https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/gpl-2.0.LICENSE",
},
],
+ license_expression_spdx_keys: [
+ {
+ key: "Apache-2.0",
+ spdx_url: "https://spdx.org/licenses/Apache-2.0",
+ },
+ {
+ key: "GPL-2.0-only",
+ spdx_url: "https://spdx.org/licenses/GPL-2.0-only",
+ },
+ ],
+ rule_identifier: "apache-2.0_or_gpl-2.0_24.RULE",
+ rule_relevance: 100,
+ rule_url:
+ "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/apache-2.0_or_gpl-2.0_24.RULE",
+ path: "rx-lite/package.json",
},
],
file_regions: [
- { path: "rx-lite/package.json", start_line: 56, end_line: 59 },
+ {
+ path: "rx-lite/package.json",
+ start_line: 56,
+ end_line: 59,
+ },
],
+ id: 1,
+ fileId: 5,
+ filePath: "rx-lite/package.json",
+ fileClueIdx: 0,
+ score: 52,
+ license_expression: "apache-2.0 OR gpl-2.0",
+ rule_identifier: "apache-2.0_or_gpl-2.0_24.RULE",
+ reviewed: false,
},
],
expectedLicenseDetections: [
@@ -81,6 +96,7 @@ export const LicenseSamples: {
rule_relevance: 100,
rule_url:
"https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/apache-2.0_65.RULE",
+ license_expression_spdx: "Apache-2.0",
license_expression_keys: [
{
key: "apache-2.0",
@@ -96,7 +112,6 @@ export const LicenseSamples: {
spdx_url: "https://spdx.org/licenses/Apache-2.0",
},
],
- license_expression_spdx: "Apache-2.0",
path: "anglesharp.css.0.16.4/file_with_multiple_licenses.txt",
},
{
@@ -111,6 +126,7 @@ export const LicenseSamples: {
rule_relevance: 100,
rule_url:
"https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/gpl-3.0_173.RULE",
+ license_expression_spdx: "GPL-3.0-only",
license_expression_keys: [
{
key: "gpl-3.0",
@@ -124,7 +140,6 @@ export const LicenseSamples: {
spdx_url: null,
},
],
- license_expression_spdx: "GPL-3.0-only",
path: "anglesharp.css.0.16.4/file_with_multiple_licenses.txt",
},
],
@@ -160,6 +175,7 @@ export const LicenseSamples: {
rule_url:
"https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/mit_14.RULE",
matched_text: "MIT",
+ license_expression_spdx: "MIT",
license_expression_keys: [
{
key: "mit",
@@ -174,7 +190,6 @@ export const LicenseSamples: {
spdx_url: "https://spdx.org/licenses/MIT",
},
],
- license_expression_spdx: "MIT",
path: "anglesharp.css.0.16.4/AngleSharp.Css.nuspec",
},
{
@@ -190,6 +205,7 @@ export const LicenseSamples: {
rule_relevance: 100,
rule_url: null,
matched_text: "licenses.nuget.org/MIT",
+ license_expression_spdx: "MIT",
license_expression_keys: [
{
key: "mit",
@@ -204,7 +220,6 @@ export const LicenseSamples: {
spdx_url: "https://spdx.org/licenses/MIT",
},
],
- license_expression_spdx: "MIT",
path: "anglesharp.css.0.16.4/AngleSharp.Css.nuspec",
},
],
@@ -239,6 +254,7 @@ export const LicenseSamples: {
rule_relevance: 100,
rule_url: null,
matched_text: "MIT",
+ license_expression_spdx: "MIT",
license_expression_keys: [
{
key: "mit",
@@ -253,7 +269,6 @@ export const LicenseSamples: {
spdx_url: "https://spdx.org/licenses/MIT",
},
],
- license_expression_spdx: "MIT",
path: "anglesharp.css.0.16.4/AngleSharp.Css.nuspec",
},
],
@@ -290,6 +305,7 @@ export const LicenseSamples: {
"https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/package-manifest-unknown-16117ed57856733eaaf6d91ea575b186a9dab3df",
matched_text:
"license {'LegalCopyright': 'Copyright © AngleSharp, 2013-2019', 'LegalTrademarks': '', 'License': None}",
+ license_expression_spdx: "LicenseRef-scancode-unknown",
license_expression_keys: [
{
key: "unknown",
@@ -306,7 +322,6 @@ export const LicenseSamples: {
"https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/unknown.LICENSE",
},
],
- license_expression_spdx: "LicenseRef-scancode-unknown",
path: "anglesharp.css.0.16.4/AngleSharp.Css.dll",
},
],
@@ -349,6 +364,7 @@ export const LicenseSamples: {
rule_url:
"https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/apache-2.0_48.RULE",
matched_text: "Apache License, Version 2.0",
+ license_expression_spdx: "Apache-2.0",
license_expression_keys: [
{
key: "apache-2.0",
@@ -364,7 +380,6 @@ export const LicenseSamples: {
spdx_url: "https://spdx.org/licenses/Apache-2.0",
},
],
- license_expression_spdx: "Apache-2.0",
path: "rx-lite/package.json",
},
],
@@ -399,6 +414,7 @@ export const LicenseSamples: {
rule_url:
"https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/apache-2.0_20.RULE",
matched_text: "http://www.apache.org/licenses/LICENSE-2.0.html",
+ license_expression_spdx: "Apache-2.0",
license_expression_keys: [
{
key: "apache-2.0",
@@ -414,7 +430,6 @@ export const LicenseSamples: {
spdx_url: "https://spdx.org/licenses/Apache-2.0",
},
],
- license_expression_spdx: "Apache-2.0",
path: "rx-lite/package.json",
},
],
@@ -707,22 +722,19 @@ export const LicenseSamples: {
],
expectedFlatFiles: [
{
+ license_clues: [],
+ license_policy: [],
+ license_detections: [],
detected_license_expression: null,
detected_license_expression_spdx: null,
percentage_of_license_text: 0,
- license_policy: [],
- license_clues: [],
- license_detections: [],
},
{
- detected_license_expression: "apache-2.0 AND gpl-3.0",
- detected_license_expression_spdx: "Apache-2.0 AND GPL-3.0-only",
- percentage_of_license_text: 100,
+ license_clues: [],
license_policy: [
"apache-2.0 - Approved License",
"gpl-3.0 - Restricted License",
],
- license_clues: [],
license_detections: [
{
license_expression: "apache-2.0 AND gpl-3.0",
@@ -730,21 +742,21 @@ export const LicenseSamples: {
"apache_2_0_and_gpl_3_0-494ca0ae-1282-09a2-139f-a52c04fde6dc",
},
],
+ detected_license_expression: "apache-2.0 AND gpl-3.0",
+ detected_license_expression_spdx: "Apache-2.0 AND GPL-3.0-only",
+ percentage_of_license_text: 100,
},
{
+ license_clues: [],
+ license_policy: [],
+ license_detections: [],
detected_license_expression: null,
detected_license_expression_spdx: null,
percentage_of_license_text: 0,
- license_policy: [],
- license_clues: [],
- license_detections: [],
},
{
- detected_license_expression: "mit",
- detected_license_expression_spdx: "MIT",
- percentage_of_license_text: 6.36,
- license_policy: [],
license_clues: [],
+ license_policy: [],
license_detections: [
{
license_expression: "mit",
@@ -752,20 +764,19 @@ export const LicenseSamples: {
identifier: "mit-b941df29-6c4b-fe7e-752f-a5fc7f9a28b5",
},
],
+ detected_license_expression: "mit",
+ detected_license_expression_spdx: "MIT",
+ percentage_of_license_text: 6.36,
},
{
+ license_clues: [],
+ license_policy: [],
+ license_detections: [],
detected_license_expression: null,
detected_license_expression_spdx: null,
percentage_of_license_text: 0,
- license_policy: [],
- license_clues: [],
- license_detections: [],
},
{
- detected_license_expression: null,
- detected_license_expression_spdx: null,
- percentage_of_license_text: 6.37,
- license_policy: [],
license_clues: [
{
score: 52,
@@ -779,6 +790,7 @@ export const LicenseSamples: {
rule_relevance: 100,
rule_url:
"https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/apache-2.0_or_gpl-2.0_24.RULE",
+ license_expression_spdx: "Apache-2.0 OR GPL-2.0-only",
fileId: 5,
filePath: "rx-lite/package.json",
fileClueIdx: 0,
@@ -791,11 +803,7 @@ export const LicenseSamples: {
match_coverage: 52,
matcher: "3-seq",
license_expression: "apache-2.0 OR gpl-2.0",
- rule_identifier: "apache-2.0_or_gpl-2.0_24.RULE",
- rule_relevance: 100,
- rule_url:
- "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/apache-2.0_or_gpl-2.0_24.RULE",
- path: "rx-lite/package.json",
+ license_expression_spdx: "Apache-2.0 OR GPL-2.0-only",
license_expression_keys: [
{
key: "apache-2.0",
@@ -812,14 +820,37 @@ export const LicenseSamples: {
"https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/gpl-2.0.LICENSE",
},
],
+ license_expression_spdx_keys: [
+ {
+ key: "Apache-2.0",
+ spdx_url: "https://spdx.org/licenses/Apache-2.0",
+ },
+ {
+ key: "GPL-2.0-only",
+ spdx_url: "https://spdx.org/licenses/GPL-2.0-only",
+ },
+ ],
+ rule_identifier: "apache-2.0_or_gpl-2.0_24.RULE",
+ rule_relevance: 100,
+ rule_url:
+ "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/apache-2.0_or_gpl-2.0_24.RULE",
+ path: "rx-lite/package.json",
},
],
file_regions: [
- { path: "rx-lite/package.json", start_line: 56, end_line: 59 },
+ {
+ path: "rx-lite/package.json",
+ start_line: 56,
+ end_line: 59,
+ },
],
},
],
+ license_policy: [],
license_detections: [],
+ detected_license_expression: null,
+ detected_license_expression_spdx: null,
+ percentage_of_license_text: 6.37,
},
],
},