From 8c26db9c524968ffb2c59bfe4379f8815adbbb36 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Sun, 12 Jun 2022 07:35:53 +1200 Subject: [PATCH] feat: support parsing commits from `package-lock.json` files --- README.md | 4 +- pkg/lockfile/fixtures/npm/commits.v1.json | 103 ++++++++ pkg/lockfile/fixtures/npm/commits.v2.json | 272 ++++++++++++++++++++++ pkg/lockfile/parse-npm-lock-v1_test.go | 103 ++++++++ pkg/lockfile/parse-npm-lock-v2_test.go | 97 ++++++++ pkg/lockfile/parse-npm-lock.go | 31 ++- pkg/lockfile/parse-yarn-lock.go | 4 + 7 files changed, 609 insertions(+), 5 deletions(-) create mode 100644 pkg/lockfile/fixtures/npm/commits.v1.json create mode 100644 pkg/lockfile/fixtures/npm/commits.v2.json diff --git a/README.md b/README.md index 2e65f4aa..89290ef6 100644 --- a/README.md +++ b/README.md @@ -70,8 +70,8 @@ typically a few hours ahead of the offline databases, and supports commits; however it currently can produce false negatives for some ecosystems. > While the API supports commits, the detector currently has limited support for -> extracting them - only the `composer.lock` & `yarn.lock` parsers include -> commit details +> extracting them - only the `composer.lock`, `package-lock.json`, & `yarn.lock` +> parsers include commit details You cannot use the API in `--offline` mode, but you can use both the offline databases and the API together; the detector will remove any duplicate results. diff --git a/pkg/lockfile/fixtures/npm/commits.v1.json b/pkg/lockfile/fixtures/npm/commits.v1.json new file mode 100644 index 00000000..489af74e --- /dev/null +++ b/pkg/lockfile/fixtures/npm/commits.v1.json @@ -0,0 +1,103 @@ +{ + "lockfileVersion": 2, + "requires": true, + "dependencies": { + "@segment/analytics.js-integration-facebook-pixel": { + "version": "git+ssh://git@github.com/segmentio/analytics.js-integrations.git#3b1bb80b302c2e552685dc8a029797ec832ea7c9", + "from": "@segment/analytics.js-integration-facebook-pixel@github:segmentio/analytics.js-integrations#2.4.1", + "requires": { + "@ndhoule/each": "^2.0.1", + "@ndhoule/foldl": "^2.0.1", + "@segment/analytics.js-integration": "^3.1.0", + "dateformat": "^1.0.12", + "is": "^3.2.1", + "reject": "0.0.1", + "segmentio-facade": "^3.1.0", + "to-camel-case": "^1.0.0" + } + }, + "ansi-styles": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-1.0.0.tgz", + "integrity": "sha1-yxAt8cVvUSPquLZ817mAJ6AnkXg=" + }, + "babel-preset-php": { + "version": "gitlab:kornelski/babel-preset-php#c5a7ba5e0ad98b8db1cb8ce105403dd4b768cced", + "from": "gitlab:kornelski/babel-preset-php#main", + "requires": { + "php-parser": "^2.0.6" + } + }, + "is-number-1": { + "version": "git+ssh://git@github.com/jonschlinkert/is-number.git#af885e2e890b9ef0875edd2b117305119ee5bdc5", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + } + }, + "is-number-2": { + "version": "github:jonschlinkert/is-number#d5ac0584ee9ae7bd9288220a39780f155b9ad4c8", + "from": "github:jonschlinkert/is-number#main" + }, + "is-number-3": { + "version": "git+ssh://git@github.com/jonschlinkert/is-number.git#d5ac0584ee9ae7bd9288220a39780f155b9ad4c8", + "dev": true, + "from": "is-number@https://github.com/jonschlinkert/is-number.git#2.0.0" + }, + "is-number-4": { + "version": "git+ssh://git@github.com/jonschlinkert/is-number.git#af885e2e890b9ef0875edd2b117305119ee5bdc5", + "dev": true, + "from": "is-number@git+ssh://git@github.com:jonschlinkert/is-number.git", + "requires": { + "kind-of": "^3.0.2" + } + }, + "is-number-5": { + "version": "git+https://dummy-token@github.com/jonschlinkert/is-number.git#af885e2e890b9ef0875edd2b117305119ee5bdc5", + "dev": true, + "from": "is-number@https://dummy-token@github.com/jonschlinkert/is-number.git#main", + "requires": { + "kind-of": "^3.0.2" + } + }, + "is-number-6": { + "version": "git+https://github.com/jonschlinkert/is-number.git#af885e2e890b9ef0875edd2b117305119ee5bdc5", + "dev": true, + "requires": { + "kind-of": "3.2.2" + } + }, + "postcss-calc": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/postcss-calc/-/postcss-calc-7.0.1.tgz", + "integrity": "sha512-oXqx0m6tb4N3JGdmeMSc/i91KppbYsFZKdH0xMOqK8V1rJlzrKlTdokz8ozUXLVejydRN6u2IddxpcijRj2FqQ==", + "dependencies": { + "is-number-1": { + "version": "git+ssh://git@github.com/jonschlinkert/is-number.git#be5935f8d2595bcd97b05718ef1eeae08d812e10", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + } + }, + "is-number-2": { + "version": "github:jonschlinkert/is-number#82dcc8e914dabd9305ab9ae580709a7825e824f5", + "from": "github:jonschlinkert/is-number#main" + }, + "is-number-3": { + "version": "git+ssh://git@github.com/jonschlinkert/is-number.git#82ae8802978da40d7f1be5ad5943c9e550ab2c89", + "dev": true, + "from": "is-number@https://github.com/jonschlinkert/is-number.git#2.0.0" + } + } + }, + "raven-js": { + "version": "git+ssh://git@github.com/getsentry/raven-js.git#c2b377e7a254264fd4a1fe328e4e3cfc9e245570", + "from": "raven-js@getsentry/raven-js#3.23.1" + }, + "slick-carousel": { + "version": "git://github.com/brianfryer/slick.git#280b560161b751ba226d50c7db1e0a14a78c2de0", + "dev": true, + "from": "git://github.com/brianfryer/slick.git" + } + } +} diff --git a/pkg/lockfile/fixtures/npm/commits.v2.json b/pkg/lockfile/fixtures/npm/commits.v2.json new file mode 100644 index 00000000..78a3a2a8 --- /dev/null +++ b/pkg/lockfile/fixtures/npm/commits.v2.json @@ -0,0 +1,272 @@ +{ + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "bump-test", + "version": "0.0.1", + "license": "ISC", + "dependencies": { + "chalk": "0.4.0", + "lodash": "^1.2.1", + "stopwords": "0.0.1" + }, + "devDependencies": { + "@segment/analytics.js-integration-facebook-pixel": "github:segmentio/analytics.js-integrations#2.4.1", + "babel-preset-php": "gitlab:kornelski/babel-preset-php#main", + "is-number-1": "https://github.com/jonschlinkert/is-number.git", + "is-number-2": "https://github.com/jonschlinkert/is-number.git#d5ac058", + "is-number-3": "https://github.com/jonschlinkert/is-number.git#2.0.0", + "is-number-4": "git+ssh://git@github.com:jonschlinkert/is-number.git", + "is-number-5": "https://dummy-token@github.com/jonschlinkert/is-number.git#main", + "raven-js": "getsentry/raven-js#3.23.1", + "slick-carousel": "git://github.com/brianfryer/slick" + } + }, + "node_modules/@segment/analytics.js-integration-facebook-pixel": { + "version": "2.4.1", + "resolved": "git+ssh://git@github.com/segmentio/analytics.js-integrations.git#3b1bb80b302c2e552685dc8a029797ec832ea7c9", + "integrity": "sha512-1YAMDYJHS/owJXgqD8/JoT/V0qqGwh4Ca+7uUBeXZ4zhWviqZbdjgU6FZJdstgclkxkfcoJ0dQhFlS4k0lLBCw==", + "license": "SEE LICENSE IN LICENSE", + "dependencies": { + "@ndhoule/each": "^2.0.1", + "@ndhoule/foldl": "^2.0.1", + "@segment/analytics.js-integration": "^3.1.0", + "dateformat": "^1.0.12", + "is": "^3.2.1", + "reject": "0.0.1", + "segmentio-facade": "^3.1.0", + "to-camel-case": "^1.0.0" + } + }, + "node_modules/ansi-styles": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-1.0.0.tgz", + "integrity": "sha1-yxAt8cVvUSPquLZ817mAJ6AnkXg=", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/babel-preset-php": { + "version": "1.1.1", + "resolved": "git+ssh://git@gitlab.com/kornelski/babel-preset-php.git#c5a7ba5e0ad98b8db1cb8ce105403dd4b768cced", + "integrity": "sha512-PhaHzhRj5Qr2+XH6DmO/e0iki/h7nZGUR+DCaSnDxEjFpWLZoWJ40wTr7hSQ3s04cKSyFXUW6GF5l9M0rNUiqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "php-parser": "^2.0.6" + } + }, + "node_modules/is-number-1": { + "version": "3.0.0", + "resolved": "git+ssh://git@github.com/jonschlinkert/is-number.git#af885e2e890b9ef0875edd2b117305119ee5bdc5", + "integrity": "sha512-xjDEFh9GXVljB+6/Q3eYsNoFShbxGGq7YKqz8sCBSvl7uEy7Gr5NzM2WMNdsarDLLOsbzrIul425XhH3g84FxQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number-2": { + "version": "2.0.0", + "resolved": "git+ssh://git@github.com/jonschlinkert/is-number.git#d5ac0584ee9ae7bd9288220a39780f155b9ad4c8", + "integrity": "sha512-h1VffkXxrWWHTUM5up3BDE7BmcHOg/6BkU7Ic1OZWgIRzySUKHtQ/mP9LSkpu8mJaRpJE1PDQSgO8D+dQz4s1g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number-3": { + "version": "2.0.0", + "resolved": "git+ssh://git@github.com/jonschlinkert/is-number.git#d5ac0584ee9ae7bd9288220a39780f155b9ad4c8", + "integrity": "sha512-h1VffkXxrWWHTUM5up3BDE7BmcHOg/6BkU7Ic1OZWgIRzySUKHtQ/mP9LSkpu8mJaRpJE1PDQSgO8D+dQz4s1g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number-4": { + "version": "3.0.0", + "resolved": "git+ssh://git@github.com/jonschlinkert/is-number.git#af885e2e890b9ef0875edd2b117305119ee5bdc5", + "integrity": "sha512-xjDEFh9GXVljB+6/Q3eYsNoFShbxGGq7YKqz8sCBSvl7uEy7Gr5NzM2WMNdsarDLLOsbzrIul425XhH3g84FxQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number-5": { + "version": "3.0.0", + "resolved": "git+https://dummy-token@github.com/jonschlinkert/is-number.git#af885e2e890b9ef0875edd2b117305119ee5bdc5", + "integrity": "sha512-xjDEFh9GXVljB+6/Q3eYsNoFShbxGGq7YKqz8sCBSvl7uEy7Gr5NzM2WMNdsarDLLOsbzrIul425XhH3g84FxQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postcss-calc": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/postcss-calc/-/postcss-calc-7.0.1.tgz", + "integrity": "sha512-oXqx0m6tb4N3JGdmeMSc/i91KppbYsFZKdH0xMOqK8V1rJlzrKlTdokz8ozUXLVejydRN6u2IddxpcijRj2FqQ==", + "dependencies": { + "css-unit-converter": "^1.1.1", + "postcss": "^7.0.5", + "postcss-selector-parser": "^5.0.0-rc.4", + "postcss-value-parser": "^3.3.1" + } + }, + "node_modules/postcss-calc/node_modules/is-number-1": { + "version": "3.0.0", + "resolved": "git+ssh://git@github.com/jonschlinkert/is-number.git#be5935f8d2595bcd97b05718ef1eeae08d812e10", + "integrity": "sha512-xjDEFh9GXVljB+6/Q3eYsNoFShbxGGq7YKqz8sCBSvl7uEy7Gr5NzM2WMNdsarDLLOsbzrIul425XhH3g84FxQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postcss-calc/node_modules/is-number-2": { + "version": "2.0.0", + "resolved": "git+ssh://git@github.com/jonschlinkert/is-number.git#82dcc8e914dabd9305ab9ae580709a7825e824f5", + "integrity": "sha512-xjDEFh9GXVljB+6/Q3eYsNoFShbxGGq7YKqz8sCBSvl7uEy7Gr5NzM2WMNdsarDLLOsbzrIul425XhH3g84FxQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postcss-calc/node_modules/is-number-3": { + "version": "3.0.0", + "resolved": "git+ssh://git@github.com/jonschlinkert/is-number.git#82ae8802978da40d7f1be5ad5943c9e550ab2c89", + "integrity": "sha512-xjDEFh9GXVljB+6/Q3eYsNoFShbxGGq7YKqz8sCBSvl7uEy7Gr5NzM2WMNdsarDLLOsbzrIul425XhH3g84FxQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/raven-js": { + "resolved": "git+ssh://git@github.com/getsentry/raven-js.git#c2b377e7a254264fd4a1fe328e4e3cfc9e245570" + }, + "node_modules/slick-carousel": { + "version": "1.7.1", + "resolved": "git+ssh://git@github.com/brianfryer/slick.git#280b560161b751ba226d50c7db1e0a14a78c2de0", + "integrity": "sha512-oraDlz7tkdtzSOgofC3ZpuCsz6Xf1sIu+bdTrE5qasqQayr5m30bO+DJUxLM+5GyolBDCvmcsg8wltozYdO/XQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "jquery": ">=1.7.2" + } + } + }, + "dependencies": { + "@segment/analytics.js-integration-facebook-pixel": { + "version": "git+ssh://git@github.com/segmentio/analytics.js-integrations.git#3b1bb80b302c2e552685dc8a029797ec832ea7c9", + "from": "@segment/analytics.js-integration-facebook-pixel@github:segmentio/analytics.js-integrations#2.4.1", + "requires": { + "@ndhoule/each": "^2.0.1", + "@ndhoule/foldl": "^2.0.1", + "@segment/analytics.js-integration": "^3.1.0", + "dateformat": "^1.0.12", + "is": "^3.2.1", + "reject": "0.0.1", + "segmentio-facade": "^3.1.0", + "to-camel-case": "^1.0.0" + } + }, + "ansi-styles": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-1.0.0.tgz", + "integrity": "sha1-yxAt8cVvUSPquLZ817mAJ6AnkXg=" + }, + "babel-preset-php": { + "version": "git+ssh://git@gitlab.com/kornelski/babel-preset-php.git#c5a7ba5e0ad98b8db1cb8ce105403dd4b768cced", + "dev": true, + "from": "babel-preset-php@gitlab:kornelski/babel-preset-php#main", + "requires": { + "php-parser": "^2.0.6" + } + }, + "is-number-1": { + "version": "git+ssh://git@github.com/jonschlinkert/is-number.git#af885e2e890b9ef0875edd2b117305119ee5bdc5", + "dev": true, + "from": "is-number@https://github.com/jonschlinkert/is-number.git", + "requires": { + "kind-of": "^3.0.2" + } + }, + "is-number-2": { + "version": "git+ssh://git@github.com/jonschlinkert/is-number.git#d5ac0584ee9ae7bd9288220a39780f155b9ad4c8", + "dev": true, + "from": "is-number@https://github.com/jonschlinkert/is-number.git#d5ac058" + }, + "is-number-3": { + "version": "git+ssh://git@github.com/jonschlinkert/is-number.git#d5ac0584ee9ae7bd9288220a39780f155b9ad4c8", + "dev": true, + "from": "is-number@https://github.com/jonschlinkert/is-number.git#2.0.0" + }, + "is-number-4": { + "version": "git+ssh://git@github.com/jonschlinkert/is-number.git#af885e2e890b9ef0875edd2b117305119ee5bdc5", + "dev": true, + "from": "is-number@git+ssh://git@github.com:jonschlinkert/is-number.git", + "requires": { + "kind-of": "^3.0.2" + } + }, + "is-number-5": { + "version": "git+https://dummy-token@github.com/jonschlinkert/is-number.git#af885e2e890b9ef0875edd2b117305119ee5bdc5", + "dev": true, + "from": "is-number@https://dummy-token@github.com/jonschlinkert/is-number.git#main", + "requires": { + "kind-of": "^3.0.2" + } + }, + "postcss-calc": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/postcss-calc/-/postcss-calc-7.0.1.tgz", + "integrity": "sha512-oXqx0m6tb4N3JGdmeMSc/i91KppbYsFZKdH0xMOqK8V1rJlzrKlTdokz8ozUXLVejydRN6u2IddxpcijRj2FqQ==", + "dependencies": { + "is-number-1": { + "version": "git+ssh://git@github.com/jonschlinkert/is-number.git#be5935f8d2595bcd97b05718ef1eeae08d812e10", + "dev": true, + "requires": { + "kind-of": "^3.0.2" + } + }, + "is-number-2": { + "version": "github:jonschlinkert/is-number#82dcc8e914dabd9305ab9ae580709a7825e824f5", + "from": "github:jonschlinkert/is-number#main" + }, + "is-number-3": { + "version": "git+ssh://git@github.com/jonschlinkert/is-number.git#82ae8802978da40d7f1be5ad5943c9e550ab2c89", + "dev": true, + "from": "is-number@https://github.com/jonschlinkert/is-number.git#2.0.0" + } + } + }, + "raven-js": { + "version": "git+ssh://git@github.com/getsentry/raven-js.git#c2b377e7a254264fd4a1fe328e4e3cfc9e245570", + "from": "raven-js@getsentry/raven-js#3.23.1" + }, + "slick-carousel": { + "version": "git+ssh://git@github.com/brianfryer/slick.git#280b560161b751ba226d50c7db1e0a14a78c2de0", + "dev": true, + "from": "slick-carousel@git://github.com/brianfryer/slick" + } + } +} diff --git a/pkg/lockfile/parse-npm-lock-v1_test.go b/pkg/lockfile/parse-npm-lock-v1_test.go index 79b54aea..9b127ff7 100644 --- a/pkg/lockfile/parse-npm-lock-v1_test.go +++ b/pkg/lockfile/parse-npm-lock-v1_test.go @@ -187,3 +187,106 @@ func TestParseNpmLock_v1_NestedDependenciesDup(t *testing.T) { Ecosystem: lockfile.NpmEcosystem, }) } + +func TestParseNpmLock_v1_Commits(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseNpmLock("fixtures/npm/commits.v1.json") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "@segment/analytics.js-integration-facebook-pixel", + Version: "", + Ecosystem: lockfile.NpmEcosystem, + Commit: "3b1bb80b302c2e552685dc8a029797ec832ea7c9", + }, + { + Name: "ansi-styles", + Version: "1.0.0", + Ecosystem: lockfile.NpmEcosystem, + Commit: "", + }, + { + Name: "babel-preset-php", + Version: "", + Ecosystem: lockfile.NpmEcosystem, + Commit: "c5a7ba5e0ad98b8db1cb8ce105403dd4b768cced", + }, + { + Name: "is-number-1", + Version: "", + Ecosystem: lockfile.NpmEcosystem, + Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", + }, + { + Name: "is-number-1", + Version: "", + Ecosystem: lockfile.NpmEcosystem, + Commit: "be5935f8d2595bcd97b05718ef1eeae08d812e10", + }, + { + Name: "is-number-2", + Version: "", + Ecosystem: lockfile.NpmEcosystem, + Commit: "d5ac0584ee9ae7bd9288220a39780f155b9ad4c8", + }, + { + Name: "is-number-2", + Version: "", + Ecosystem: lockfile.NpmEcosystem, + Commit: "82dcc8e914dabd9305ab9ae580709a7825e824f5", + }, + { + Name: "is-number-3", + Version: "", + Ecosystem: lockfile.NpmEcosystem, + Commit: "d5ac0584ee9ae7bd9288220a39780f155b9ad4c8", + }, + { + Name: "is-number-3", + Version: "", + Ecosystem: lockfile.NpmEcosystem, + Commit: "82ae8802978da40d7f1be5ad5943c9e550ab2c89", + }, + { + Name: "is-number-4", + Version: "", + Ecosystem: lockfile.NpmEcosystem, + Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", + }, + { + Name: "is-number-5", + Version: "", + Ecosystem: lockfile.NpmEcosystem, + Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", + }, + { + Name: "is-number-6", + Version: "", + Ecosystem: lockfile.NpmEcosystem, + Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", + }, + { + Name: "postcss-calc", + Version: "7.0.1", + Ecosystem: lockfile.NpmEcosystem, + Commit: "", + }, + { + Name: "raven-js", + Version: "", + Ecosystem: lockfile.NpmEcosystem, + Commit: "c2b377e7a254264fd4a1fe328e4e3cfc9e245570", + }, + { + Name: "slick-carousel", + Version: "", + Ecosystem: lockfile.NpmEcosystem, + Commit: "280b560161b751ba226d50c7db1e0a14a78c2de0", + }, + }) +} diff --git a/pkg/lockfile/parse-npm-lock-v2_test.go b/pkg/lockfile/parse-npm-lock-v2_test.go index 25615aa1..2158f9f3 100644 --- a/pkg/lockfile/parse-npm-lock-v2_test.go +++ b/pkg/lockfile/parse-npm-lock-v2_test.go @@ -177,3 +177,100 @@ func TestParseNpmLock_v2_NestedDependenciesDup(t *testing.T) { }, }) } + +func TestParseNpmLock_v2_Commits(t *testing.T) { + t.Parallel() + + packages, err := lockfile.ParseNpmLock("fixtures/npm/commits.v2.json") + + if err != nil { + t.Errorf("Got unexpected error: %v", err) + } + + expectPackages(t, packages, []lockfile.PackageDetails{ + { + Name: "@segment/analytics.js-integration-facebook-pixel", + Version: "2.4.1", + Ecosystem: lockfile.NpmEcosystem, + Commit: "3b1bb80b302c2e552685dc8a029797ec832ea7c9", + }, + { + Name: "ansi-styles", + Version: "1.0.0", + Ecosystem: lockfile.NpmEcosystem, + Commit: "", + }, + { + Name: "babel-preset-php", + Version: "1.1.1", + Ecosystem: lockfile.NpmEcosystem, + Commit: "c5a7ba5e0ad98b8db1cb8ce105403dd4b768cced", + }, + { + Name: "is-number-1", + Version: "3.0.0", + Ecosystem: lockfile.NpmEcosystem, + Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", + }, + { + Name: "is-number-1", + Version: "3.0.0", + Ecosystem: lockfile.NpmEcosystem, + Commit: "be5935f8d2595bcd97b05718ef1eeae08d812e10", + }, + { + Name: "is-number-2", + Version: "2.0.0", + Ecosystem: lockfile.NpmEcosystem, + Commit: "d5ac0584ee9ae7bd9288220a39780f155b9ad4c8", + }, + { + Name: "is-number-2", + Version: "2.0.0", + Ecosystem: lockfile.NpmEcosystem, + Commit: "82dcc8e914dabd9305ab9ae580709a7825e824f5", + }, + { + Name: "is-number-3", + Version: "2.0.0", + Ecosystem: lockfile.NpmEcosystem, + Commit: "d5ac0584ee9ae7bd9288220a39780f155b9ad4c8", + }, + { + Name: "is-number-3", + Version: "3.0.0", + Ecosystem: lockfile.NpmEcosystem, + Commit: "82ae8802978da40d7f1be5ad5943c9e550ab2c89", + }, + { + Name: "is-number-4", + Version: "3.0.0", + Ecosystem: lockfile.NpmEcosystem, + Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", + }, + { + Name: "is-number-5", + Version: "3.0.0", + Ecosystem: lockfile.NpmEcosystem, + Commit: "af885e2e890b9ef0875edd2b117305119ee5bdc5", + }, + { + Name: "postcss-calc", + Version: "7.0.1", + Ecosystem: lockfile.NpmEcosystem, + Commit: "", + }, + { + Name: "raven-js", + Version: "", + Ecosystem: lockfile.NpmEcosystem, + Commit: "c2b377e7a254264fd4a1fe328e4e3cfc9e245570", + }, + { + Name: "slick-carousel", + Version: "1.7.1", + Ecosystem: lockfile.NpmEcosystem, + Commit: "280b560161b751ba226d50c7db1e0a14a78c2de0", + }, + }) +} diff --git a/pkg/lockfile/parse-npm-lock.go b/pkg/lockfile/parse-npm-lock.go index 6289eb57..581790f4 100644 --- a/pkg/lockfile/parse-npm-lock.go +++ b/pkg/lockfile/parse-npm-lock.go @@ -15,6 +15,7 @@ type NpmLockDependency struct { type NpmLockPackage struct { Version string `json:"version"` + Resolved string `json:"resolved"` Dependencies map[string]string `json:"dependencies"` } @@ -60,10 +61,24 @@ func parseNpmLockDependencies(dependencies map[string]NpmLockDependency) map[str details = mergePkgDetailsMap(details, parseNpmLockDependencies(detail.Dependencies)) } - details[name+"@"+detail.Version] = PackageDetails{ + version := detail.Version + finalVersion := version + commit := tryExtractCommit(detail.Version) + + // if there is a commit, we want to deduplicate based on that rather than + // the version (the versions must match anyway for the commits to match) + // + // we also don't actually know what the "version" is, so blank it + if commit != "" { + finalVersion = "" + version = commit + } + + details[name+"@"+version] = PackageDetails{ Name: name, - Version: detail.Version, + Version: finalVersion, Ecosystem: NpmEcosystem, + Commit: commit, } } @@ -89,11 +104,21 @@ func parseNpmLockPackages(packages map[string]NpmLockPackage) map[string]Package continue } finalName := extractNpmPackageName(namePath) + finalVersion := detail.Version + + commit := tryExtractCommit(detail.Resolved) + + // if there is a commit, we want to deduplicate based on that rather than + // the version (the versions must match anyway for the commits to match) + if commit != "" { + finalVersion = commit + } - details[finalName+"@"+detail.Version] = PackageDetails{ + details[finalName+"@"+finalVersion] = PackageDetails{ Name: finalName, Version: detail.Version, Ecosystem: NpmEcosystem, + Commit: commit, } } diff --git a/pkg/lockfile/parse-yarn-lock.go b/pkg/lockfile/parse-yarn-lock.go index 93abdc17..c1f06898 100644 --- a/pkg/lockfile/parse-yarn-lock.go +++ b/pkg/lockfile/parse-yarn-lock.go @@ -104,6 +104,10 @@ func tryExtractCommit(resolution string) string { `(?:^|.+@)https://.+\.git#(\w+)$`, `https://codeload\.github\.com(?:/[\w-.]+){2}/tar\.gz/(\w+)$`, `.+#commit[:=](\w+)$`, + // github:... + // gitlab:... + // bitbucket:... + `^(?:github|gitlab|bitbucket):.+#(\w+)$`, } for _, matcher := range matchers {