From 5d57af5c235f98047a7265f6e0d28f6005a58ccd Mon Sep 17 00:00:00 2001
From: Riccardo Schirone <rschiron@redhat.com>
Date: Wed, 25 May 2022 14:47:09 +0200
Subject: [PATCH 1/2] Add a IsBundled dependency property
When possible differentiate between runtime dependencies and bundled
code. By bundled code we can refer to copy-pasted code, vendored code
(e.g. github, node_modules), copy-pasted code, webpacked, etc.
---
cmd/deplist/deplist.go | 12 ++++++++--
dependencies.go | 9 +++----
deplist.go | 52 ++++++++++++++++++++++++++++-------------
internal/scan/nodejs.go | 24 +++++++++++++++++++
internal/utils/utils.go | 2 --
5 files changed, 75 insertions(+), 24 deletions(-)
diff --git a/cmd/deplist/deplist.go b/cmd/deplist/deplist.go
index bb79c79e..664ea640 100644
--- a/cmd/deplist/deplist.go
+++ b/cmd/deplist/deplist.go
@@ -36,13 +36,21 @@ func main() {
version := dep.Version
inst, _ := purl.FromString(fmt.Sprintf("pkg:%s/%s@%s", deplist.GetLanguageStr(dep.DepType), dep.Path, version))
- fmt.Println(inst)
+ fmt.Print(inst)
+ if dep.IsBundled {
+ fmt.Print(" [bundled]")
+ }
+ fmt.Println()
}
} else {
deptype := deplist.Bitmask(*deptypePtr)
for _, dep := range deps {
if (dep.DepType & deptype) == deptype {
- fmt.Printf("%s@%s\n", dep.Path, dep.Version)
+ fmt.Printf("%s@%s", dep.Path, dep.Version)
+ if dep.IsBundled {
+ fmt.Print(" [bundled]")
+ }
+ fmt.Println()
}
}
}
diff --git a/dependencies.go b/dependencies.go
index 3cf8c2e0..fbdceac5 100644
--- a/dependencies.go
+++ b/dependencies.go
@@ -5,10 +5,11 @@ type Bitmask uint32
// Dependency per dependency info
type Dependency struct {
- DepType Bitmask // golang, nodejs, python etc
- Path string // the module path, github.com/teris-io/shortid
- Version string // v0.0.0-20171029131806-771a37caa5cf
- Files []string // if available, list of all files for a package
+ DepType Bitmask // golang, nodejs, python etc
+ Path string // the module path, github.com/teris-io/shortid
+ Version string // v0.0.0-20171029131806-771a37caa5cf
+ Files []string // if available, list of all files for a package
+ IsBundled bool // whether the dependency is bundled somehow in the dependency (e.g. vendored code, copy-pasted code, embedded code, etc.)
// /usr/lib/go-1.13/src/regexp/syntax/compile.go
// /usr/lib/go-1.13/src/regexp/syntax/doc.go
}
diff --git a/deplist.go b/deplist.go
index 53597ac9..406be567 100644
--- a/deplist.go
+++ b/deplist.go
@@ -80,16 +80,32 @@ func getDeps(fullPath string) ([]Dependency, Bitmask, error) {
}
if info.IsDir() {
- // prevent walking down the vendors, docs, etc
+ // prevent walking down the docs, .git, tests, etc.
if utils.BelongsToIgnoreList(info.Name()) {
return filepath.SkipDir
}
} else {
// Two checks, one for filenames and the second switch for full
// paths. Useful if we're looking for top of repo
-
switch filename := info.Name(); filename {
// for now only go for yarn and npm
+ case "package.json":
+ pkg, err := scan.GetNodeJSPackage(path)
+ if err != nil {
+ log.Debugf("failed to scan for nodejs package: %s", path)
+ return nil
+ }
+
+ foundTypes.DepFoundAddFlag(LangNodeJS)
+
+ deps = append(deps,
+ Dependency{
+ DepType: LangNodeJS,
+ Path: pkg.Name,
+ Version: pkg.Version,
+ Files: []string{},
+ IsBundled: true,
+ })
case "package-lock.json":
// if theres not a yarn.lock fall thru
if _, err := os.Stat(
@@ -160,10 +176,11 @@ func getDeps(fullPath string) ([]Dependency, Bitmask, error) {
if !strings.HasSuffix(version, "-javadoc") && !strings.HasSuffix(version, "-sources") {
deps = append(deps,
Dependency{
- DepType: LangJava,
- Path: name,
- Version: version,
- Files: []string{},
+ DepType: LangJava,
+ Path: name,
+ Version: version,
+ Files: []string{},
+ IsBundled: true,
})
}
}
@@ -184,10 +201,11 @@ func getDeps(fullPath string) ([]Dependency, Bitmask, error) {
for path, goPkg := range pkgs {
d := Dependency{
- DepType: LangGolang,
- Path: path,
- Files: goPkg.Gofiles,
- Version: goPkg.Version,
+ DepType: LangGolang,
+ Path: path,
+ Files: goPkg.Gofiles,
+ Version: goPkg.Version,
+ IsBundled: true,
}
deps = append(deps, d)
}
@@ -202,9 +220,10 @@ func getDeps(fullPath string) ([]Dependency, Bitmask, error) {
}
for _, goPkg := range pkgs {
d := Dependency{
- DepType: LangGolang,
- Path: goPkg.Name,
- Version: goPkg.Version,
+ DepType: LangGolang,
+ Path: goPkg.Name,
+ Version: goPkg.Version,
+ IsBundled: true,
}
deps = append(deps, d)
}
@@ -219,9 +238,10 @@ func getDeps(fullPath string) ([]Dependency, Bitmask, error) {
}
for _, goPkg := range pkgs {
d := Dependency{
- DepType: LangGolang,
- Path: goPkg.Name,
- Version: goPkg.Version,
+ DepType: LangGolang,
+ Path: goPkg.Name,
+ Version: goPkg.Version,
+ IsBundled: true,
}
deps = append(deps, d)
}
diff --git a/internal/scan/nodejs.go b/internal/scan/nodejs.go
index 275cac1f..d4e10c05 100644
--- a/internal/scan/nodejs.go
+++ b/internal/scan/nodejs.go
@@ -3,6 +3,7 @@ package scan
import (
"encoding/json"
"fmt"
+ "os"
"os/exec"
"path/filepath"
"strings"
@@ -25,6 +26,10 @@ type yarnOutput struct {
}
}
+type packageJsonFormat struct {
+ Name string `json:"name"`
+ Version string `json:"version"`
+}
type npmDependency struct {
Version string `json:"version"`
Dependencies map[string]npmDependency `json:"dependencies"`
@@ -121,6 +126,25 @@ func GetNodeJSDeps(path string) (map[string]NodeJSGather, error) {
return nil, fmt.Errorf("unknown NodeJS dependency file %q", path)
}
+func GetNodeJSPackage(path string) (NodeJSGather, error) {
+ log.Debugf("GetNodeJSPackage %s", path)
+
+ data, err := os.ReadFile(path)
+ if err != nil {
+ return NodeJSGather{}, err
+ }
+
+ var packageJson packageJsonFormat
+ err = json.Unmarshal(data, &packageJson)
+ if err != nil {
+ return NodeJSGather{}, err
+ }
+ if packageJson.Name == "" {
+ return NodeJSGather{}, fmt.Errorf("Empty package")
+ }
+ return NodeJSGather{Name: packageJson.Name, Version: packageJson.Version}, nil
+}
+
func getYarnDeps(path string) (map[string]NodeJSGather, error) {
var yarnOutput yarnOutput
gatheredNode = make(map[string]NodeJSGather)
diff --git a/internal/utils/utils.go b/internal/utils/utils.go
index ae8b1d7f..7521c3cd 100644
--- a/internal/utils/utils.go
+++ b/internal/utils/utils.go
@@ -9,8 +9,6 @@ import (
func BelongsToIgnoreList(needle string) bool {
switch needle {
case
- "node_modules",
- "vendor",
"scripts",
"docs",
"test",
From 0c9c28c418424c10cffc8a2d62eab220e290ef9a Mon Sep 17 00:00:00 2001
From: Riccardo Schirone <sirmy15@gmail.com>
Date: Wed, 13 Jul 2022 10:10:45 +0200
Subject: [PATCH 2/2] Add cmd flag to choose which type of dependencies to
print
---
cmd/deplist/deplist.go | 51 +++++++++++++++++++++++++++++++++++++-----
1 file changed, 45 insertions(+), 6 deletions(-)
diff --git a/cmd/deplist/deplist.go b/cmd/deplist/deplist.go
index 664ea640..85678c9b 100644
--- a/cmd/deplist/deplist.go
+++ b/cmd/deplist/deplist.go
@@ -3,18 +3,56 @@ package main
import (
"flag"
"fmt"
+ "strings"
"github.com/RedHatProductSecurity/deplist"
purl "github.com/mcoops/packageurl-go"
log "github.com/sirupsen/logrus"
)
+type searchModeFlag []string
+
+func (i *searchModeFlag) String() string {
+ return strings.Join(*i, ", ")
+}
+
+func (i *searchModeFlag) Set(value string) error {
+ values := strings.Split(value, ",")
+ for _, value := range values {
+ switch value {
+ case "deps":
+ *i = append(*i, "deps")
+ case "bundled":
+ *i = append(*i, "bundled")
+ }
+ }
+ return nil
+}
+
+func (i *searchModeFlag) ShouldHandleDep(dep deplist.Dependency) bool {
+ res := false
+ for _, searchMode := range *i {
+ switch searchMode {
+ case "deps":
+ res = res || !dep.IsBundled
+ case "bundled":
+ res = res || dep.IsBundled
+ }
+ }
+ return res
+}
+
func main() {
deptypePtr := flag.Int("deptype", -1, "golang, nodejs, python etc")
debugPtr := flag.Bool("debug", false, "debug logging (default false)")
+ var searchModes searchModeFlag
+ flag.Var(&searchModes, "modes", "search mode (bundled, deps)")
flag.Parse()
+ if len(searchModes) == 0 {
+ searchModes = []string{"deps", "bundled"}
+ }
if *debugPtr == true {
log.SetLevel(log.DebugLevel)
}
@@ -33,23 +71,24 @@ func main() {
if *deptypePtr == -1 {
for _, dep := range deps {
+ if !searchModes.ShouldHandleDep(dep) {
+ continue
+ }
version := dep.Version
inst, _ := purl.FromString(fmt.Sprintf("pkg:%s/%s@%s", deplist.GetLanguageStr(dep.DepType), dep.Path, version))
fmt.Print(inst)
- if dep.IsBundled {
- fmt.Print(" [bundled]")
- }
fmt.Println()
}
} else {
deptype := deplist.Bitmask(*deptypePtr)
for _, dep := range deps {
+ if !searchModes.ShouldHandleDep(dep) {
+ continue
+ }
+
if (dep.DepType & deptype) == deptype {
fmt.Printf("%s@%s", dep.Path, dep.Version)
- if dep.IsBundled {
- fmt.Print(" [bundled]")
- }
fmt.Println()
}
}