Add a IsBundled dependency property

When possible differentiate between runtime dependencies and bundled
code. By bundled code we can refer to copy-pasted code, vendored code
(e.g. github, node_modules), copy-pasted code, webpacked, etc.

cmd/deplist/deplist.go

@@ -36,13 +36,21 @@ func main() {
 			version := dep.Version
 
 			inst, _ := purl.FromString(fmt.Sprintf("pkg:%s/%s@%s", deplist.GetLanguageStr(dep.DepType), dep.Path, version))
-			fmt.Println(inst)
+			fmt.Print(inst)
+			if dep.IsBundled {
+				fmt.Print(" [bundled]")
+			}
+			fmt.Println()
 		}
 	} else {
 		deptype := deplist.Bitmask(*deptypePtr)
 		for _, dep := range deps {
 			if (dep.DepType & deptype) == deptype {
-				fmt.Printf("%s@%s\n", dep.Path, dep.Version)
+				fmt.Printf("%s@%s", dep.Path, dep.Version)
+				if dep.IsBundled {
+					fmt.Print(" [bundled]")
+				}
+				fmt.Println()
 			}
 		}
 	}

dependencies.go

@@ -5,10 +5,11 @@ type Bitmask uint32
 
 // Dependency per dependency info
 type Dependency struct {
-	DepType Bitmask  // golang, nodejs, python etc
-	Path    string   // the module path, github.com/teris-io/shortid
-	Version string   // v0.0.0-20171029131806-771a37caa5cf
-	Files   []string // if available, list of all files for a package
+	DepType   Bitmask  // golang, nodejs, python etc
+	Path      string   // the module path, github.com/teris-io/shortid
+	Version   string   // v0.0.0-20171029131806-771a37caa5cf
+	Files     []string // if available, list of all files for a package
+	IsBundled bool     // whether the dependency is bundled somehow in the dependency (e.g. vendored code, copy-pasted code, embedded code, etc.)
 	// /usr/lib/go-1.13/src/regexp/syntax/compile.go
 	// /usr/lib/go-1.13/src/regexp/syntax/doc.go
 }

deplist.go

@@ -80,16 +80,32 @@ func getDeps(fullPath string) ([]Dependency, Bitmask, error) {
 		}
 
 		if info.IsDir() {
-			// prevent walking down the vendors, docs, etc
+			// prevent walking down the docs, .git, tests, etc.
 			if utils.BelongsToIgnoreList(info.Name()) {
 				return filepath.SkipDir
 			}
 		} else {
 			// Two checks, one for filenames and the second switch for full
 			// paths. Useful if we're looking for top of repo
-
 			switch filename := info.Name(); filename {
 			// for now only go for yarn and npm
+			case "package.json":
+				pkg, err := scan.GetNodeJSPackage(path)
+				if err != nil {
+					log.Debugf("failed to scan for nodejs package: %s", path)
+					return nil
+				}
+
+				foundTypes.DepFoundAddFlag(LangNodeJS)
+
+				deps = append(deps,
+					Dependency{
+						DepType:   LangNodeJS,
+						Path:      pkg.Name,
+						Version:   pkg.Version,
+						Files:     []string{},
+						IsBundled: true,
+					})
 			case "package-lock.json":
 				// if theres not a yarn.lock fall thru
 				if _, err := os.Stat(
@@ -160,10 +176,11 @@ func getDeps(fullPath string) ([]Dependency, Bitmask, error) {
 							if !strings.HasSuffix(version, "-javadoc") && !strings.HasSuffix(version, "-sources") {
 								deps = append(deps,
 									Dependency{
-										DepType: LangJava,
-										Path:    name,
-										Version: version,
-										Files:   []string{},
+										DepType:   LangJava,
+										Path:      name,
+										Version:   version,
+										Files:     []string{},
+										IsBundled: true,
 									})
 							}
 						}
@@ -184,10 +201,11 @@ func getDeps(fullPath string) ([]Dependency, Bitmask, error) {
 
 				for path, goPkg := range pkgs {
 					d := Dependency{
-						DepType: LangGolang,
-						Path:    path,
-						Files:   goPkg.Gofiles,
-						Version: goPkg.Version,
+						DepType:   LangGolang,
+						Path:      path,
+						Files:     goPkg.Gofiles,
+						Version:   goPkg.Version,
+						IsBundled: true,
 					}
 					deps = append(deps, d)
 				}
@@ -202,9 +220,10 @@ func getDeps(fullPath string) ([]Dependency, Bitmask, error) {
 				}
 				for _, goPkg := range pkgs {
 					d := Dependency{
-						DepType: LangGolang,
-						Path:    goPkg.Name,
-						Version: goPkg.Version,
+						DepType:   LangGolang,
+						Path:      goPkg.Name,
+						Version:   goPkg.Version,
+						IsBundled: true,
 					}
 					deps = append(deps, d)
 				}
@@ -219,9 +238,10 @@ func getDeps(fullPath string) ([]Dependency, Bitmask, error) {
 				}
 				for _, goPkg := range pkgs {
 					d := Dependency{
-						DepType: LangGolang,
-						Path:    goPkg.Name,
-						Version: goPkg.Version,
+						DepType:   LangGolang,
+						Path:      goPkg.Name,
+						Version:   goPkg.Version,
+						IsBundled: true,
 					}
 					deps = append(deps, d)
 				}

internal/scan/nodejs.go

@@ -3,6 +3,7 @@ package scan
 import (
 	"encoding/json"
 	"fmt"
+	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
@@ -25,6 +26,10 @@ type yarnOutput struct {
 	}
 }
 
+type packageJsonFormat struct {
+	Name    string `json:"name"`
+	Version string `json:"version"`
+}
 type npmDependency struct {
 	Version      string                   `json:"version"`
 	Dependencies map[string]npmDependency `json:"dependencies"`
@@ -121,6 +126,25 @@ func GetNodeJSDeps(path string) (map[string]NodeJSGather, error) {
 	return nil, fmt.Errorf("unknown NodeJS dependency file %q", path)
 }
 
+func GetNodeJSPackage(path string) (NodeJSGather, error) {
+	log.Debugf("GetNodeJSPackage %s", path)
+
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return NodeJSGather{}, err
+	}
+
+	var packageJson packageJsonFormat
+	err = json.Unmarshal(data, &packageJson)
+	if err != nil {
+		return NodeJSGather{}, err
+	}
+	if packageJson.Name == "" {
+		return NodeJSGather{}, fmt.Errorf("Empty package")
+	}
+	return NodeJSGather{Name: packageJson.Name, Version: packageJson.Version}, nil
+}
+
 func getYarnDeps(path string) (map[string]NodeJSGather, error) {
 	var yarnOutput yarnOutput
 	gatheredNode = make(map[string]NodeJSGather)

internal/utils/utils.go

@@ -9,8 +9,6 @@ import (
 func BelongsToIgnoreList(needle string) bool {
 	switch needle {
 	case
-		"node_modules",
-		"vendor",
 		"scripts",
 		"docs",
 		"test",