Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Misc cleanups #24

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Misc cleanups #24

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fd14c9e
WIP
kittylyst Dec 4, 2024
13b829e
Some more initial cleanups
kittylyst Dec 4, 2024
7139dd1
Fix minor stream handling ugliness
kittylyst Dec 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,6 @@
<artifactId>zt-exec</artifactId>
<version>1.12</version>
</dependency>

<!-- TEST -->
<dependency>
<groupId>commons-codec</groupId>
Expand Down
138 changes: 74 additions & 64 deletions src/main/java/com/contrastsecurity/Libraries.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.nio.file.Files;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.util.ArrayList;
Expand All @@ -29,12 +30,14 @@
public class Libraries {

// FIXME: set Scope.EXCLUDED for non-invoked libraries - private Set<Component> invoked = new HashSet<>();
private Set<String> codesourceExamined = new HashSet<>();
private Set<Component> libraries = new HashSet<>();
private Set<org.cyclonedx.model.Dependency> dependencies = new HashSet<>();
private final Set<String> codesourceExamined = new HashSet<>();
private final Set<Component> libraries = new HashSet<>();
private final Set<org.cyclonedx.model.Dependency> dependencies = new HashSet<>();
private Hash rootSHA1;
private Hash rootMD5;

public static final char[] HEX_ARRAY = "0123456789abcdef".toCharArray();

public void runScan(File jarPath) throws Exception {
addAllLibraries( null, jarPath.getAbsolutePath() );
}
Expand All @@ -51,7 +54,7 @@ public void save( String outputPath ) {
}

// find containing jar file and include ALL libraries
public void addAllLibraries( Class clazz, String codesource ) {
public void addAllLibraries( Class<?> clazz, String codesource ) {

// FIXME - change codesourceExamined to a Map<codesource, Library>
// increment library.classesUsed;
Expand All @@ -62,29 +65,31 @@ public void addAllLibraries( Class clazz, String codesource ) {

try {
String filepath = codesource.substring( codesource.lastIndexOf(":") + 1);
String parts[] = filepath.split( "!/" );
String path = parts[0];
String[] parts = filepath.split( "!/" );
String pathStr = parts[0];
if(File.separator.equals("\\")) {
path = path.replace("\\", "/");
pathStr = pathStr.replace("\\", "/");
}
if ( codesourceExamined.contains( path ) ) {
if ( codesourceExamined.contains( pathStr ) ) {
return;
}
codesourceExamined.add( path );
codesourceExamined.add( pathStr );

File f = new File( path );
File f = new File( pathStr );

String sha1 = hash( new FileInputStream( f ), MessageDigest.getInstance("SHA1") );
String sha1 = hash( Files.newInputStream( f.toPath() ), MessageDigest.getInstance("SHA1") );
rootSHA1 = new Hash( Hash.Algorithm.SHA1, sha1 );
String md5 = hash( new FileInputStream( f ), MessageDigest.getInstance("MD5") );

String md5 = hash( Files.newInputStream( f.toPath() ), MessageDigest.getInstance("MD5") );
rootMD5 = new Hash( Hash.Algorithm.MD5, md5 );

// scan for nested libraries
JarInputStream jis3 = new JarInputStream( new FileInputStream( f ) );
JarFile jarfile = new JarFile( f );
scan( jarfile, jis3, f.getAbsolutePath() );
addRootHashesToRootJar();
try (JarInputStream jis3 =
new JarInputStream(Files.newInputStream(f.toPath()));
JarFile jarfile = new JarFile( f )) {
scan( jarfile, jis3, f.getAbsolutePath() );
addRootHashesToRootJar();
}
} catch( Exception e ) {
Logger.log( "The jbom project needs your help to deal with unusual CodeSources." );
Logger.log( "Report issue here: https://github.com/Contrast-Security-OSS/jbom/issues/new/choose" );
Expand All @@ -95,12 +100,12 @@ public void addAllLibraries( Class clazz, String codesource ) {
}

private void addRootHashesToRootJar() {
for( Component lib : libraries.stream()
.filter(lib-> lib.getHashes()==null||lib.getHashes().isEmpty())
.collect(Collectors.toList())) {
lib.addHash(rootSHA1);
lib.addHash(rootMD5);
}
libraries.stream()
.filter(lib -> lib.getHashes() == null || lib.getHashes().isEmpty())
.forEach(lib -> {
lib.addHash(rootSHA1);
lib.addHash(rootMD5);
});
}

public void scan( JarFile jarFile, JarInputStream jis, String codesource ) throws Exception {
Expand Down Expand Up @@ -138,7 +143,6 @@ public void scan( JarFile jarFile, JarInputStream jis, String codesource ) throw
}

public void scanInner( String codesource, JarFile jarFile, JarInputStream jis, JarEntry entry ) throws Exception {

Library innerlib = new Library();
// FIXME: set Scope.EXCLUDED for non-invoked libraries
innerlib.setScope( Scope.REQUIRED );
Expand All @@ -149,41 +153,43 @@ public void scanInner( String codesource, JarFile jarFile, JarInputStream jis, J
libraries.add( innerlib );
innerlib.setType( Library.Type.LIBRARY );

InputStream nis1 = jarFile.getInputStream( entry );
String md5 = hash( nis1, MessageDigest.getInstance("MD5") );
innerlib.addHash( new Hash( Hash.Algorithm.MD5, md5 ) );

InputStream nis2 = jarFile.getInputStream( entry );
String sha1 = hash( nis2, MessageDigest.getInstance("SHA1") );
innerlib.addHash( new Hash( Hash.Algorithm.SHA1, sha1 ) );

innerlib.addProperty( "maven", "https://search.maven.org/search?q=1:" + sha1 );
try (InputStream nis1 = jarFile.getInputStream( entry )) {
String md5 = hash( nis1, MessageDigest.getInstance("MD5") );
innerlib.addHash( new Hash( Hash.Algorithm.MD5, md5 ) );
}

InputStream nis3 = jarFile.getInputStream( entry );
JarInputStream innerJis = new JarInputStream( nis3 );
try (InputStream nis2 = jarFile.getInputStream( entry )) {
String sha1 = hash( nis2, MessageDigest.getInstance("SHA1") );
innerlib.addHash( new Hash( Hash.Algorithm.SHA1, sha1 ) );
innerlib.addProperty( "maven", "https://search.maven.org/search?q=1:" + sha1 );
}

Manifest mf = innerJis.getManifest();
if ( mf != null ) {
Attributes attr = mf.getMainAttributes();
String group = attr.getValue( "Implementation-Vendor-Id" );
String artifact = attr.getValue( "Implementation-Title" );
if ( group != null ) innerlib.setGroup(group);
if ( artifact != null ) innerlib.setName(artifact);
try (InputStream nis3 = jarFile.getInputStream( entry );
JarInputStream innerJis = new JarInputStream( nis3 )) {
Manifest mf = innerJis.getManifest();
if ( mf != null ) {
Attributes attr = mf.getMainAttributes();
String group = attr.getValue( "Implementation-Vendor-Id" );
String artifact = attr.getValue( "Implementation-Title" );
if ( group != null ) innerlib.setGroup(group);
if ( artifact != null ) innerlib.setName(artifact);
}
}

// scan through this jar to find any pom files
InputStream nis4 = jarFile.getInputStream( entry );
JarInputStream innerJis4 = new JarInputStream( nis4 );
while ((entry = innerJis4.getNextJarEntry()) != null) {
if ( isPom(entry) ) {
try {
parsePom( innerJis4, innerlib );
} catch( Exception e ) {
// Logger.log( "Problem parsing POM from " + nestedName + " based on " + codesource + ". Continuing." );
try (InputStream nis4 = jarFile.getInputStream( entry );
JarInputStream innerJis4 = new JarInputStream( nis4 )) {
while ((entry = innerJis4.getNextJarEntry()) != null) {
if ( isPom(entry) ) {
try {
parsePom( innerJis4, innerlib );
} catch( Exception e ) {
// Logger.log( "Problem parsing POM from " + nestedName + " based on " + codesource + ". Continuing." );
}
}
}
}

try {
if ( innerlib.getGroup() != null && innerlib.getName() != null ) {
innerlib.setPurl(new PackageURL( PackageURL.StandardTypes.MAVEN, innerlib.getGroup(), innerlib.getName(), innerlib.getVersion(), null, null));
Expand Down Expand Up @@ -222,27 +228,29 @@ private void parsePom(JarInputStream is, Library lib) throws Exception {
MavenXpp3Reader reader = new MavenXpp3Reader();
Model model = reader.read(is);
// Model model = reader.read(new StringReader( pom ));
String g = model.getGroupId();
String a = model.getArtifactId();
String v = model.getVersion();
if ( g != null ) lib.setGroup( g );
if ( a != null ) lib.setName( a );
if ( v != null ) lib.setVersion( v );
String group = model.getGroupId();
String artifact = model.getArtifactId();
String version = model.getVersion();

// Null handling should be more robust?
if ( group != null ) lib.setGroup( group );
if ( artifact != null ) lib.setName( artifact );
if ( version != null ) lib.setVersion( version );
lib.setBomRef(getUniqueRef(lib.getGroup(),lib.getName(),lib.getVersion()));
org.cyclonedx.model.Dependency cycloneDep = new org.cyclonedx.model.Dependency(getUniqueRef(lib.getGroup(),lib.getName(),lib.getVersion()));
for(Dependency dep : model.getDependencies()) {
for (Dependency dep : model.getDependencies()) {
org.cyclonedx.model.Dependency subDep = new org.cyclonedx.model.Dependency(getUniqueRef(dep.getGroupId(),dep.getArtifactId(),dep.getVersion()));
cycloneDep.addDependency(subDep);
}
dependencies.add(cycloneDep);
}

public List<Component> getLibraries() {
return new ArrayList<Component>(libraries);
return new ArrayList<>(libraries);
}

public List<org.cyclonedx.model.Dependency> getDependencies() {
return new ArrayList<org.cyclonedx.model.Dependency>(dependencies);
return new ArrayList<>(dependencies);
}

public void dump() {
Expand All @@ -259,19 +267,21 @@ public String getPOM( InputStream is ) throws Exception {
while ((len = is.read(buf, 0, buf.length)) != -1) {
baos.write(buf, 0, len);
}
return new String( baos.toByteArray(), "UTF-8" );
return baos.toString("UTF-8");
}

// streaming hash, low memory use
public static String hash( InputStream is, MessageDigest md ) throws Exception {
DigestInputStream dis = new DigestInputStream(is, md);
byte[] buf = new byte[8192];
for (int len; (len = dis.read(buf)) != -1;) {
int len;

try (DigestInputStream dis = new DigestInputStream(is, md)) {
while ((len = dis.read(buf)) != -1) {
}
}
return toHexString(md.digest());
}

public static final char[] HEX_ARRAY = "0123456789abcdef".toCharArray();
public static String toHexString(byte[] bytes) {
char[] hexChars = new char[bytes.length * 2];
for (int j = 0; j < bytes.length; j++) {
Expand Down
6 changes: 6 additions & 0 deletions src/test/java/com/contrastsecurity/LibrariesTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
import java.io.FileInputStream;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.security.MessageDigest;
import java.util.Optional;

import org.apache.commons.codec.digest.DigestUtils;
Expand All @@ -25,6 +27,10 @@ public void testFile() throws Exception {
Libraries libs = jbom.doLocalFile( jar, "target/test" );
assertTrue( "Incorrect number of libraries found. " + libs.getLibraries().size() + " instead of 135", libs.getLibraries().size() == 135 );
compareHashToFile(jar,libs,"petclinic");
assertEquals("8ef0a3efc567782abda2feac290ce1ffad85f045", hashFileSHA1(jar));

String sha1 = Libraries.hash(Files.newInputStream(jar.toPath()), MessageDigest.getInstance("SHA1") );
assertEquals("8ef0a3efc567782abda2feac290ce1ffad85f045", sha1);
}

@Test
Expand Down