updates

README.md

@@ -3,7 +3,8 @@ SootFX is a static code feature extraction tool for Java and Android, built to b
 
 # Building
 SootFX is a java project and uses maven. It can be imported as a maven project to your favourite IDE and can be built with maven plugins.
-Or simply build it with:
+It depends on soot-infoflow-android which is not available on maven central, this library can be installed to the local maven repository by running the `install_dependencies.sh` script in `dependencies` folder.
+After that it can be built with:
 
 ```
 mvn install
@@ -22,6 +23,12 @@ Set<MethodFeatureSet> featureSets = sootFX.extractAllMethodFeatures();
 sootFX.printMultiSetToCSV(featureSets, outPath); //path to output csv file
 ```
 ## Python API
+Python requirements are defined in [requirements.txt](SootFXPy/requirements.txt), and can be installed by running:
+
+```
+pip install -r requirements.txt
+```
+
 The Python API can be accessed in two steps:
 1. run [api.SootFXEntryPoint](src/main/java/api/SootFXEntryPoint.java). It starts a Py4J gateway server.
 2. run [main.py in SootFXPy](SootFXPy/main.py), which enables accessing the Java API over the gateway
@@ -46,7 +53,13 @@ df = converter.to_dataframe(extracted_features)
 ```
 
 ## CLI
-[SootFX.jar](out/artifacts/SootFX_jar/SootFX.jar) can be used as a CLI tool.  
+An executable jar can be built with:
+
+```
+mvn package
+```
+
+This will create `SootFX-1.0-SNAPSHOT-jar-with-dependencies.jar` under `target` folder. it can be used as a CLI tool.  
 in case of jar files:
 - to extract all the features:
     ```
@@ -66,4 +79,4 @@ in case of apk files:
     java -jar SootFX.jar "path/to/apk" "path/to/out/" "path/to/config.yaml" "path/to/Android/sdk/platforms"
     ```
 
-inclusion and exclusion lists for different type of feature extraction units can be defined in [config.yaml](config.yaml).
+inclusion and exclusion lists for different type of feature extraction units can be defined in [config.yaml](config.yaml). Make sure to provide either an inclusion list, or an exclusion list. Inclusion list only extracts the selected features. Exclusion list extracts all but the selected features.

SootFXPy/main.py

@@ -1,4 +1,5 @@
 from py4j.java_gateway import JavaGateway
+from IPython.display import display
 import converter
 
 # obtaning the API handle
@@ -19,6 +20,6 @@
 selected_features = gateway.jvm.java.util.ArrayList()
 selected_features.add('MethodAssignStmtCount')
 selected_features.add('MethodBranchCount')
-extracted_features = sootFX.extractMethodFeatures(selected_features)
+extracted_features = sootFX.extractMethodFeaturesInclude(selected_features)
 df = converter.to_dataframe(extracted_features)
-
+display(df)

SootFXPy/requirements.txt

@@ -0,0 +1,3 @@
+ipython==7.27.0
+pandas==1.3.2
+py4j==0.10.9.2

UPDATE.md

@@ -0,0 +1,21 @@
+# Update
+Below we address the issues:
+1. There is no soot-infoflow-android in maven central:
+    - This library is not available in maven central, therefore we now include it in `dependencies` folder. It can be installed to the local maven repository by running the `install_dependencies.sh` script.
+
+2. Pom is not configured to generate a standalone jar:
+    - we now use the maven assembly plugin, a jar with dependencies can be built with `mvn package` this will create `SootFX-1.0-SNAPSHOT-jar-with-dependencies.jar` in `target` folder.
+
+3. Missing python requirements:
+    - Python requirements can be installed with `pip install -r requirements.txt`
+
+4. Fixed python implementation issues.
+
+5. Fixed NullPointerException when methodFeatureInclusion/methodFeatureExclusion is not specified in config.yaml
+
+6. NullPointerException when both Inclusion and Exclusion lists are given:
+    - Only one of the lists must be provided. Now we throw an appropriate exception when both are set.
+
+7. Fixed NullPointerException when file.getParentFile() returns null.
+
+8. SootFX wasn't tested with other java version, therefore we recommend using Java 8.

config.yaml

@@ -1,4 +1,4 @@
 methodFeatureInclusion:
   - MethodAssignStmtCount
-methodFeatureExclusion:
-  - MethodBranchCount
+#methodFeatureExclusion:
+#  - MethodBranchCountK

dependencies/install_dependencies.sh

@@ -0,0 +1 @@
+mvn install:install-file -Dfile=./soot-infoflow-android-2.5.1.jar -DgroupId=de.tud.sse -DartifactId=soot-infoflow-android -Dversion=2.5.1 -Dpackaging=jar -DgeneratePom=true

pom.xml

@@ -20,6 +20,30 @@
                 <artifactId>maven-compiler-plugin</artifactId>
                 <version>3.6.1</version>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                        <configuration>
+                            <archive>
+                                <manifest>
+                                    <mainClass>
+                                        api.CLI
+                                    </mainClass>
+                                </manifest>
+                            </archive>
+                            <descriptorRefs>
+                                <descriptorRef>jar-with-dependencies</descriptorRef>
+                            </descriptorRefs>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
         </plugins>
     </build>
     <dependencies>

src/main/java/api/CLI.java

@@ -4,7 +4,6 @@
 import core.rm.ManifestFeatureSet;
 import core.rm.MethodFeatureSet;
 import core.rm.WholeProgramFeatureSet;
-import org.apache.commons.lang3.ArrayUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.yaml.snakeyaml.Yaml;
 import org.yaml.snakeyaml.constructor.Constructor;
@@ -58,9 +57,10 @@ public static void main(String[] args) throws IOException {
             methodFeatures(classPath, outPath, config.getMethodFeatureInclusion(), config.getMethodFeatureExclusion());
             classFeatures(classPath, outPath, config.getClassFeatureInclusion(), config.getClassFeatureExclusion());
             wpFeatures(classPath, outPath, config.getWholeProgFeatureInclusion(), config.getWholeProgFeatureExclusion());
-            manifestFeatures(classPath, outPath, androidJars, config.getManifestFeatureInclusion(), config.getManifestFeatureExclusion());
+            if(!StringUtils.isEmpty(androidJars) && classPath.endsWith(".apk")) {
+                manifestFeatures(classPath, outPath, androidJars, config.getManifestFeatureInclusion(), config.getManifestFeatureExclusion());
+            }
         }
-
     }
 
 
@@ -71,10 +71,12 @@ public static void methodFeatures(String path, String out, List<String> include,
         Set<MethodFeatureSet> featureSets = null;
         if((include==null || include.isEmpty()) && (exclude==null || exclude.isEmpty())){
             featureSets = sootFX.extractAllMethodFeatures();
-        }else if(include.isEmpty()){
+        }else if(include==null || include.isEmpty()){
             featureSets = sootFX.extractMethodFeaturesExclude(new HashSet<>(exclude));
-        }else if(exclude.isEmpty()){
+        }else if(exclude==null || exclude.isEmpty()){
             featureSets = sootFX.extractMethodFeaturesInclude(include);
+        }else if(!include.isEmpty() && !exclude.isEmpty()){
+            throw new RuntimeException("You must either provide methodFeatureInclusion or methodFeatureExclusion in config.yaml. \n Inclusion list only extracts the selected features. Exclusion list extracts all but the selected features.");
         }
         sootFX.printMultiSetToCSV(featureSets, out + "method.csv");
     }
@@ -86,10 +88,12 @@ public static void classFeatures(String path, String out, List<String> include,
         Set<ClassFeatureSet> featureSets = null;
         if((include==null || include.isEmpty()) && (exclude==null || exclude.isEmpty())){
             featureSets = sootFX.extractAllClassFeatures();
-        }else if(include.isEmpty()){
+        }else if(include==null || include.isEmpty()){
             featureSets = sootFX.extractClassFeaturesExclude(new HashSet<>(exclude));
-        }else if(exclude.isEmpty()){
+        }else if(exclude==null || exclude.isEmpty()){
             featureSets = sootFX.extractClassFeaturesInclude(include);
+        }else if(!include.isEmpty() && !exclude.isEmpty()){
+            throw new RuntimeException("You must either provide classFeatureInclusion or classFeatureExclusion in config.yaml. \n Inclusion list only extracts the selected features. Exclusion list extracts all but the selected features.");
         }
         sootFX.printMultiSetToCSV(featureSets, out + "class.csv");
     }
@@ -101,10 +105,12 @@ public static void wpFeatures(String path, String out, List<String> include, Lis
         WholeProgramFeatureSet featureSet = null;
         if((include==null || include.isEmpty()) && (exclude==null || exclude.isEmpty())){
             featureSet = sootFX.extractAllWholeProgramFeatures();
-        } else if(include.isEmpty()){
+        } else if(include==null || include.isEmpty()){
             featureSet = sootFX.extractWholeProgramFeaturesExclude(new HashSet<>(exclude));
-        } else if(exclude.isEmpty()){
+        } else if(exclude==null || exclude.isEmpty()){
             featureSet = sootFX.extractWholeProgramFeaturesInclude(include);
+        }else if(!include.isEmpty() && !exclude.isEmpty()){
+            throw new RuntimeException("You must either provide wholeProgFeatureInclusion or wholeProgFeatureExclusion  in config.yaml. \n Inclusion list only extracts the selected features. Exclusion list extracts all but the selected features.");
         }
         sootFX.printSingleSetToCSV(featureSet, out + "wp.csv");
     }
@@ -117,10 +123,12 @@ public static void manifestFeatures(String path, String out, String androidJars,
         ManifestFeatureSet featureSet = null;
         if((include==null || include.isEmpty()) && (exclude==null || exclude.isEmpty())){
             featureSet = sootFX.extractAllManifestFeatures();
-        } else if(include.isEmpty()){
+        } else if(include==null || include.isEmpty()){
             featureSet = sootFX.extractManifestFeaturesExclude(new HashSet<>(exclude));
-        } else if(exclude.isEmpty()){
+        } else if(exclude==null || exclude.isEmpty()){
             featureSet = sootFX.extractManifestFeaturesInclude(include);
+        } else if(!include.isEmpty() && !exclude.isEmpty()){
+            throw new RuntimeException("You must either provide manifestFeatureInclusion or manifestFeatureExclusion in config.yaml. \n Inclusion list only extracts the selected features. Exclusion list extracts all but the selected features.");
         }
         sootFX.printSingleSetToCSV(featureSet, out + "manifest.csv");
     }
@@ -129,7 +137,7 @@ public static Config getConfig(String configPath) throws FileNotFoundException {
         Yaml yaml = new Yaml(new Constructor(Config.class));
         InputStream inputStream = new FileInputStream(configPath);
         Config config = yaml.load(inputStream);
-        return config;
+        return config==null ? new Config() : config;
     }
 
 }

src/main/java/api/Config.java

@@ -15,6 +15,7 @@ public class Config {
     private List<String> manifestFeatureInclusion;
     private List<String> manifestFeatureExclusion;
 
+
     public List<String> getMethodFeatureInclusion() {
         return methodFeatureInclusion;
     }

src/main/java/api/SootFX.java

@@ -187,7 +187,9 @@ public void printMethodToCSV(Set<MethodFeatureSet> set, String path) {
     public void printMultiSetToCSV(Set<? extends AbstractFeatureSet> set, String path) throws IOException {
         boolean isFirst = true;
         File file = new File(path);
-        file.getParentFile().mkdirs();
+        if(file.getParentFile()!=null){
+            file.getParentFile().mkdirs();
+        }
         file.createNewFile();
         try (OutputStream out = new FileOutputStream(file);
              Writer writer = new OutputStreamWriter(out, "UTF-8")) {