Skip to content

Commit

Permalink
Add flog for aggregating directory output into single file.
Browse files Browse the repository at this point in the history
  • Loading branch information
@daveneiman
daveneiman committed Dec 19, 2019
1 parent e8a2adc commit 88bb454
Show file tree
Hide file tree
Showing 2 changed files with 63 additions and 28 deletions.
18 changes: 11 additions & 7 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.harvard.huit.lts</groupId>
<artifactId>fits</artifactId>
<version>1.5.0</version>
<version>1.5.1-SNAPSHOT</version>
<packaging>jar</packaging>

<name>FITS</name>
Expand Down Expand Up @@ -42,7 +42,11 @@
<nzmetool.version>3.6GA</nzmetool.version>
<xerces-impl.version>2.9.1</xerces-impl.version>
<saxon.version>9.8.0-10</saxon.version>
<mvn-compiler-plgn.version>3.0</mvn-compiler-plgn.version>
<mvn-compiler-plugin.version>3.8.1</mvn-compiler-plugin.version>
<mvn-jar-plugin.version>3.2.0</mvn-jar-plugin.version>
<mvn-assembly-plugin.version>3.2.0</mvn-assembly-plugin.version>
<mvn-deploy-plugin.version>2.8.2</mvn-deploy-plugin.version>
<mvn-javadoc-plugin.version>3.1.1</mvn-javadoc-plugin.version>
</properties>

<build>
Expand All @@ -62,7 +66,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>${mvn-compiler-plgn.version}</version>
<version>${mvn-compiler-plugin.version}</version>
<configuration>
<source>${java.version}</source>
<target>${java.target}</target>
Expand All @@ -74,7 +78,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.0.2</version>
<version>${mvn-jar-plugin.version}</version>
<configuration>
<archive>
<manifest>
Expand All @@ -89,7 +93,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.1.0</version>
<version>${mvn-assembly-plugin.version}</version>
<configuration>
<descriptors>
<descriptor>src/assembly/assembly.xml</descriptor>
Expand All @@ -111,12 +115,12 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.2</version>
<version>${mvn-deploy-plugin.version}</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>3.0.0</version>
<version>${mvn-javadoc-plugin.version}</version>
<executions>
<execution>
<goals>
Expand Down
73 changes: 52 additions & 21 deletions src/main/java/edu/harvard/hul/ois/fits/Fits.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.lang.reflect.Constructor;
import java.net.URI;
import java.net.URISyntaxException;
Expand Down Expand Up @@ -49,6 +50,7 @@
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.jdom.Document;
import org.jdom.Element;
import org.jdom.output.Format;
import org.jdom.output.XMLOutputter;

Expand Down Expand Up @@ -76,6 +78,7 @@ public class Fits {

private static boolean traverseDirs;
private static boolean nestDirs; // whether traversing nested directories of input files creates nest output directories - if false, all output goes in same output directory
private static boolean aggregateOutputDir; // put output from all files in a (nested) directory into the same output file.
private static XMLOutputFactory xmlOutputFactory = XMLOutputFactory.newInstance();

private static final String FITS_CONFIG_FILE_NAME = "fits.xml";
Expand Down Expand Up @@ -254,6 +257,7 @@ public static void main( String[] args ) throws FitsException, IOException, Pars
options.addOption( "r", false, "process directories recursively when -i is a directory " );
options.addOption( "n", false, "output directories are nested when recursively processing nested directories (when -r is set and -i is a directory) (optional)" );
options.addOption( "o", true, "Directs the FITS output to a file or directory (if -i is a directory) rather than console" );
options.addOption( "a", false, "Aggregate all file output into a siglle file (only if -o references a directory)" );
options.addOption( "h", false, "print this message" );
options.addOption( "v", false, "print version information" );
options.addOption( "f", true, "alternate fits.xml configuration file location (optional)" );
Expand Down Expand Up @@ -291,6 +295,11 @@ public static void main( String[] args ) throws FitsException, IOException, Pars
} else {
nestDirs = false;
}
if (cmd.hasOption( "a" )) {
aggregateOutputDir = true;
} else {
aggregateOutputDir = false;
}

File fitsConfigFile = null;
try {
Expand All @@ -316,7 +325,16 @@ public static void main( String[] args ) throws FitsException, IOException, Pars
"When FITS is run in directory processing mode the output location must be a directory." );
}
Fits fits = constructFits(fitsConfigFile);
fits.doDirectory( inputFile, new File( outputDir ), cmd.hasOption( "x" ), cmd.hasOption( "xc" ) );
Element fitsCollectionElem = new Element("fitsCollection");
fitsCollectionElem = fits.doDirectory( inputFile, new File( outputDir ), cmd.hasOption( "x" ), cmd.hasOption( "xc" ), fitsCollectionElem );
// if aggregating output to a single file then output for each file will not have been output to disk
if (aggregateOutputDir && fitsCollectionElem.getContentSize() > 0) {
Document rootDoc = new Document(fitsCollectionElem);
XMLOutputter serializer = new XMLOutputter(Format.getPrettyFormat());
OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(new File(outputDir, fitsCollectionElem.getName() + ".xml")),"UTF-8");
serializer.output(rootDoc, out);
}

} else { // inputFile is a file so output -o must either be a file or not set at all
String outputFile = cmd.getOptionValue( "o" );
if (outputFile != null && (new File( outputFile ).isDirectory())) {
Expand Down Expand Up @@ -386,21 +404,25 @@ private static void setFitsVersionFromFile() {
}

/**
* Recursively processes all files in the directory.
* Processes all files in the in the input directory and, if required, recursively process files in any nested directories.
* Also, output may go into nested directories in output destination if required by application parameter.
* @param useStandardSchemas
* @param fitsAggregateElem TODO
* @param intputFile
*
* @param intputFile
* @param useStandardSchemas
* @throws IOException
* @return TODO
* @throws IOException
* @throws XMLStreamException
* @throws FitsException
*/
private void doDirectory(File inputDir, File outputDir, boolean useStandardSchemas, boolean standardCombinedFormat) throws FitsException, XMLStreamException, IOException {
private Element doDirectory(File inputDir, File outputDir, boolean useStandardSchemas, boolean standardCombinedFormat, Element fitsAggregateElem) throws FitsException, XMLStreamException, IOException {
if(inputDir.listFiles() == null) {
return;
return fitsAggregateElem;
}

logger.info("Processing directory " + inputDir.getAbsolutePath());

//Element rootElement = new Element("fitsCollection");
for (File f : inputDir.listFiles()) {

if(f == null || !f.exists() || !f.canRead()) {
Expand All @@ -412,13 +434,13 @@ private void doDirectory(File inputDir, File outputDir, boolean useStandardSchem
if (f.isDirectory() && traverseDirs) {
// need to reset original directory after return from recursive call when nesting output
File savedDir = outputDir;
if (nestDirs) {
if (nestDirs && !aggregateOutputDir) {
outputDir = new File(outputDir, f.getName());
if ( !outputDir.exists()) {
outputDir.mkdir();
}
}
doDirectory(f, outputDir, useStandardSchemas, standardCombinedFormat);
doDirectory(f, outputDir, useStandardSchemas, standardCombinedFormat, fitsAggregateElem);
outputDir = savedDir;
} else if (f.isFile()) {
if (".DS_Store".equals(f.getName())) {
Expand All @@ -427,23 +449,32 @@ private void doDirectory(File inputDir, File outputDir, boolean useStandardSchem
continue;
}
FitsOutput result = doSingleFile(f);
String outputFile = outputDir.getPath() + File.separator + f.getName() + "." + FITS_CONFIG_FILE_NAME;
File output = new File(outputFile);
if (output.exists()) {
int cnt = 1;
while (true) {
outputFile = outputDir.getPath() + File.separator + f.getName() + "-" + cnt + "." + FITS_CONFIG_FILE_NAME;
output = new File(outputFile);
if (!output.exists()) {
break;
if (aggregateOutputDir) {
// aggregate each file's data for output to a single file
Element fitsElement = result.getFitsXml().detachRootElement();
fitsAggregateElem.addContent(fitsElement);
} else {
// output as file
String outputFile = outputDir.getPath() + File.separator + f.getName() + "." + FITS_CONFIG_FILE_NAME;
File output = new File(outputFile);
if (output.exists()) {
int cnt = 1;
while (true) {
outputFile = outputDir.getPath() + File.separator + f.getName() + "-" + cnt + "." + FITS_CONFIG_FILE_NAME;
output = new File(outputFile);
if (!output.exists()) {
break;
}
cnt++;
}
cnt++;
}
outputResults(result, outputFile, useStandardSchemas,
standardCombinedFormat, true);
}
outputResults(result, outputFile, useStandardSchemas,
standardCombinedFormat, true);
}
}

return fitsAggregateElem;
}

/**
Expand Down

0 comments on commit 88bb454

Please sign in to comment.