Merge pull request #103 from fact-project/gzip

Add option for gzip compression in JSONWriter

ChangeLog.md → CHANGELOG.md

@@ -1,5 +1,10 @@
 #Changelog for the fact-tools
 
+## Version 0.12.3
+
+* `JSONWriter` now supports directly writing gzipped files with the 
+option `gzip="true"`
+
 ## Version 0.12.2
 
 Changes:

pom.xml

@@ -4,7 +4,7 @@
   <groupId>de.sfb876</groupId>
   <artifactId>fact-tools</artifactId>
   <name>fact-tools</name>
-  <version>0.12.2</version>
+  <version>0.12.3</version>
   <url>http://sfb876.de/fact-tools/</url>
 
 	<description>

src/main/java/fact/io/JSONWriter.java

@@ -15,13 +15,11 @@
 import stream.annotations.Parameter;
 import stream.data.DataFactory;
 
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
+import java.io.*;
 import java.math.BigDecimal;
 import java.math.MathContext;
 import java.net.URL;
+import java.util.zip.GZIPOutputStream;
 
 
 /**
@@ -88,6 +86,11 @@
  * you can use:
  * <code>append="true"</code>
  * </p>
+ * <p>
+ * The JSONWriter can also add gzip compression on the fly. Use the <code>gzip</code> Option
+ * to directly write gzip compressed files.
+ * <code>append="true"</code>
+ * </p>
  * Created by bruegge on 7/30/14.
  * Refactored by maxnoe on 2/2/2016
  */
@@ -106,6 +109,8 @@ public class JSONWriter implements StatefulProcessor {
     private boolean pixelSetsAsInt = true;
     @Parameter(required = false, description = "If true, Infinity, -Infinity and NaN are converted to strings 'inf', '-inf' and 'nan'", defaultValue = "false")
     private boolean specialDoubleValuesAsString = false;
+    @Parameter(required = false, description = "If true, use gzip compression")
+    private boolean gzip = false;
 
     @Parameter(required = true)
     private URL url;
@@ -154,7 +159,13 @@ public Data process(Data data) {
 
     @Override
     public void init(ProcessContext processContext) throws Exception {
-        bw = new BufferedWriter(new FileWriter(new File(url.getFile()), append));
+        if (gzip){
+            GZIPOutputStream gzip = new GZIPOutputStream(new FileOutputStream(new File(url.getFile()), append));
+            bw = new BufferedWriter(new OutputStreamWriter(gzip, "UTF-8"));
+        }
+        else {
+            bw = new BufferedWriter(new FileWriter(new File(url.getFile()), append));
+        }
 
         GsonBuilder gsonBuilder  = new GsonBuilder().serializeSpecialFloatingPointValues();
         gsonBuilder.enableComplexMapKeySerialization();
@@ -222,6 +233,10 @@ public void setJsonl(boolean jsonl) {
         this.jsonl = jsonl;
     }
 
+    public void setGzip(boolean gzip) {
+        this.gzip = gzip;
+    }
+
     public void setDoubleSignDigits(int doubleSignDigits) {
 		this.doubleSignDigits = doubleSignDigits;
 	}