Merge pull request #391 from fact-project/empty_fits

Better error message for empty FITS files

.travis.yml

@@ -2,7 +2,7 @@ language: java
 sudo: false
 script: mvn clean verify
 jdk:
-- oraclejdk8
+- openjdk8
 
 after_script:
 - bash ./deploy_site.sh
@@ -25,7 +25,7 @@ deploy:
   on:
     tags: true
     repo: fact-project/fact-tools
-    java: oraclejdk8
+    java: openjdk8
 
 notifications:
   slack:

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog for the fact-tools
 
+# Version 1.1.2 -- 11.10.2019
+
+* Better error message for broken FITS files 
+* Allow only skipping empty files in FileListMultiStream
+
 # Version 1.1.2 -- 17.12.2018
 
 * Fix reading of uncompressed values from zfits heap tiles (Issue #389)

examples/forErna/std_analysis_erna_simulations.xml

@@ -14,7 +14,7 @@
   <service id="gainService" class="fact.gainservice.GainService" />
 
   <!-- Has to be a  FactFileListMultiStream in order to work on tasks from gridmap and the executor script.-->
-  <stream id="fact" class="fact.io.FactFileListMultiStream"   url="${input}">
+  <stream id="fact" class="fact.io.FactFileListMultiStream" skipEmpty="true" url="${input}">
     <stream class="fact.io.CeresStream" id="_" />
   </stream>
 

pom.xml

@@ -4,7 +4,7 @@
   <groupId>de.sfb876</groupId>
   <artifactId>fact-tools</artifactId>
   <name>fact-tools</name>
-  <version>1.1.2</version>
+  <version>1.1.3</version>
   <url>http://sfb876.de/fact-tools/</url>
 
   <description>

src/main/java/fact/io/FactFileListMultiStream.java

@@ -2,6 +2,9 @@
 
 import com.google.gson.Gson;
 import com.google.gson.reflect.TypeToken;
+import fact.io.hdureader.FITSStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import stream.Data;
 import stream.annotations.Parameter;
 import stream.io.AbstractStream;
@@ -43,6 +46,8 @@
  */
 public class FactFileListMultiStream extends AbstractMultiStream {
 
+    Logger log = LoggerFactory.getLogger(FactFileListMultiStream.class);
+
     private DataDrsPair dataDrsPair;
 
     public FactFileListMultiStream(SourceURL url) {
@@ -67,16 +72,19 @@ public DataDrsPair(String dataFile, String drsFile) {
 
 
     @Parameter(required = true, description = "A file containing a json array of dicts with the paths to the files.")
-    private SourceURL url = null;
+    public SourceURL url = null;
 
     @Parameter(required = false, description = "Flag indicating whether next file should be tried in case of errors in underlying stream.", defaultValue = "false")
-    private boolean skipErrors = false;
+    public boolean skipErrors = false;
+
+    @Parameter(required = false, description = "Similar to skipErrors but will only skip the MissingHDU exception, i.e. empty files", defaultValue = "false")
+    public boolean skipEmpty = false;
 
     @Parameter(required = false, defaultValue = "drs_path")
-    private String drsPathKey = "drs_path";
+    public String drsPathKey = "drs_path";
 
     @Parameter(required = false, defaultValue = "data_path")
-    private String dataPathKey = "data_path";
+    public String dataPathKey = "data_path";
 
     //counts how many files have been processed
     private int filesCounter = 0;
@@ -91,6 +99,9 @@ public DataDrsPair(String dataFile, String drsFile) {
      */
     @Override
     public void init() throws Exception {
+        log.info("Skipping empty files: {}", skipEmpty);
+        log.info("Skipping broken files: {}", skipErrors);
+
         if (!fileQueue.isEmpty()) {
             log.debug("files already loaded");
             return;
@@ -155,11 +166,22 @@ public Data readNext() throws Exception {
             data.put("@drsFile", dataDrsPair.drsFile);
             return data;
 
+
+        } catch (FITSStream.MissingHDUException e) {
+            log.info("File: {} does not contain the Events HDU", stream.getUrl());
+            if (skipEmpty) {
+                log.warn("Skipping empty file {}. Continuing with next file.", stream.getUrl());
+                stream = null;
+                return this.readNext();
+            } else {
+                log.error("Stopping stream because of missing HDU in one input file");
+                stream.close();
+                throw new RuntimeException(e);
+            }
         } catch (IOException e) {
             log.info("File: " + stream.getUrl().toString() + " throws IOException.");
-
             if (skipErrors) {
-                log.info("Skipping broken files. Continuing with next file.");
+                log.warn("Skipping broken file {}. Continuing with next file.", stream.getUrl());
                 e.printStackTrace();
                 stream = null;
                 return this.readNext();
@@ -177,20 +199,9 @@ public void close() throws Exception {
         log.info("In total {} files were processed.", filesCounter);
     }
 
+
+    @Override
     public void setUrl(SourceURL url) {
         this.url = url;
     }
-
-    public void setDrsPathKey(String drsPathKey) {
-        this.drsPathKey = drsPathKey;
-    }
-
-    public void setDataPathKey(String dataPathKey) {
-        this.dataPathKey = dataPathKey;
-    }
-
-    public void setSkipErrors(boolean skipErrors) {
-        this.skipErrors = skipErrors;
-    }
-
 }

src/main/java/fact/io/hdureader/FITS.java

@@ -102,7 +102,11 @@ public FITS(URL url) throws IOException {
             }
 
         } catch (EOFException e) {
-            primaryHDU = hdus.get(0);
+            if (hdus.size() < 1) {
+                log.warn("File '{}' does not contain any HDUs", url);
+            } else {
+                primaryHDU = hdus.get(0);
+            }
             stream.close();
 
             log.debug("A total of {} HDUs were found in the file.", hdus.size());

src/main/java/fact/io/hdureader/FITSStream.java

@@ -72,7 +72,7 @@ public void init() throws Exception {
 
         }
         // create a refrence to the events hdu
-        eventHDU = fits.getHDU(nameHDU).orElseThrow(() -> new IOException("Inputfile did not contain HDU '" + nameHDU + "'"));
+        eventHDU = fits.getHDU(nameHDU).orElseThrow(() -> new MissingHDUException("Inputfile did not contain HDU '" + nameHDU + "'"));
 
         BinTable eventsTable = eventHDU.getBinTable();
 
@@ -137,4 +137,10 @@ public Data readNext() throws Exception {
     public void skipRows(int amount) throws IOException{
         reader.skipRows(amount);
     }
+
+    public class MissingHDUException extends IOException {
+        public MissingHDUException(String errorMessage) {
+            super(errorMessage);
+        }
+    }
 }

src/test/java/fact/io/FITSStreamTest.java

@@ -61,4 +61,12 @@ public void testFitsKeys() {
         }
     }
 
+    @Test(expected=FITSStream.MissingHDUException.class)
+    public void testEmptyFile() throws Exception {
+        URL u = FITSStreamTest.class.getResource("/empty_fits.fits.gz");
+        SourceURL url = new SourceURL(u);
+        FITSStream stream = new FITSStream(url);
+        stream.init();
+    }
+
 }

src/test/java/fact/io/ListMultiStreamTest.java

@@ -4,12 +4,13 @@
 import org.junit.Test;
 import stream.Data;
 import stream.io.SourceURL;
+import stream.shell.Run;
 
 import java.io.File;
 import java.net.URL;
 
 import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertThat;
+import static org.junit.Assert.*;
 
 /**
  * Created by kaibrugge on 14.04.15.
@@ -35,12 +36,45 @@ public void testDrsInjection() throws Exception {
 
         FITSStream m = new FITSStream();
         multiStream.addStream("test", m);
+        multiStream.skipEmpty = true;
 
         multiStream.init();
 
         Data data = multiStream.readNext();
+        assertTrue(multiStream.skipEmpty);
         File drsFile = (File) data.get("@drsFile");
 
         assertThat(drsFile.getName(), is("20140920_66.drs.fits"));
     }
+
+    @Test
+    public void testNotSkipEmpty () throws Exception {
+        URL u = FITSStreamTest.class.getResource("/dummy_files/file_list_empty.json");
+
+        FactFileListMultiStream multiStream = new FactFileListMultiStream(new SourceURL(u));
+        multiStream.setUrl(new SourceURL(u));
+        FITSStream m = new FITSStream();
+        multiStream.addStream("test", m);
+        multiStream.init();
+
+        try {
+            Data data = multiStream.readNext();
+            fail("Runtime exception expected");
+        } catch (RuntimeException e) {}
+
+    }
+
+    @Test
+    public void testSkipEmpty () throws Exception {
+        URL u = FITSStreamTest.class.getResource("/dummy_files/file_list_empty.json");
+
+        FactFileListMultiStream multiStream = new FactFileListMultiStream(new SourceURL(u));
+        multiStream.skipEmpty = true;
+        multiStream.setUrl(new SourceURL(u));
+        FITSStream m = new FITSStream();
+        multiStream.addStream("test", m);
+        multiStream.init();
+
+        Data data = multiStream.readNext();
+    }
 }

src/test/resources/dummy_files/file_list_empty.json

@@ -0,0 +1,4 @@
+[
+  {"drs_path":"src\/main\/resources\/dummy_files\/20140920_66.drs.fits","later":"src\/main\/resources\/20140920_66.drs.fits","night":"20140920","data_path":"src\/test\/resources\/empty_fits.fits.gz"},
+  {"drs_path":"src\/main\/resources\/dummy_files\/20140920_66.drs.fits","later":"src\/main\/resources\/20140920_66.drs.fits","night":"20140920","data_path":"src\/main\/resources\/testDataFile.fits.gz"}
+]