Merge pull request #348 from fact-project/parameter_improvment

Parameter improvment and some small new abilities

src/main/java/fact/Utils.java

@@ -534,4 +534,34 @@ public static double[] flatten2dArray(double[][] array2d) {
                 .toArray();
     }
 
+
+    /**
+     * Check if the dataitem has the timestamp key, if not return the MC data default timestamp
+     *
+     * @param item The event to process
+     * @return The timestamp of the event
+     */
+    public static ZonedDateTime getTimeStamp(Data item) {
+        return getTimeStamp(item, "timestamp");
+    }
+
+    /**
+     * Check if the dataitem has the timestamp key, if not return the MC data default timestamp
+     *
+     * @param item The event to process
+     * @return The timestamp of the event
+     */
+    public static ZonedDateTime getTimeStamp(Data item, String timeStampKey) {
+        ZonedDateTime timeStamp = null;
+        if (item.containsKey(timeStampKey)) {
+            Utils.isKeyValid(item, timeStampKey, ZonedDateTime.class);
+            timeStamp = (ZonedDateTime) item.get(timeStampKey);
+        } else {
+            // MC Files don't have a UnixTimeUTC in the data item. Here the timestamp is hardcoded to 1.1.2000
+            // => The 12 bad pixels we have from the beginning on are used.
+            timeStamp = ZonedDateTime.of(2000, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC);
+        }
+
+        return timeStamp;
+    }
 }

src/main/java/fact/cleaning/TwoLevelTimeMedian.java

@@ -66,16 +66,7 @@ public class TwoLevelTimeMedian extends BasicCleaning implements Processor {
     @Override
     public Data process(Data item) {
 
-        ZonedDateTime timeStamp;
-        if (item.containsKey("UnixTimeUTC") == true) {
-            Utils.isKeyValid(item, "UnixTimeUTC", int[].class);
-            int[] eventTime = (int[]) item.get("UnixTimeUTC");
-            timeStamp = Utils.unixTimeUTCToZonedDateTime(eventTime);
-        } else {
-            // MC Files don't have a UnixTimeUTC in the data item. Here the timestamp is hardcoded to 1.1.2000
-            // => The 12 bad pixels we have from the beginning on are used.
-            timeStamp = ZonedDateTime.of(2000, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC);
-        }
+        ZonedDateTime timeStamp = Utils.getTimeStamp(item);
 
         double[] photonCharge = Utils.toDoubleArray(item.get(photonChargeKey));
         double[] arrivalTimes = Utils.toDoubleArray(item.get(arrivalTimeKey));

src/main/java/fact/cleaning/TwoLevelTimeNeighbor.java

@@ -77,16 +77,7 @@ public Data process(Data item) {
         Utils.isKeyValid(item, arrivalTimeKey, double[].class);
         Utils.isKeyValid(item, photonChargeKey, double[].class);
 
-        ZonedDateTime timeStamp = null;
-        if (item.containsKey("UnixTimeUTC") == true) {
-            Utils.isKeyValid(item, "UnixTimeUTC", int[].class);
-            int[] eventTime = (int[]) item.get("UnixTimeUTC");
-            timeStamp = Utils.unixTimeUTCToZonedDateTime(eventTime);
-        } else {
-            // MC Files don't have a UnixTimeUTC in the data item. Here the timestamp is hardcoded to 1.1.2000
-            // => The 12 bad pixels we have from the beginning on are used.
-            timeStamp = ZonedDateTime.of(2000, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC);
-        }
+        ZonedDateTime timeStamp = Utils.getTimeStamp(item);
 
         double[] photonCharge = Utils.toDoubleArray(item.get(photonChargeKey));
         double[] arrivalTimes = Utils.toDoubleArray(item.get(arrivalTimeKey));

src/main/java/fact/datacorrection/DrsCalibration.java

@@ -4,6 +4,7 @@
 package fact.datacorrection;
 
 import fact.Constants;
+import fact.Utils;
 import fact.io.hdureader.BinTable;
 import fact.io.hdureader.BinTableReader;
 import fact.io.hdureader.FITS;
@@ -26,23 +27,36 @@
  * either as File or URL and will read the DRS data from that. This data is then
  * applied to all FactEvents processed by this class.
  *
- * @author Christian Bockermann <[email protected]>
+ * @author Christian Bockermann <[email protected]> , Michael Bulinski <[email protected]>
  */
 public class DrsCalibration implements StatefulProcessor {
     static Logger log = LoggerFactory.getLogger(DrsCalibration.class);
 
-    @Parameter
+    @Parameter(required = false, description = "The data key that will hold the resulting data array.")
     public String outputKey = "DataCalibrated";
 
     @Parameter(required = false, description = "Data array to be calibrated", defaultValue = "Data")
     public String key = "Data";
 
+    @Parameter(required = false, description = "Key to the StartCellData.")
+    public String startCellKey = "StartCellData";
+
     @Parameter(required = false, description = "A URL to the DRS calibration data (in FITS formats)",
             defaultValue = "Null. Will try to find path to drsFile from the stream.")
     public String url = "@drsFile";
 
     public URL drsFileURL = null;
 
+    @Parameter(required = false, description = "The name of the key that holds the drs filename.",
+            defaultValue = "@drsFile")
+    public String drsKey = "@drsFile";
+
+    @Parameter(required = false, description = "Whether to reverse the process.", defaultValue = "false")
+    public boolean reverse = false;
+
+    private double dconv = 2000.0f / 4096.0f;
+
+    Data drsData = null;
 
     private File currentDrsFile = new File("");
 
@@ -86,7 +100,6 @@ protected void loadDrsData(URL in) {
             this.drsTriggerOffsetRms = row.getFloatArray("TriggerOffsetRms").orElseThrow(() -> new RuntimeException("File does not contain key TriggerOffsetRms"));
             this.drsGainMean = row.getFloatArray("GainMean").orElseThrow(() -> new RuntimeException("File does not contain key GainMean"));
             this.drsGainRms = row.getFloatArray("GainRms").orElseThrow(() -> new RuntimeException("File does not contain key GainRms"));
-
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
@@ -101,7 +114,7 @@ public Data process(Data data) {
 
         if (this.drsFileURL == null) {
             //file not loaded yet. try to find by magic.
-            File drsFile = (File) data.get("@drsFile");
+            File drsFile = (File) data.get(drsKey);
             if (drsFile != null) {
                 if (!drsFile.equals(currentDrsFile)) {
                     currentDrsFile = drsFile;
@@ -110,55 +123,70 @@ public Data process(Data data) {
                         loadDrsData(drsFile.toURI().toURL());
                     } catch (MalformedURLException e) {
                         //pass.
+                        throw new RuntimeException("URL malformed");
                     }
                 }
             } else {
                 throw new IllegalArgumentException("No drs file set or no @drsFile key in data stream");
             }
         }
 
+        double[] rawfloatData;
         log.debug("Processing Data item by applying DRS calibration...");
-        short[] rawData = (short[]) data.get(key);
-        if (rawData == null) {
-            log.error(" data .fits file did not contain the value for the key "
-                    + key + ". cannot apply drscalibration");
-            throw new RuntimeException(
-                    " data .fits file did not contain the value for the key \"" + key + "\". Cannot apply drs calibration)");
-        }
+        if (!reverse) {
+            short[] rawData = (short[]) data.get(key);
+            if (rawData == null) {
+                log.error(" data .fits file did not contain the value for the key "
+                        + key + ". cannot apply drscalibration");
+                throw new RuntimeException(
+                        " data .fits file did not contain the value for the key \""
+                        + key + "\". Cannot apply drs calibration)");
+            }
 
-        double[] rawfloatData = new double[rawData.length];
-        // System.arraycopy(rawData, 0, rawfloatData, 0, rawfloatData.length);
-        for (int i = 0; i < rawData.length; i++) {
-            rawfloatData[i] = rawData[i];
+            rawfloatData = new double[rawData.length];
+            for (int i = 0; i < rawData.length; i++) {
+                rawfloatData[i] = rawData[i];
+            }
+        } else {
+            Utils.isKeyValid(data, key, double[].class);
+            rawfloatData = (double[]) data.get(key);
         }
 
-        short[] startCell = (short[]) data.get("StartCellData");
+        short[] startCell = (short[]) data.get(startCellKey);
         if (startCell == null) {
             log.error(" data .fits file did not contain startcell data. cannot apply drscalibration");
             return null;
         }
-        log.debug("raw data has {} elements", rawData.length);
+        log.debug("raw data has {} elements", rawfloatData.length);
         log.debug("StartCellData has {} elements", startCell.length);
 
         double[] output = rawfloatData;
         if (!key.equals(outputKey)) {
-            output = new double[rawData.length];
+            output = new double[rawfloatData.length];
         }
 
-        double[] calibrated = applyDrsCalibration(rawfloatData, output, startCell);
-
-        data.put(outputKey, calibrated);
-
-        // add color value if set
+        if (!reverse) {
+            double[] calibrated = applyDrsCalibration(rawfloatData, output, startCell);
+            data.put(outputKey, calibrated);
+        } else {
+            double[] calibrated = reverseDrsCalibration(rawfloatData, output, startCell);
+            short[] decalibratedShortData = new short[calibrated.length];
+            for (int i = 0; i < calibrated.length; i++) {
+                decalibratedShortData[i] = (short) calibrated[i];
+            }
+            data.put(outputKey, decalibratedShortData);
+        }
 
         return data;
     }
 
     public double[] applyDrsCalibration(double[] data, double[] destination,
                                         short[] startCellVector) {
 
-        if (destination == null || destination.length != data.length)
+        if (destination == null)
             destination = new double[data.length];
+        else if (destination.length != data.length)
+            throw new RuntimeException("The data array and the destination array have different lengths, "+data.length+" vs "+destination.length);
         int roi = data.length / Constants.N_PIXELS;
 
         // We do not entirely know how the calibration constants, which are
@@ -227,7 +255,6 @@ public double[] applyDrsCalibration(double[] data, double[] destination,
         // TrueValue[c][s] = ( RawValue[c][s] - Offset[c][ (c+t)%1024 ] ) /
         // Gain[c][ (c+t)%1024 ] * 1907.35 - TriggerOffset[c][s]
 
-        double dconv = 2000.0f / 4096.0f;
         double vraw;
 
         int pos, offsetPos, triggerOffsetPos;
@@ -236,15 +263,15 @@ public double[] applyDrsCalibration(double[] data, double[] destination,
 
                 pos = pixel * roi + slice;
                 // Offset and Gain vector *should look the same
-                int start = startCellVector[pixel] != -1 ? startCellVector[pixel]
-                        : 0;
+                int start = startCellVector[pixel] != -1 ? startCellVector[pixel] : 0;
 
                 offsetPos = pixel * drsBaselineMean.length / Constants.N_PIXELS
                         + ((slice + start) % (drsBaselineMean.length / Constants.N_PIXELS));
 
                 triggerOffsetPos = pixel * drsTriggerOffsetMean.length / Constants.N_PIXELS + slice;
 
-                vraw = data[pos] * dconv;
+                vraw = data[pos];
+                vraw *= dconv;
                 vraw -= drsBaselineMean[offsetPos];
                 vraw -= drsTriggerOffsetMean[triggerOffsetPos];
                 vraw /= drsGainMean[offsetPos];
@@ -266,6 +293,50 @@ public double[] applyDrsCalibration(double[] data, double[] destination,
         return destination;
     }
 
+    /**
+     * Reverses the drsCalibration performed in applyDrsCalibration.
+     *
+     * @param data The calibrated data the decalibrate.
+     * @param destination If given use this as the destination array otherwise a new one is created.
+     * @param startCellVector The array containing the start cells used to know which calibration constants to use.
+     * @return The decalibrated data array
+     */
+    public double[] reverseDrsCalibration(double[] data, double[] destination, short[] startCellVector) {
+        if (destination == null)
+            destination = new double[data.length];
+        else if (destination.length != data.length)
+            throw new RuntimeException("The data array and the destination array have different lengths, "+data.length+" vs "+destination.length);
+
+        int roi = data.length / Constants.N_PIXELS;
+
+        double vraw;
+
+        int pos, offsetPos, triggerOffsetPos;
+        for (int pixel = 0; pixel < Constants.N_PIXELS; pixel++) {
+            for (int slice = 0; slice < roi; slice++) {
+
+                pos = pixel * roi + slice;
+                // Offset and Gain vector *should look the same
+                int start = startCellVector[pixel] != -1 ? startCellVector[pixel] : 0;
+
+                offsetPos = pixel * drsBaselineMean.length / Constants.N_PIXELS
+                        + ((slice + start) % (drsBaselineMean.length / Constants.N_PIXELS));
+
+                triggerOffsetPos = pixel * drsTriggerOffsetMean.length / Constants.N_PIXELS + slice;
+
+                vraw = data[pos];
+                vraw /= 1907.35;
+                vraw *= drsGainMean[offsetPos];
+                vraw += drsTriggerOffsetMean[triggerOffsetPos];
+                vraw += drsBaselineMean[offsetPos];
+                vraw /= dconv;
+
+                destination[pos] = vraw;
+            }
+        }
+
+        return destination;
+    }
 
     @Override
     public void init(ProcessContext processContext) throws Exception {
@@ -297,6 +368,4 @@ public void resetState() throws Exception {
     public void finish() throws Exception {
 
     }
-
-
 }

src/main/java/fact/datacorrection/InterpolatePixelArray.java

@@ -50,17 +50,7 @@ public Data process(Data item) {
 
         double[] input = (double[]) item.get(inputKey);
 
-        ZonedDateTime timeStamp;
-
-        if (item.containsKey("UnixTimeUTC")) {
-            Utils.isKeyValid(item, "UnixTimeUTC", int[].class);
-            int[] eventTime = (int[]) item.get("UnixTimeUTC");
-            timeStamp = Utils.unixTimeUTCToZonedDateTime(eventTime);
-        } else {
-            // MC Files don't have a UnixTimeUTC in the data item. Here the timestamp is hardcoded to 1.1.2000
-            // => The 12 bad pixels we have from the beginning on are used.
-            timeStamp = ZonedDateTime.of(2000, 1, 1, 0, 0,0,0, ZoneOffset.UTC);
-        }
+        ZonedDateTime timeStamp = Utils.getTimeStamp(item);
         PixelSet badPixelsSet = calibService.getBadPixels(timeStamp);
 
         double[] output;

src/main/java/fact/datacorrection/InterpolateTimeSeries.java

@@ -36,6 +36,12 @@ public class InterpolateTimeSeries implements Processor {
     @Parameter(required = false, description = "The minimum number of neighboring pixels required for interpolation", defaultValue = "3")
     public int minPixelToInterpolate = 3;
 
+    @Parameter(required = false, description = "The key for the resulting badPixelSet.")
+    public String badPixelKey = "badPixels";
+
+    @Parameter(required = false, description = "The key to the timestamp of the Event.")
+    public String timeStampKey = "timestamp";
+
     private FactPixelMapping pixelMap = FactPixelMapping.getInstance();
 
 
@@ -44,17 +50,7 @@ public Data process(Data item) {
         Utils.isKeyValid(item, dataKey, double[].class);
         double[] data = (double[]) item.get(dataKey);
 
-        ZonedDateTime timeStamp = null;
-
-        if (item.containsKey("UnixTimeUTC") == true) {
-            Utils.isKeyValid(item, "UnixTimeUTC", int[].class);
-            int[] eventTime = (int[]) item.get("UnixTimeUTC");
-            timeStamp = Utils.unixTimeUTCToZonedDateTime(eventTime);
-        } else {
-            // MC Files don't have a UnixTimeUTC in the data item. Here the timestamp is hardcoded to 1.1.2000
-            // => The 12 bad pixels we have from the beginning on are used.
-            timeStamp = ZonedDateTime.of(2000, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC);
-        }
+        ZonedDateTime timeStamp = Utils.getTimeStamp(item, timeStampKey);
 
         PixelSet badPixelSet = calibService.getBadPixels(timeStamp);
 
@@ -67,7 +63,7 @@ public Data process(Data item) {
         }
 
         item.put(dataOutputKey, data);
-        item.put("badPixels", badPixelSet);
+        item.put(badPixelKey, badPixelSet);
         return item;
     }
 

src/main/java/fact/datacorrection/PatchJumpRemoval.java

@@ -38,6 +38,9 @@ public class PatchJumpRemoval implements Processor {
     @Parameter(required = true)
     public String startCellKey = null;
 
+    @Parameter(required = false, description = "The key containing the UnixTimeUTC")
+    public String unixTimeKey = "UnixTimeUTC";
+
     @Parameter
     public double jumpLimit = 5.0;
 
@@ -94,10 +97,10 @@ public Data process(Data item) {
         Utils.isKeyValid(item, prevEventsKey, PreviousEventInfoContainer.class);
         Utils.isKeyValid(item, startCellKey, short[].class);
         Utils.isKeyValid(item, "NROI", Integer.class);
-        Utils.isKeyValid(item, "UnixTimeUTC", int[].class);
+        Utils.isKeyValid(item, unixTimeKey, int[].class);
 
         // Get variables out of data item
-        int[] currentTime = (int[]) item.get("UnixTimeUTC");
+        int[] currentTime = (int[]) item.get(unixTimeKey);
         roi = (Integer) item.get("NROI");
         short[] currentStartCells = (short[]) item.get(startCellKey);
         double[] data = (double[]) item.get(dataKey);