Parameter improvment and some small new abilities

pom.xml

@@ -4,7 +4,7 @@
   <groupId>de.sfb876</groupId>
   <artifactId>fact-tools</artifactId>
   <name>fact-tools</name>
-  <version>1.0.0</version>
+  <version>1.0.2-SNAPSHOT</version>
   <url>http://sfb876.de/fact-tools/</url>
 
   <description>

src/main/java/fact/datacorrection/DrsCalibration.java

@@ -4,6 +4,7 @@
 package fact.datacorrection;
 
 import fact.Constants;
+import fact.Utils;
 import fact.io.hdureader.BinTable;
 import fact.io.hdureader.BinTableReader;
 import fact.io.hdureader.FITS;
@@ -26,21 +27,32 @@
  * either as File or URL and will read the DRS data from that. This data is then
  * applied to all FactEvents processed by this class.
  *
- * @author Christian Bockermann <[email protected]>
+ * @author Christian Bockermann <[email protected]> , Michael Bulinski <[email protected]>
  */
 public class DrsCalibration implements StatefulProcessor {
     static Logger log = LoggerFactory.getLogger(DrsCalibration.class);
 
-    @Parameter
+    @Parameter(required = false, description = "The data key that will hold the resulting data array.")
     public String outputKey = "DataCalibrated";
 
     @Parameter(required = false, description = "Data array to be calibrated", defaultValue = "Data")
     public String key = "Data";
 
+    @Parameter(required = false, description = "Key to the StartCellData.")
+    public String startCellKey = "StartCellData";
+
     @Parameter(required = false, description = "A URL to the DRS calibration data (in FITS formats)",
             defaultValue = "Null. Will try to find path to drsFile from the stream.")
     public URL url = null;
 
+    @Parameter(required = false, description = "The name of the key that holds the drs filename.",
+            defaultValue = "@drsFile")
+    public String drsKey = "@drsFile";
+
+    @Parameter(required = false, description = "Whether to reverse the process.", defaultValue = "false")
+    public boolean reverse = false;
+
+    Data drsData = null;
 
     private File currentDrsFile = new File("");
 
@@ -84,7 +96,6 @@ protected void loadDrsData(URL in) {
             this.drsTriggerOffsetRms = row.getFloatArray("TriggerOffsetRms").orElseThrow(() -> new RuntimeException("File does not contain key TriggerOffsetRms"));
             this.drsGainMean = row.getFloatArray("GainMean").orElseThrow(() -> new RuntimeException("File does not contain key GainMean"));
             this.drsGainRms = row.getFloatArray("GainRms").orElseThrow(() -> new RuntimeException("File does not contain key GainRms"));
-
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
@@ -99,7 +110,7 @@ public Data process(Data data) {
 
         if (this.url == null) {
             //file not loaded yet. try to find by magic.
-            File drsFile = (File) data.get("@drsFile");
+            File drsFile = (File) data.get(drsKey);
             if (drsFile != null) {
                 if (!drsFile.equals(currentDrsFile)) {
                     currentDrsFile = drsFile;
@@ -108,52 +119,67 @@ public Data process(Data data) {
                         loadDrsData(drsFile.toURI().toURL());
                     } catch (MalformedURLException e) {
                         //pass.
+                        throw new RuntimeException("URL malformed");
                     }
                 }
             } else {
                 throw new IllegalArgumentException("No drs file set or no @drsFile key in data stream");
             }
         }
 
+        double[] rawfloatData;
         log.debug("Processing Data item by applying DRS calibration...");
-        short[] rawData = (short[]) data.get(key);
-        if (rawData == null) {
-            log.error(" data .fits file did not contain the value for the key "
-                    + key + ". cannot apply drscalibration");
-            throw new RuntimeException(
-                    " data .fits file did not contain the value for the key \"" + key + "\". Cannot apply drs calibration)");
-        }
+        if (!reverse) {
+            short[] rawData = (short[]) data.get(key);
+            if (rawData == null) {
+                log.error(" data .fits file did not contain the value for the key "
+                        + key + ". cannot apply drscalibration");
+                throw new RuntimeException(
+                        " data .fits file did not contain the value for the key \"" + key + "\". Cannot apply drs calibration)");
+            }
 
-        double[] rawfloatData = new double[rawData.length];
-        // System.arraycopy(rawData, 0, rawfloatData, 0, rawfloatData.length);
-        for (int i = 0; i < rawData.length; i++) {
-            rawfloatData[i] = rawData[i];
+            rawfloatData = new double[rawData.length];
+            // System.arraycopy(rawData, 0, rawfloatData, 0, rawfloatData.length);
+            for (int i = 0; i < rawData.length; i++) {
+                rawfloatData[i] = rawData[i];
+            }
+        } else {
+            Utils.isKeyValid(data, key, double[].class);
+            rawfloatData = (double[]) data.get(key);
         }
 
-        short[] startCell = (short[]) data.get("StartCellData");
+        short[] startCell = (short[]) data.get(startCellKey);
         if (startCell == null) {
             log.error(" data .fits file did not contain startcell data. cannot apply drscalibration");
             return null;
         }
-        log.debug("raw data has {} elements", rawData.length);
+        log.debug("raw data has {} elements", rawfloatData.length);
         log.debug("StartCellData has {} elements", startCell.length);
 
         double[] output = rawfloatData;
         if (!key.equals(outputKey)) {
-            output = new double[rawData.length];
+            output = new double[rawfloatData.length];
         }
 
-        double[] calibrated = applyDrsCalibration(rawfloatData, output, startCell);
-
-        data.put(outputKey, calibrated);
+        double[] calibrated = applyDrsCalibration(rawfloatData, output, startCell, reverse);
+        if (!reverse)
+            data.put(outputKey, calibrated);
+        else {
+            short[] decalibratedShortData = new short[calibrated.length];
+            // System.arraycopy(rawData, 0, rawfloatData, 0, rawfloatData.length);
+            for (int i = 0; i < calibrated.length; i++) {
+                decalibratedShortData[i] = (short) calibrated[i];
+            }
+            data.put(outputKey, decalibratedShortData);
+        }
 
-        // add color value if set
+        //TODO add color value if set
 
         return data;
     }
 
     public double[] applyDrsCalibration(double[] data, double[] destination,
-                                        short[] startCellVector) {
+                                        short[] startCellVector, boolean reverse_calibration) {
 
         if (destination == null || destination.length != data.length)
             destination = new double[data.length];
@@ -240,13 +266,24 @@ public double[] applyDrsCalibration(double[] data, double[] destination,
                 offsetPos = pixel * drsBaselineMean.length / Constants.N_PIXELS
                         + ((slice + start) % (drsBaselineMean.length / Constants.N_PIXELS));
 
-                triggerOffsetPos = pixel * drsTriggerOffsetMean.length / Constants.N_PIXELS + slice;
-
-                vraw = data[pos] * dconv;
-                vraw -= drsBaselineMean[offsetPos];
-                vraw -= drsTriggerOffsetMean[triggerOffsetPos];
-                vraw /= drsGainMean[offsetPos];
-                vraw *= 1907.35;
+                triggerOffsetPos = pixel * drsTriggerOffsetMean.length / Constants.N_PIXELS
+                        + slice;
+
+                if (!reverse_calibration) {
+                    vraw = data[pos];
+                    vraw *= dconv;
+                    vraw -= drsBaselineMean[offsetPos];
+                    vraw -= drsTriggerOffsetMean[triggerOffsetPos];
+                    vraw /= drsGainMean[offsetPos];
+                    vraw *= 1907.35;
+                } else {
+                    vraw = data[pos];
+                    vraw /= 1907.35;
+                    vraw *= drsGainMean[offsetPos];
+                    vraw += drsTriggerOffsetMean[triggerOffsetPos];
+                    vraw += drsBaselineMean[offsetPos];
+                    vraw /= dconv;
+                }
 
                 // slice_pt = pixel_pt + sl;
                 // drs_cal_offset = ( sl + StartCellVector[ pixel ] ) %
@@ -286,6 +323,4 @@ public void resetState() throws Exception {
     public void finish() throws Exception {
 
     }
-
-
 }

src/main/java/fact/datacorrection/InterpolateTimeSeries.java

@@ -36,6 +36,12 @@ public class InterpolateTimeSeries implements Processor {
     @Parameter(required = false, description = "The minimum number of neighboring pixels required for interpolation", defaultValue = "3")
     public int minPixelToInterpolate = 3;
 
+    @Parameter(required = false, description = "The key for the resulting badPixelSet.")
+    public String badPixelKey = "badPixels";
+
+    @Parameter(required = false, description = "The key to the unixTimeUTC of the Event.")
+    public String unixTimeKey = "UnixTimeUTC";
+
     private FactPixelMapping pixelMap = FactPixelMapping.getInstance();
 
 
@@ -46,9 +52,9 @@ public Data process(Data item) {
 
         ZonedDateTime timeStamp = null;
 
-        if (item.containsKey("UnixTimeUTC") == true) {
-            Utils.isKeyValid(item, "UnixTimeUTC", int[].class);
-            int[] eventTime = (int[]) item.get("UnixTimeUTC");
+        if (item.containsKey(unixTimeKey) == true) {
+            Utils.isKeyValid(item, unixTimeKey, int[].class);
+            int[] eventTime = (int[]) item.get(unixTimeKey);
             timeStamp = Utils.unixTimeUTCToZonedDateTime(eventTime);
         } else {
             // MC Files don't have a UnixTimeUTC in the data item. Here the timestamp is hardcoded to 1.1.2000
@@ -67,7 +73,7 @@ public Data process(Data item) {
         }
 
         item.put(dataOutputKey, data);
-        item.put("badPixels", badPixelSet);
+        item.put(badPixelKey, badPixelSet);
         return item;
     }
 

src/main/java/fact/datacorrection/PatchJumpRemoval.java

@@ -38,6 +38,9 @@ public class PatchJumpRemoval implements Processor {
     @Parameter(required = true)
     public String startCellKey = null;
 
+    @Parameter(required = false, description = "The key containing the UnixTimeUTC")
+    public String unixTimeKey = "UnixTimeUTC";
+
     @Parameter
     public double jumpLimit = 5.0;
 
@@ -94,10 +97,10 @@ public Data process(Data item) {
         Utils.isKeyValid(item, prevEventsKey, PreviousEventInfoContainer.class);
         Utils.isKeyValid(item, startCellKey, short[].class);
         Utils.isKeyValid(item, "NROI", Integer.class);
-        Utils.isKeyValid(item, "UnixTimeUTC", int[].class);
+        Utils.isKeyValid(item, unixTimeKey, int[].class);
 
         // Get variables out of data item
-        int[] currentTime = (int[]) item.get("UnixTimeUTC");
+        int[] currentTime = (int[]) item.get(unixTimeKey);
         roi = (Integer) item.get("NROI");
         short[] currentStartCells = (short[]) item.get(startCellKey);
         double[] data = (double[]) item.get(dataKey);

src/main/java/fact/features/source/SourcePosition.java

@@ -58,6 +58,9 @@ public class SourcePosition implements StatefulProcessor {
     @Service(required = false, description = "Name of the service that provides aux files")
     public AuxiliaryService auxService;
 
+    @Parameter(required = false, description = "The key containing the UnixTimeUTC")
+    public String unixTimeKey = "UnixTimeUTC";
+
     @Parameter(description = "If set, the fixed x position of the source in mm")
     public Double x = null;
 
@@ -133,9 +136,9 @@ public Data process(Data data) {
 
         } else {
             // Assume observations
-            int[] unixTimeUTC = (int[]) data.get("UnixTimeUTC");
+            int[] unixTimeUTC = (int[]) data.get(unixTimeKey);
             if (unixTimeUTC == null) {
-                log.error("The key \"UnixTimeUTC\" was not found in the event. Ignoring event");
+                log.error("The key \""+unixTimeKey+"\" was not found in the event. Ignoring event");
                 return null;
             }
 

src/main/java/fact/statistics/TimeseriesFeatures.java

@@ -34,6 +34,9 @@ public class TimeseriesFeatures implements Processor {
     @Parameter(required = true, description = "name of the key of the calculated features")
     public String outputKey = null;
 
+    @Parameter(required = false, description = "Whether to do the calculation with the substraction of the smoothed data.")
+    public boolean substractSmoothData = false;
+
     private int numberOfBins = 200;
     private double histogramMinBin = -10.0;
     private double histogramMaxBin = 10.0;
@@ -82,6 +85,13 @@ public Data process(Data input) {
             for (int sl = searchWindowLeft; sl < searchWindowRight; sl++) {
                 int slice = pix * roi + sl;
                 values[sl - searchWindowLeft] = data[slice];
+
+                if (substractSmoothData) {
+                    values[sl - searchWindowLeft] = data[slice] - movingAverage[slice];
+                } else {
+                    values[sl - searchWindowLeft] = data[slice];
+                }
+
                 int binNumber = findBinNumber((data[slice] - movingAverage[slice]));
                 histogram[binNumber] += 1;
             }

src/main/java/fact/utils/CombineDataArrays.java

@@ -0,0 +1,73 @@
+/**
+ *
+ */
+package fact.utils;
+
+import fact.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import stream.Data;
+import stream.Processor;
+import stream.annotations.Parameter;
+
+/**
+ * Combine two Pixel Arrays of the same size with each other
+ * Allows for either addition, substraction, multiplication or division.
+ *
+ * @author Michael Bulinski <[email protected]>
+ */
+public class CombineDataArrays implements Processor {
+    static Logger log = LoggerFactory.getLogger(RemappingKeys.class);
+
+    @Parameter(required = true, description = "The key to your first array.")
+    public String firstArrayKey;
+    @Parameter(required = true, description = "The key to your second array.")
+    public String secondArrayKey;
+    @Parameter(required = false, description = "The key for the resulting array.")
+    public String outputKey;
+    @Parameter(required = true, description = "The operation to perform, (add, sub, mul, div)")
+    public String op;
+
+
+    /* (non-Javadoc)
+     * @see stream.Processor#process(stream.Data)
+     */
+    @Override
+    public Data process(Data input) {
+        Utils.isKeyValid(input, firstArrayKey, double[].class);
+        Utils.isKeyValid(input, secondArrayKey, double[].class);
+
+        double[] array1 = (double[]) input.get(firstArrayKey);
+        double[] array2 = (double[]) input.get(secondArrayKey);
+
+        if (array1.length != array2.length) {
+            throw new RuntimeException("Given arrays are different lengths");
+        }
+        double[] resultArray = new double[array1.length];
+
+
+        if (op.equals("add")) {
+            for (int pix = 0; pix < array1.length; pix++) {
+                resultArray[pix] = (double) (array1[pix] + array2[pix]);
+            }
+        } else if (op.equals("sub")) {
+            for (int pix = 0; pix < array1.length; pix++) {
+                resultArray[pix] = (double) (array1[pix] - array2[pix]);
+            }
+        } else if (op.equals("mul")) {
+            for (int pix = 0; pix < array1.length; pix++) {
+                resultArray[pix] = (double) (array1[pix] * array2[pix]);
+            }
+        } else if (op.equals("div")) {
+            for (int pix = 0; pix < array1.length; pix++) {
+                resultArray[pix] = (double) (array1[pix] / array2[pix]);
+            }
+        } else {
+            throw new RuntimeException("The given operation op: " + op + " is not supported");
+        }
+
+        input.put(outputKey, resultArray);
+
+        return input;
+    }
+}

src/main/java/fact/utils/PreviousEventInfo.java

@@ -18,7 +18,11 @@ public class PreviousEventInfo implements Processor {
     @Parameter(required = true)
     public String outputKey = null;
 
-    int limitEvents = 20;
+    @Parameter(required = false, description = "The key containing the UnixTimeUTC")
+    public String unixTimeKey = "UnixTimeUTC";
+
+    @Parameter(required = false, description = "Set the amount of events to buffer for the jumpremoval.")
+    public int limitEvents = 20;
 
     PreviousEventInfoContainer previousEventInfo = new PreviousEventInfoContainer();
 
@@ -28,9 +32,9 @@ public Data process(Data item) {
 
         Utils.isKeyValid(item, startCellKey, short[].class);
         Utils.isKeyValid(item, "NROI", Integer.class);
-        Utils.isKeyValid(item, "UnixTimeUTC", int[].class);
+        Utils.isKeyValid(item, unixTimeKey, int[].class);
 
-        int[] eventTime = (int[]) item.get("UnixTimeUTC");
+        int[] eventTime = (int[]) item.get(unixTimeKey);
         short[] startCellArray = (short[]) item.get(startCellKey);
         int length = (Integer) item.get("NROI");