Merge pull request #164 from fact-project/singlePulseExtractionImprov…

…ement

Single pulse extraction improvement, fix #159, fix #147

.gitignore

@@ -13,6 +13,7 @@ docs.idx
 
 #intellij stuff
 .idea/
+.project
 
 out/
 

.travis.yml

@@ -2,7 +2,6 @@ language: java
 sudo: false
 script: mvn clean verify
 jdk:
-- openjdk7
 - oraclejdk8
 
 after_script:

CHANGELOG.md

@@ -1,5 +1,16 @@
 #Changelog for the fact-tools
 
+## Version 0.17.0 -- 21.11.2016
+
+* Improvements on Single Pulse Extractor
+   * No more static API, now several instances of the SinglePulseExtractor can run in parallel with different settings.
+   * No more static (global) variable to steer the behaviour of the SinglePulseExtractor.
+   * Reconstruction of amplitude time series from the photon stream (`fact.utils.ConvertSinglePulses2Timeseries`).
+   * Added example XML `singlePeMinimalExample.xml`
+   * Use Java 8
+* Refactoring
+  * Renamed `fact.statistics.TimerowFeatures` to `fact.statistics.TimeseriesFeatures`
+
 ## Version 0.16.2 -- 26.10.2016
 
 * ZFitsStream

examples/studies/pedestalNsbStudy.xml

@@ -23,7 +23,7 @@
     <fact.extraction.RisingEdgeForPositions dataKey="DataCalibrated" amplitudePositionsKey="maxPos" outputKey="arrivalTimePos" maxSlopesKey="maxSlopesPos" />
     <fact.extraction.RisingEdgePolynomFit dataKey="DataCalibrated" risingEdgeKey="arrivalTimePos" outputKey="arrivalTime" numberOfPoints="11" maxSlopesKey="maxSlopes" />
     <fact.filter.MovingAverage key="DataCalibrated" outputKey="DataSmothed" length="5" />
-    <fact.statistics.TimerowFeatures dataKey="DataCalibrated" movingAverageKey="DataSmothed" searchWindowLeft="35" searchWindowRight="200" outputKey="voltages" />
+    <fact.statistics.TimeseriesFeatures dataKey="DataCalibrated" movingAverageKey="DataSmothed" searchWindowLeft="35" searchWindowRight="200" outputKey="voltages" />
     <!--<fact.features.TimeSpread weightsKey="photoncharge" arrivalTimeKey="arrivalTime" showerKey="shower" outputKey="Timespread" />-->
     <!---->
     <fact.extraction.WaveformFluctuation key="DataCalibrated" outputKey="Nsb" skipFirst="35" skipLast="100" windowSize="30" />

examples/studies/singlePeExtractor/singlePeMinimalExample.xml

@@ -0,0 +1,56 @@
+<container>
+
+    <properties url="classpath:/default/settings.properties" />
+
+    <property name="infile" value="file:src/main/resources/testDataFile.fits.gz" />
+    <property name="drsfile" value="file:src/main/resources/testDrsFile.drs.fits.gz" />
+
+    <property name="integralGainFile" value="classpath:/default/gain_sorted_20131127.csv" />
+    <property name="pixelDelayFile" value="classpath:/default/delays_lightpulser_20150217.csv" />
+
+    <property name="output" value="file:testoutfile.json" />
+
+    <property name="auxFolder" value="file:src/main/resources/aux/" />
+    <service id="auxService" class="fact.auxservice.AuxFileService" auxFolder="${auxFolder}" />
+
+    <service id="calibService" class="fact.calibrationservice.ConstantCalibService" />
+
+    <!--In case you want to analyze .zfits files.-->
+    <!-- <stream id="fact" class="fact.io.zfits.ZFitsStream"  url="${infile}"/> -->
+
+    <stream id="fact" class="fact.io.FitsStream"  url="${infile}"/>
+
+    <process id="2" input="fact">
+        <!-- prevEventAndSkip: -->
+        <!-- PreviousEventInfo, Skip(no Data Trigger) -->
+        <include url="classpath:/default/data/prevEventAndSkip.xml" />
+        <!-- Output: Data -->
+
+        <!-- Calibration: -->
+        <!-- DrsCalibration, PatchJumpRemoval, RemoveSpikes,
+        DrsTimeCalibration, ArrayTimeCorrection, InterpolateBadPixel -->
+        <include url="classpath:/default/data/calibration.xml" />
+        <!-- Output: DataCalibrated -->
+
+        <!-- Single Pe Extraction -->
+        <fact.extraction.SinglePulseExtraction
+                dataKey="DataCalibrated"
+                outputKey="PhotonArrivals"
+        />
+
+        <!-- Data Time Series Reconstruction from Single Pe Data -->
+        <fact.utils.ConvertSinglePulses2Timeseries
+                singlePulsesKey="PhotonArrivals"
+                baseLineKey="PhotonArrivalsBaseLine"
+                timeSeriesKey="DataReconstructed"
+        />
+
+        <!--<fact.ShowViewer key="DataCalibrated" />-->
+
+        <!--<fact.io.JSONWriter keys="${keysForOutput}"-->
+                            <!--url="${output}"-->
+                            <!--writeListOfItems="True"-->
+                            <!--pixelSetsAsInt="True"-->
+        <!--/>-->
+    </process>
+</container>

examples/studies/singlePeExtractor/std_analysis_on_reconstructed_data.xml

@@ -0,0 +1,83 @@
+<container>
+
+    <properties url="classpath:/default/settings.properties" />
+
+    <property name="infile" value="file:src/main/resources/testDataFile.fits.gz" />
+    <property name="drsfile" value="file:src/main/resources/testDrsFile.drs.fits.gz" />
+
+    <property name="integralGainFile" value="classpath:/default/gain_sorted_20131127.csv" />
+    <property name="pixelDelayFile" value="classpath:/default/delays_lightpulser_20150217.csv" />
+
+    <property name="output" value="file:testoutfile.json" />
+
+    <property name="auxFolder" value="file:src/main/resources/aux/" />
+    <service id="auxService" class="fact.auxservice.AuxFileService" auxFolder="${auxFolder}" />
+
+    <service id="calibService" class="fact.calibrationservice.ConstantCalibService" />
+
+    <!--In case you want to analyze .zfits files.-->
+    <!-- <stream id="fact" class="fact.io.zfits.ZFitsStream"  url="${infile}"/> -->
+
+    <stream id="fact" class="fact.io.FitsStream"  url="${infile}"/>
+
+    <process id="2" input="fact">
+        <!-- prevEventAndSkip: -->
+        <!-- PreviousEventInfo, Skip(no Data Trigger) -->
+        <include url="classpath:/default/data/prevEventAndSkip.xml" />
+        <!-- Output: Data -->
+
+        <!-- Calibration: -->
+        <!-- DrsCalibration, PatchJumpRemoval, RemoveSpikes,
+        DrsTimeCalibration, ArrayTimeCorrection, InterpolateBadPixel -->
+        <include url="classpath:/default/data/calibration.xml" />
+        <!-- Output: DataCalibrated -->
+
+        <!-- Single Pe Extraction -->
+        <fact.extraction.SinglePulseExtraction
+                dataKey="DataCalibrated"
+                outputKey="PhotonArrivals"
+        />
+
+        <!-- Data Time Series Reconstruction from Single Pe Data -->
+        <fact.utils.ConvertSinglePulses2Timeseries
+                singlePulsesKey="PhotonArrivals"
+                baseLineKey="PhotonArrivalsBaseLine"
+                timeSeriesKey="DataCalibrated"
+        />
+
+
+        <!-- Extraction -->
+        <!-- BasicExtraction, RisingEdgeForPositions, RisingEdgePolynomFit, TimeOverThreshold,
+        PhotonChargeTimeOverThreshold, HandleSaturation, CorrectPixelDelays-->
+        <include url="classpath:/default/data/extraction.xml" />
+        <!-- Output: photoncharge, arrivalTime -->
+
+        <!-- Cleaning -->
+        <!-- SourcePosition(Cetatauri), CoreNeighborCleanTimeNeighbor-->
+        <include url="classpath:/default/data/cleaning.xml" />
+        <!-- Output: shower -->
+
+        <!-- Parameter calculation (only source independent) -->
+        <!-- ArrayMean(photoncharge,arrivalTime), ArrayStatistics(photoncharge,arrivalTime,maxSlopes,
+        arrivalTimePos,maxSlopesPos,maxAmplitudePosition,photonchargeSaturated,arrivalTimeTOT),
+        Size, DistributionFromShower, M3Long, Length, Width, NumberOfIslands, TimeGraident,
+        Concentration, ConcentrationCore, ConcentrationAtCenterOfGravity, Leakage, TimeSpread,
+        ShowerSlope, Disp -->
+        <include url="classpath:/default/data/parameterCalc.xml" />
+        <!-- Output: source independent parameters -->
+
+        <!-- Parameter calculation (only source dependent) -->
+        <!-- SourcePosition(${sourcename}), AntiSourcePosition(5), Alpha(for 6 Sources),
+        Distance(for 6 Sources), CosDeltaAlpha(for 6 Sources), Theta(for 6 Sources) -->
+        <include url="classpath:/default/data/sourceParameter.xml" />
+        <!-- Output: source dependent parameters -->
+
+        <!--<fact.ShowViewer key="DataCalibrated" />-->
+
+        <fact.io.JSONWriter keys="${keysForOutput}"
+                            url="${output}"
+                            writeListOfItems="True"
+                            pixelSetsAsInt="True"
+        />
+    </process>
+</container>

examples/viewer.xml

@@ -23,7 +23,12 @@
     <fact.extraction.SinglePulseExtraction 
         dataKey="DataCalibrated"
         outputKey="PhotonArrivals"
-        maxIterations="50"
+    />
+
+    <fact.utils.ConvertSinglePulses2Timeseries 
+      singlePulsesKey="PhotonArrivals"
+      baseLineKey="PhotonArrivalsBaseLine"
+      timeSeriesKey="ReconstructedDataCalibrated"
     />
 
     <fact.ShowViewer key="DataCalibrated" />

pom.xml

@@ -4,7 +4,7 @@
   <groupId>de.sfb876</groupId>
   <artifactId>fact-tools</artifactId>
   <name>fact-tools</name>
-  <version>0.16.2</version>
+  <version>0.17.0</version>
   <url>http://sfb876.de/fact-tools/</url>
 
   <description>
@@ -330,6 +330,14 @@
           <outputEncoding>UTF-8</outputEncoding>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <configuration>
+          <source>1.8</source>
+          <target>1.8</target>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
 

src/main/java/fact/Utils.java

@@ -522,4 +522,17 @@ public static  DescriptiveStatistics[] calculateTimeseriesStatistics(double[][]
 		return pixelStatistics;
 	}
 
+
+	/**
+	 * Flatten a 2d array
+     *
+     * @param array2d 2dim double array
+     * @return flattend double array
+	 */
+	public static double[] flatten2dArray(double[][] array2d) {
+		return Arrays.stream(array2d)
+				.flatMapToDouble(Arrays::stream)
+				.toArray();
+	}
+
 }