From fdeca23c538b8f23347f382bd98174a092560f46 Mon Sep 17 00:00:00 2001 From: sebastian Date: Sat, 21 Jan 2017 21:55:49 +0100 Subject: [PATCH 1/7] create a new calibration service to provide the integral single pulse gains. This is done, because there are two processors (BasicExtraction and SinglePulseExtraction) needing this gain calibration. --- examples/apply_model.xml | 2 +- examples/fellwalker_example.xml | 1 + examples/forErna/std_analysis_erna.xml | 1 + examples/forErna/std_analysis_mc_erna.xml | 1 + examples/mc_viewer.xml | 1 + examples/measure_performance.xml | 1 + examples/stdAnalysis/data/analysis.xml | 3 +- examples/stdAnalysis/mc/analysis_mc.xml | 2 +- .../stdAnalysis/mc/analysis_mc_dispRF.xml | 1 + .../studies/closedShutterGainCalibration.xml | 1 + examples/studies/extractionTest.xml | 1 + examples/studies/fits_writer.xml | 1 + examples/studies/jumpStudy.xml | 1 + examples/studies/muon_fitting.xml | 1 + examples/studies/muon_identification.xml | 1 + examples/studies/pedestalNsbStudy.xml | 3 +- .../singlePeMinimalExample.xml | 1 + .../std_analysis_on_reconstructed_data.xml | 1 + examples/studies/test_saturated_pulses.xml | 5 +- .../studies/timeCalibrationComparison.xml | 1 + examples/viewer.xml | 3 +- .../SinglePulseGainCalibService.java | 69 +++++++++++++ .../java/fact/extraction/BasicExtraction.java | 51 ++-------- .../extraction/SinglePulseExtraction.java | 97 +++++++------------ .../resources/default/data/extraction.xml | 4 +- .../resources/default/mc/extraction_mc.xml | 4 +- .../fact/features/BasicExtractionTest.java | 8 +- .../java/fact/parameter/ParameterTest.java | 7 +- .../parameter/PhotonChargeParameterTest.java | 8 +- 29 files changed, 160 insertions(+), 121 deletions(-) create mode 100644 src/main/java/fact/calibrationservice/SinglePulseGainCalibService.java diff --git a/examples/apply_model.xml b/examples/apply_model.xml index d5d7325d95..71eaa865d6 100644 --- a/examples/apply_model.xml +++ b/examples/apply_model.xml @@ -12,7 +12,7 @@ - + diff --git a/examples/fellwalker_example.xml b/examples/fellwalker_example.xml index a99eaeae5e..a569b2ab1a 100644 --- a/examples/fellwalker_example.xml +++ b/examples/fellwalker_example.xml @@ -10,6 +10,7 @@ + diff --git a/examples/forErna/std_analysis_erna.xml b/examples/forErna/std_analysis_erna.xml index 8c591342ba..5bef51d55a 100644 --- a/examples/forErna/std_analysis_erna.xml +++ b/examples/forErna/std_analysis_erna.xml @@ -11,6 +11,7 @@ + diff --git a/examples/forErna/std_analysis_mc_erna.xml b/examples/forErna/std_analysis_mc_erna.xml index a5983bf9f8..4f9d5585d5 100644 --- a/examples/forErna/std_analysis_mc_erna.xml +++ b/examples/forErna/std_analysis_mc_erna.xml @@ -8,6 +8,7 @@ + diff --git a/examples/mc_viewer.xml b/examples/mc_viewer.xml index 505b1e25f6..bc2a701c6a 100644 --- a/examples/mc_viewer.xml +++ b/examples/mc_viewer.xml @@ -8,6 +8,7 @@ + diff --git a/examples/measure_performance.xml b/examples/measure_performance.xml index daae93157f..3316de5f55 100644 --- a/examples/measure_performance.xml +++ b/examples/measure_performance.xml @@ -7,6 +7,7 @@ + diff --git a/examples/stdAnalysis/data/analysis.xml b/examples/stdAnalysis/data/analysis.xml index 421e3c1cd5..aa70016536 100644 --- a/examples/stdAnalysis/data/analysis.xml +++ b/examples/stdAnalysis/data/analysis.xml @@ -9,8 +9,9 @@ - + + diff --git a/examples/stdAnalysis/mc/analysis_mc.xml b/examples/stdAnalysis/mc/analysis_mc.xml index 990c7dbda8..518642b85c 100644 --- a/examples/stdAnalysis/mc/analysis_mc.xml +++ b/examples/stdAnalysis/mc/analysis_mc.xml @@ -8,7 +8,7 @@ - + diff --git a/examples/stdAnalysis/mc/analysis_mc_dispRF.xml b/examples/stdAnalysis/mc/analysis_mc_dispRF.xml index a997a79cee..4289d213aa 100644 --- a/examples/stdAnalysis/mc/analysis_mc_dispRF.xml +++ b/examples/stdAnalysis/mc/analysis_mc_dispRF.xml @@ -11,6 +11,7 @@ + diff --git a/examples/studies/closedShutterGainCalibration.xml b/examples/studies/closedShutterGainCalibration.xml index b83bd42b17..6371d362c9 100644 --- a/examples/studies/closedShutterGainCalibration.xml +++ b/examples/studies/closedShutterGainCalibration.xml @@ -8,6 +8,7 @@ + diff --git a/examples/studies/extractionTest.xml b/examples/studies/extractionTest.xml index 8bc0ecf7a5..9bb0702fb7 100644 --- a/examples/studies/extractionTest.xml +++ b/examples/studies/extractionTest.xml @@ -6,6 +6,7 @@ + diff --git a/examples/studies/fits_writer.xml b/examples/studies/fits_writer.xml index ceadb9ad04..69d52e3d9b 100644 --- a/examples/studies/fits_writer.xml +++ b/examples/studies/fits_writer.xml @@ -9,6 +9,7 @@ + diff --git a/examples/studies/jumpStudy.xml b/examples/studies/jumpStudy.xml index de1a27149d..419c10205a 100644 --- a/examples/studies/jumpStudy.xml +++ b/examples/studies/jumpStudy.xml @@ -6,6 +6,7 @@ + diff --git a/examples/studies/muon_fitting.xml b/examples/studies/muon_fitting.xml index 8213fed911..b5818806c9 100644 --- a/examples/studies/muon_fitting.xml +++ b/examples/studies/muon_fitting.xml @@ -7,6 +7,7 @@ + diff --git a/examples/studies/muon_identification.xml b/examples/studies/muon_identification.xml index cbb7bc23b7..47a52209fd 100644 --- a/examples/studies/muon_identification.xml +++ b/examples/studies/muon_identification.xml @@ -10,6 +10,7 @@ + diff --git a/examples/studies/pedestalNsbStudy.xml b/examples/studies/pedestalNsbStudy.xml index ddff530cb4..5f5f7d36d1 100644 --- a/examples/studies/pedestalNsbStudy.xml +++ b/examples/studies/pedestalNsbStudy.xml @@ -6,6 +6,7 @@ + @@ -18,7 +19,7 @@ - diff --git a/examples/studies/singlePeExtractor/singlePeMinimalExample.xml b/examples/studies/singlePeExtractor/singlePeMinimalExample.xml index e8cd4bcf66..d073301cc0 100644 --- a/examples/studies/singlePeExtractor/singlePeMinimalExample.xml +++ b/examples/studies/singlePeExtractor/singlePeMinimalExample.xml @@ -11,6 +11,7 @@ + diff --git a/examples/studies/singlePeExtractor/std_analysis_on_reconstructed_data.xml b/examples/studies/singlePeExtractor/std_analysis_on_reconstructed_data.xml index a842d1492f..d8877a228c 100644 --- a/examples/studies/singlePeExtractor/std_analysis_on_reconstructed_data.xml +++ b/examples/studies/singlePeExtractor/std_analysis_on_reconstructed_data.xml @@ -11,6 +11,7 @@ + diff --git a/examples/studies/test_saturated_pulses.xml b/examples/studies/test_saturated_pulses.xml index e802852706..a1dbdb41af 100644 --- a/examples/studies/test_saturated_pulses.xml +++ b/examples/studies/test_saturated_pulses.xml @@ -4,6 +4,7 @@ + @@ -19,7 +20,7 @@ - @@ -27,7 +28,7 @@ - diff --git a/examples/studies/timeCalibrationComparison.xml b/examples/studies/timeCalibrationComparison.xml index a98c3b5712..8d0899f6cb 100644 --- a/examples/studies/timeCalibrationComparison.xml +++ b/examples/studies/timeCalibrationComparison.xml @@ -6,6 +6,7 @@ + diff --git a/examples/viewer.xml b/examples/viewer.xml index 4a87079a3f..fa19a0b8c4 100644 --- a/examples/viewer.xml +++ b/examples/viewer.xml @@ -8,6 +8,7 @@ + @@ -25,7 +26,7 @@ Date: Tue, 24 Jan 2017 09:52:39 +0100 Subject: [PATCH 2/7] Implented a processor, to calibrate Timelines from mV to SinglePulseAmplitude; This should be used before BasicExtraction or any extration happens. --- .../SinglePulseGainCalibration.java | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 src/main/java/fact/datacorrection/SinglePulseGainCalibration.java diff --git a/src/main/java/fact/datacorrection/SinglePulseGainCalibration.java b/src/main/java/fact/datacorrection/SinglePulseGainCalibration.java new file mode 100644 index 0000000000..1a2b47f00b --- /dev/null +++ b/src/main/java/fact/datacorrection/SinglePulseGainCalibration.java @@ -0,0 +1,94 @@ +package fact.datacorrection; + +import fact.Constants; +import fact.calibrationservice.SinglePulseGainCalibService; +import fact.features.singlePulse.timeSeriesExtraction.TemplatePulse; +import stream.Data; +import stream.ProcessContext; +import stream.StatefulProcessor; +import stream.annotations.Parameter; + +/** + * Calibrates the unit of TimeLines from mV to SinglePulseAmplitude. + * + * This means, after this calibration the *height* or *amplitude* of a single pulse is 1. + * This means further, that the integral typically performed by BasicExtraction, + * will not be 1. It will instead have the same value, the *TemplatePulse* has, when integrated + * the same way. + */ +public class SinglePulseGainCalibration implements StatefulProcessor{ + + @Parameter( + required = true, + description = "Name of the output array" + ) + private String dataKey = "DataCalibrated"; + + @Parameter( + required = true, + description = "Name of input array" + ) + private String outputKey = "DataCalibrated"; + + @Parameter( + required = true, + description = "The calibration service for the integral single pulse gain" + ) + private SinglePulseGainCalibService gainService; + private double[] gainCorrection; + + @Override + public void init(ProcessContext processContext) throws Exception { + double factSinglePePulseIntegral = TemplatePulse.factSinglePePulseIntegral(); // unit: SinglePulseAmplitude * slices + if(gainService == null) { + gainCorrection = new double[Constants.NUMBEROFPIXEL]; + for(int i=0; i value: ~10 + } + + } + + @Override + public Data process(Data input) { + double[] TimeLines = ((double[]) input.get(dataKey)).clone(); + + final int npix = Constants.NUMBEROFPIXEL; + final int roi = TimeLines.length / npix; + + for (int pix = 0; pix < npix; pix++) { + for (int slice =0; slice < roi; slice++){ + TimeLines[pix*roi + slice] /= gainCorrection[pix]; + // now TimeLines unit: SinglePulseAmplitude, i.e. a single pulse has amplitude 1 in this units. + } + } + + input.put(outputKey, TimeLines); + return input; + } + + + @Override + public void resetState() throws Exception { + // empty on purpose + } + + @Override + public void finish() throws Exception { + // empty on purpose + } + + public void setDataKey(String dataKey) { + this.dataKey = dataKey; + } + + public void setOutputKey(String outputKey) { + this.outputKey = outputKey; + } + + public void setGainService(SinglePulseGainCalibService gainService) { + this.gainService = gainService; + } +} From d44ad0ebf7d21e305da27f9595ff96605e204138 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Tue, 24 Jan 2017 11:19:35 +0100 Subject: [PATCH 3/7] use SinglePulseGainCalibration before BasicExtraction BasicExtraction became a StatefulProcessor now: - to calculate the template integral ParameterTest uses SinglePulseGaincalibration before extraction starts. --- .../java/fact/extraction/BasicExtraction.java | 34 ++++++---- .../extraction/TimeGradientExtraction.java | 2 +- .../fact/features/BasicExtractionTest.java | 1 - .../java/fact/parameter/ParameterTest.java | 68 ++++++++++--------- .../parameter/PhotonChargeParameterTest.java | 11 +-- 5 files changed, 62 insertions(+), 54 deletions(-) diff --git a/src/main/java/fact/extraction/BasicExtraction.java b/src/main/java/fact/extraction/BasicExtraction.java index 8b03d5beca..52736ce3bc 100644 --- a/src/main/java/fact/extraction/BasicExtraction.java +++ b/src/main/java/fact/extraction/BasicExtraction.java @@ -2,12 +2,13 @@ import fact.Constants; import fact.Utils; -import fact.calibrationservice.SinglePulseGainCalibService; +import fact.features.singlePulse.timeSeriesExtraction.TemplatePulse; import org.jfree.chart.plot.IntervalMarker; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import stream.Data; -import stream.Processor; +import stream.ProcessContext; +import stream.StatefulProcessor; import stream.annotations.Parameter; /** @@ -22,7 +23,7 @@ * @author Fabian Temme * */ -public class BasicExtraction implements Processor { +public class BasicExtraction implements StatefulProcessor{ static Logger log = LoggerFactory.getLogger(BasicExtraction.class); @Parameter(required = true, description="key to the data array") protected String dataKey = null; @@ -30,8 +31,6 @@ public class BasicExtraction implements Processor { protected String outputKeyMaxAmplPos = null; @Parameter(required = true, description="outputKey for the calculated photoncharge") protected String outputKeyPhotonCharge = null; - @Parameter(required = true, description = "The calibration service for the integral single pulse gain") - SinglePulseGainCalibService gainService; @Parameter(required = false, description="start slice of the search window for the max amplitude", defaultValue="35") protected int startSearchWindow = 35; @Parameter(required = false, description="range of the search window for the max amplitude", defaultValue="90") @@ -46,6 +45,7 @@ public class BasicExtraction implements Processor { protected double[] integralGains = null; private int npix = Constants.NUMBEROFPIXEL; + public double factSinglePePulseIntegral; @Override public Data process(Data input) { @@ -53,7 +53,6 @@ public Data process(Data input) { int roi = (Integer) input.get("NROI"); npix = (Integer) input.get("NPIX"); - integralGains = gainService.getIntegralSinglePulseGain(); double[] data = (double[]) input.get(dataKey); @@ -67,11 +66,11 @@ public Data process(Data input) { for (int pix = 0; pix < npix; pix++) { positions[pix] = calculateMaxPosition(pix, startSearchWindow, startSearchWindow+rangeSearchWindow, roi, data); mPositions[pix] = new IntervalMarker(positions[pix],positions[pix] + 1); - + int halfHeightPos = calculatePositionHalfHeight(pix, positions[pix],positions[pix]-rangeHalfHeightWindow, roi, data); Utils.checkWindow(halfHeightPos, integrationWindow, validMinimalSlice, roi); - photonCharge[pix] = calculateIntegral(pix, halfHeightPos, integrationWindow, roi, data) / integralGains[pix]; + photonCharge[pix] = calculateIntegral(pix, halfHeightPos, integrationWindow, roi, data) / factSinglePePulseIntegral; mPhotonCharge[pix] = new IntervalMarker(halfHeightPos,halfHeightPos + integrationWindow); } input.put(outputKeyMaxAmplPos, positions); @@ -141,10 +140,6 @@ public void setDataKey(String dataKey) { this.dataKey = dataKey; } - public void setGainService(SinglePulseGainCalibService gainService) { - this.gainService = gainService; - } - public String getOutputKeyMaxAmplPos() { return outputKeyMaxAmplPos; } @@ -201,4 +196,19 @@ public int getValidMinimalSlice() { public void setValidMinimalSlice(int validMinimalSlice) { this.validMinimalSlice = validMinimalSlice; } + + @Override + public void init(ProcessContext processContext) throws Exception { + factSinglePePulseIntegral = TemplatePulse.factSinglePePulseIntegral(); // unit: SinglePulseAmplitude * slices + } + + @Override + public void resetState() throws Exception { + + } + + @Override + public void finish() throws Exception { + + } } diff --git a/src/main/java/fact/extraction/TimeGradientExtraction.java b/src/main/java/fact/extraction/TimeGradientExtraction.java index 4dee031b27..f7930170b1 100644 --- a/src/main/java/fact/extraction/TimeGradientExtraction.java +++ b/src/main/java/fact/extraction/TimeGradientExtraction.java @@ -69,7 +69,7 @@ public Data process(Data input) { int halfHeightPos = calculatePositionHalfHeight(px, positions[px],positions[px]-rangeHalfHeightWindow, roi, data); Utils.checkWindow(halfHeightPos, integrationWindow, validMinimalSlice, roi); - photonCharge[px] = calculateIntegral(px, halfHeightPos, integrationWindow, roi, data) / integralGains[px]; + photonCharge[px] = calculateIntegral(px, halfHeightPos, integrationWindow, roi, data) / factSinglePePulseIntegral; mPhotonCharge[px] = new IntervalMarker(halfHeightPos,halfHeightPos + integrationWindow); } diff --git a/src/test/java/fact/features/BasicExtractionTest.java b/src/test/java/fact/features/BasicExtractionTest.java index af148330c1..26157d5e4e 100644 --- a/src/test/java/fact/features/BasicExtractionTest.java +++ b/src/test/java/fact/features/BasicExtractionTest.java @@ -38,7 +38,6 @@ public void setup() throws Exception { extraction.setDataKey("test"); extraction.setOutputKeyMaxAmplPos(positionsKey); extraction.setOutputKeyPhotonCharge(photonChargeKey); - extraction.setGainService(igs); } @Test diff --git a/src/test/java/fact/parameter/ParameterTest.java b/src/test/java/fact/parameter/ParameterTest.java index 48e5b628fd..38e134f47a 100644 --- a/src/test/java/fact/parameter/ParameterTest.java +++ b/src/test/java/fact/parameter/ParameterTest.java @@ -2,6 +2,7 @@ import fact.calibrationservice.ConstantCalibService; import fact.cleaning.TwoLevelTimeMedian; +import fact.datacorrection.SinglePulseGainCalibration; import fact.extraction.BasicExtraction; import fact.extraction.RisingEdgeForPositions; import fact.features.DistributionFromShower; @@ -62,42 +63,47 @@ public void setUp() throws Exception { e.printStackTrace(); } - URL drsUrl = FITSStreamTest.class - .getResource("/testDrsFile.drs.fits.gz"); - DrsCalibration pr = new DrsCalibration(); - pr.setUrl(new SourceURL(drsUrl)); - pr.setOutputKey(key); + URL drsUrl = FITSStreamTest.class + .getResource("/testDrsFile.drs.fits.gz"); + DrsCalibration pr = new DrsCalibration(); + pr.setUrl(new SourceURL(drsUrl)); + pr.setOutputKey(key); pr.init(null); - pr.process(item); + pr.process(item); SinglePulseGainCalibService igs = new SinglePulseGainCalibService(); igs.setIntegralGainFile(new SourceURL(FITSStreamTest.class.getResource("/defaultIntegralGains.csv"))); - BasicExtraction bE = new BasicExtraction(); - bE.setDataKey(key); - bE.setOutputKeyMaxAmplPos(positions); - bE.setOutputKeyPhotonCharge(photonCharge); - bE.setGainService(igs); - bE.process(item); - - RisingEdgeForPositions pR = new RisingEdgeForPositions(); - pR.setDataKey(key); - pR.setAmplitudePositionsKey(positions); - pR.setOutputKey(arrivalTime); - pR.process(item); - - TwoLevelTimeMedian poser = new TwoLevelTimeMedian(); - poser.setCalibService(calibService); - poser.setPhotonChargeKey(photonCharge); - poser.setArrivalTimeKey(arrivalTime); - poser.setOutputKey(shower); - poser.setCorePixelThreshold(1); - poser.setNeighborPixelThreshold(0.1); - poser.setMinNumberOfPixel(1); - poser.setTimeLimit(40); - poser.process(item); - - + SinglePulseGainCalibration spgc = new SinglePulseGainCalibration(); + spgc.setGainService(igs); + spgc.setDataKey(key); + spgc.setOutputKey(key); + spgc.init(null); + spgc.process(item); + + BasicExtraction bE = new BasicExtraction(); + bE.setDataKey(key); + bE.setOutputKeyMaxAmplPos(positions); + bE.setOutputKeyPhotonCharge(photonCharge); + bE.init(null); + bE.process(item); + + RisingEdgeForPositions pR = new RisingEdgeForPositions(); + pR.setDataKey(key); + pR.setAmplitudePositionsKey(positions); + pR.setOutputKey(arrivalTime); + pR.process(item); + + TwoLevelTimeMedian poser = new TwoLevelTimeMedian(); + poser.setCalibService(calibService); + poser.setPhotonChargeKey(photonCharge); + poser.setArrivalTimeKey(arrivalTime); + poser.setOutputKey(shower); + poser.setCorePixelThreshold(1); + poser.setNeighborPixelThreshold(0.1); + poser.setMinNumberOfPixel(1); + poser.setTimeLimit(40); + poser.process(item); DistributionFromShower dist = new DistributionFromShower(); dist.setPixelSetKey(shower); diff --git a/src/test/java/fact/parameter/PhotonChargeParameterTest.java b/src/test/java/fact/parameter/PhotonChargeParameterTest.java index 3b8d173013..9657cd0a91 100644 --- a/src/test/java/fact/parameter/PhotonChargeParameterTest.java +++ b/src/test/java/fact/parameter/PhotonChargeParameterTest.java @@ -1,12 +1,9 @@ package fact.parameter; import fact.extraction.BasicExtraction; -import fact.io.FITSStreamTest; -import fact.calibrationservice.SinglePulseGainCalibService; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import stream.io.SourceURL; import static org.junit.Assert.assertTrue; @@ -27,19 +24,15 @@ public class PhotonChargeParameterTest extends ParameterTest { @Test public void testValidParameter() throws Exception { // //start processor with the correct parameter - assertTrue("Expecteds output already in data item", + assertTrue("Expected output already in data item", !item.containsKey(outputKey)); - SinglePulseGainCalibService igs = new SinglePulseGainCalibService(); - igs.setIntegralGainFile(new SourceURL(FITSStreamTest.class.getResource("/defaultIntegralGains.csv"))); - BasicExtraction extraction = new BasicExtraction(); extraction.setDataKey(key); extraction.setOutputKeyMaxAmplPos(positions); extraction.setOutputKeyPhotonCharge(outputKey); - extraction.setGainService(igs); extraction.process(item); - assertTrue("Expecteds output not in data item but it should be there", + assertTrue("Expected output not in data item but it should be there", item.containsKey(outputKey)); // item.remove(outputKey); } From aacfc67a4d95650d2ab890294f4846eb0b1189cb Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Tue, 24 Jan 2017 11:35:42 +0100 Subject: [PATCH 4/7] Get rid of SinglePulseCalibration inside SinglepulseExtraction --- .../extraction/SinglePulseExtraction.java | 56 ++----------------- 1 file changed, 5 insertions(+), 51 deletions(-) diff --git a/src/main/java/fact/extraction/SinglePulseExtraction.java b/src/main/java/fact/extraction/SinglePulseExtraction.java index 1cbd903e3e..27591b1e4a 100644 --- a/src/main/java/fact/extraction/SinglePulseExtraction.java +++ b/src/main/java/fact/extraction/SinglePulseExtraction.java @@ -3,15 +3,11 @@ import fact.Constants; import fact.Utils; import stream.Data; -import stream.StatefulProcessor; -import stream.ProcessContext; +import stream.Processor; import stream.annotations.Parameter; import java.util.Arrays; -import java.net.URL; import fact.features.singlePulse.timeSeriesExtraction.SinglePulseExtractor; import fact.features.singlePulse.timeSeriesExtraction.ElementWise; -import fact.features.singlePulse.timeSeriesExtraction.TemplatePulse; -import fact.calibrationservice.SinglePulseGainCalibService; /** * Extracts a list of arrival slice positions of photons for each pixel @@ -37,7 +33,7 @@ * Created by Sebastian Mueller * and some modifications from Jens Buss */ -public class SinglePulseExtraction implements StatefulProcessor { +public class SinglePulseExtraction implements Processor { @Parameter(required=true, description="") private String dataKey = null; @@ -85,37 +81,10 @@ public class SinglePulseExtraction implements StatefulProcessor { ) protected int extractionWindowLengthInSlices = 225; - @Parameter( - required = false, - description = "The calibration service for the integral single pulse gain", - defaultValue = "null") - SinglePulseGainCalibService gainService = null; - protected double[] gainCorrection = null; - protected double factSinglePePulseIntegral; private int npix = Constants.NUMBEROFPIXEL; private int roi = 300; - @Override - public void init(ProcessContext processContext) throws Exception { - - factSinglePePulseIntegral = TemplatePulse.factSinglePePulseIntegral(); - - if(gainService == null) { - gainCorrection = new double[npix]; - for(int i=0; i Date: Wed, 25 Jan 2017 17:32:20 +0100 Subject: [PATCH 5/7] Revert "Get rid of SinglePulseCalibration inside SinglepulseExtraction" This reverts commit aacfc67a4d95650d2ab890294f4846eb0b1189cb. --- .../extraction/SinglePulseExtraction.java | 56 +++++++++++++++++-- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/src/main/java/fact/extraction/SinglePulseExtraction.java b/src/main/java/fact/extraction/SinglePulseExtraction.java index 27591b1e4a..1cbd903e3e 100644 --- a/src/main/java/fact/extraction/SinglePulseExtraction.java +++ b/src/main/java/fact/extraction/SinglePulseExtraction.java @@ -3,11 +3,15 @@ import fact.Constants; import fact.Utils; import stream.Data; -import stream.Processor; +import stream.StatefulProcessor; +import stream.ProcessContext; import stream.annotations.Parameter; import java.util.Arrays; +import java.net.URL; import fact.features.singlePulse.timeSeriesExtraction.SinglePulseExtractor; import fact.features.singlePulse.timeSeriesExtraction.ElementWise; +import fact.features.singlePulse.timeSeriesExtraction.TemplatePulse; +import fact.calibrationservice.SinglePulseGainCalibService; /** * Extracts a list of arrival slice positions of photons for each pixel @@ -33,7 +37,7 @@ * Created by Sebastian Mueller * and some modifications from Jens Buss */ -public class SinglePulseExtraction implements Processor { +public class SinglePulseExtraction implements StatefulProcessor { @Parameter(required=true, description="") private String dataKey = null; @@ -81,10 +85,37 @@ public class SinglePulseExtraction implements Processor { ) protected int extractionWindowLengthInSlices = 225; + @Parameter( + required = false, + description = "The calibration service for the integral single pulse gain", + defaultValue = "null") + SinglePulseGainCalibService gainService = null; + protected double[] gainCorrection = null; + protected double factSinglePePulseIntegral; private int npix = Constants.NUMBEROFPIXEL; private int roi = 300; + @Override + public void init(ProcessContext processContext) throws Exception { + + factSinglePePulseIntegral = TemplatePulse.factSinglePePulseIntegral(); + + if(gainService == null) { + gainCorrection = new double[npix]; + for(int i=0; i Date: Wed, 25 Jan 2017 17:32:39 +0100 Subject: [PATCH 6/7] Revert "use SinglePulseGainCalibration before BasicExtraction" This reverts commit d44ad0ebf7d21e305da27f9595ff96605e204138. --- .../java/fact/extraction/BasicExtraction.java | 34 ++++------ .../extraction/TimeGradientExtraction.java | 2 +- .../fact/features/BasicExtractionTest.java | 1 + .../java/fact/parameter/ParameterTest.java | 68 +++++++++---------- .../parameter/PhotonChargeParameterTest.java | 11 ++- 5 files changed, 54 insertions(+), 62 deletions(-) diff --git a/src/main/java/fact/extraction/BasicExtraction.java b/src/main/java/fact/extraction/BasicExtraction.java index 52736ce3bc..8b03d5beca 100644 --- a/src/main/java/fact/extraction/BasicExtraction.java +++ b/src/main/java/fact/extraction/BasicExtraction.java @@ -2,13 +2,12 @@ import fact.Constants; import fact.Utils; -import fact.features.singlePulse.timeSeriesExtraction.TemplatePulse; +import fact.calibrationservice.SinglePulseGainCalibService; import org.jfree.chart.plot.IntervalMarker; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import stream.Data; -import stream.ProcessContext; -import stream.StatefulProcessor; +import stream.Processor; import stream.annotations.Parameter; /** @@ -23,7 +22,7 @@ * @author Fabian Temme * */ -public class BasicExtraction implements StatefulProcessor{ +public class BasicExtraction implements Processor { static Logger log = LoggerFactory.getLogger(BasicExtraction.class); @Parameter(required = true, description="key to the data array") protected String dataKey = null; @@ -31,6 +30,8 @@ public class BasicExtraction implements StatefulProcessor{ protected String outputKeyMaxAmplPos = null; @Parameter(required = true, description="outputKey for the calculated photoncharge") protected String outputKeyPhotonCharge = null; + @Parameter(required = true, description = "The calibration service for the integral single pulse gain") + SinglePulseGainCalibService gainService; @Parameter(required = false, description="start slice of the search window for the max amplitude", defaultValue="35") protected int startSearchWindow = 35; @Parameter(required = false, description="range of the search window for the max amplitude", defaultValue="90") @@ -45,7 +46,6 @@ public class BasicExtraction implements StatefulProcessor{ protected double[] integralGains = null; private int npix = Constants.NUMBEROFPIXEL; - public double factSinglePePulseIntegral; @Override public Data process(Data input) { @@ -53,6 +53,7 @@ public Data process(Data input) { int roi = (Integer) input.get("NROI"); npix = (Integer) input.get("NPIX"); + integralGains = gainService.getIntegralSinglePulseGain(); double[] data = (double[]) input.get(dataKey); @@ -66,11 +67,11 @@ public Data process(Data input) { for (int pix = 0; pix < npix; pix++) { positions[pix] = calculateMaxPosition(pix, startSearchWindow, startSearchWindow+rangeSearchWindow, roi, data); mPositions[pix] = new IntervalMarker(positions[pix],positions[pix] + 1); - + int halfHeightPos = calculatePositionHalfHeight(pix, positions[pix],positions[pix]-rangeHalfHeightWindow, roi, data); Utils.checkWindow(halfHeightPos, integrationWindow, validMinimalSlice, roi); - photonCharge[pix] = calculateIntegral(pix, halfHeightPos, integrationWindow, roi, data) / factSinglePePulseIntegral; + photonCharge[pix] = calculateIntegral(pix, halfHeightPos, integrationWindow, roi, data) / integralGains[pix]; mPhotonCharge[pix] = new IntervalMarker(halfHeightPos,halfHeightPos + integrationWindow); } input.put(outputKeyMaxAmplPos, positions); @@ -140,6 +141,10 @@ public void setDataKey(String dataKey) { this.dataKey = dataKey; } + public void setGainService(SinglePulseGainCalibService gainService) { + this.gainService = gainService; + } + public String getOutputKeyMaxAmplPos() { return outputKeyMaxAmplPos; } @@ -196,19 +201,4 @@ public int getValidMinimalSlice() { public void setValidMinimalSlice(int validMinimalSlice) { this.validMinimalSlice = validMinimalSlice; } - - @Override - public void init(ProcessContext processContext) throws Exception { - factSinglePePulseIntegral = TemplatePulse.factSinglePePulseIntegral(); // unit: SinglePulseAmplitude * slices - } - - @Override - public void resetState() throws Exception { - - } - - @Override - public void finish() throws Exception { - - } } diff --git a/src/main/java/fact/extraction/TimeGradientExtraction.java b/src/main/java/fact/extraction/TimeGradientExtraction.java index f7930170b1..4dee031b27 100644 --- a/src/main/java/fact/extraction/TimeGradientExtraction.java +++ b/src/main/java/fact/extraction/TimeGradientExtraction.java @@ -69,7 +69,7 @@ public Data process(Data input) { int halfHeightPos = calculatePositionHalfHeight(px, positions[px],positions[px]-rangeHalfHeightWindow, roi, data); Utils.checkWindow(halfHeightPos, integrationWindow, validMinimalSlice, roi); - photonCharge[px] = calculateIntegral(px, halfHeightPos, integrationWindow, roi, data) / factSinglePePulseIntegral; + photonCharge[px] = calculateIntegral(px, halfHeightPos, integrationWindow, roi, data) / integralGains[px]; mPhotonCharge[px] = new IntervalMarker(halfHeightPos,halfHeightPos + integrationWindow); } diff --git a/src/test/java/fact/features/BasicExtractionTest.java b/src/test/java/fact/features/BasicExtractionTest.java index 26157d5e4e..af148330c1 100644 --- a/src/test/java/fact/features/BasicExtractionTest.java +++ b/src/test/java/fact/features/BasicExtractionTest.java @@ -38,6 +38,7 @@ public void setup() throws Exception { extraction.setDataKey("test"); extraction.setOutputKeyMaxAmplPos(positionsKey); extraction.setOutputKeyPhotonCharge(photonChargeKey); + extraction.setGainService(igs); } @Test diff --git a/src/test/java/fact/parameter/ParameterTest.java b/src/test/java/fact/parameter/ParameterTest.java index 38e134f47a..48e5b628fd 100644 --- a/src/test/java/fact/parameter/ParameterTest.java +++ b/src/test/java/fact/parameter/ParameterTest.java @@ -2,7 +2,6 @@ import fact.calibrationservice.ConstantCalibService; import fact.cleaning.TwoLevelTimeMedian; -import fact.datacorrection.SinglePulseGainCalibration; import fact.extraction.BasicExtraction; import fact.extraction.RisingEdgeForPositions; import fact.features.DistributionFromShower; @@ -63,47 +62,42 @@ public void setUp() throws Exception { e.printStackTrace(); } - URL drsUrl = FITSStreamTest.class - .getResource("/testDrsFile.drs.fits.gz"); - DrsCalibration pr = new DrsCalibration(); - pr.setUrl(new SourceURL(drsUrl)); - pr.setOutputKey(key); + URL drsUrl = FITSStreamTest.class + .getResource("/testDrsFile.drs.fits.gz"); + DrsCalibration pr = new DrsCalibration(); + pr.setUrl(new SourceURL(drsUrl)); + pr.setOutputKey(key); pr.init(null); - pr.process(item); + pr.process(item); SinglePulseGainCalibService igs = new SinglePulseGainCalibService(); igs.setIntegralGainFile(new SourceURL(FITSStreamTest.class.getResource("/defaultIntegralGains.csv"))); - SinglePulseGainCalibration spgc = new SinglePulseGainCalibration(); - spgc.setGainService(igs); - spgc.setDataKey(key); - spgc.setOutputKey(key); - spgc.init(null); - spgc.process(item); - - BasicExtraction bE = new BasicExtraction(); - bE.setDataKey(key); - bE.setOutputKeyMaxAmplPos(positions); - bE.setOutputKeyPhotonCharge(photonCharge); - bE.init(null); - bE.process(item); - - RisingEdgeForPositions pR = new RisingEdgeForPositions(); - pR.setDataKey(key); - pR.setAmplitudePositionsKey(positions); - pR.setOutputKey(arrivalTime); - pR.process(item); - - TwoLevelTimeMedian poser = new TwoLevelTimeMedian(); - poser.setCalibService(calibService); - poser.setPhotonChargeKey(photonCharge); - poser.setArrivalTimeKey(arrivalTime); - poser.setOutputKey(shower); - poser.setCorePixelThreshold(1); - poser.setNeighborPixelThreshold(0.1); - poser.setMinNumberOfPixel(1); - poser.setTimeLimit(40); - poser.process(item); + BasicExtraction bE = new BasicExtraction(); + bE.setDataKey(key); + bE.setOutputKeyMaxAmplPos(positions); + bE.setOutputKeyPhotonCharge(photonCharge); + bE.setGainService(igs); + bE.process(item); + + RisingEdgeForPositions pR = new RisingEdgeForPositions(); + pR.setDataKey(key); + pR.setAmplitudePositionsKey(positions); + pR.setOutputKey(arrivalTime); + pR.process(item); + + TwoLevelTimeMedian poser = new TwoLevelTimeMedian(); + poser.setCalibService(calibService); + poser.setPhotonChargeKey(photonCharge); + poser.setArrivalTimeKey(arrivalTime); + poser.setOutputKey(shower); + poser.setCorePixelThreshold(1); + poser.setNeighborPixelThreshold(0.1); + poser.setMinNumberOfPixel(1); + poser.setTimeLimit(40); + poser.process(item); + + DistributionFromShower dist = new DistributionFromShower(); dist.setPixelSetKey(shower); diff --git a/src/test/java/fact/parameter/PhotonChargeParameterTest.java b/src/test/java/fact/parameter/PhotonChargeParameterTest.java index 9657cd0a91..3b8d173013 100644 --- a/src/test/java/fact/parameter/PhotonChargeParameterTest.java +++ b/src/test/java/fact/parameter/PhotonChargeParameterTest.java @@ -1,9 +1,12 @@ package fact.parameter; import fact.extraction.BasicExtraction; +import fact.io.FITSStreamTest; +import fact.calibrationservice.SinglePulseGainCalibService; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import stream.io.SourceURL; import static org.junit.Assert.assertTrue; @@ -24,15 +27,19 @@ public class PhotonChargeParameterTest extends ParameterTest { @Test public void testValidParameter() throws Exception { // //start processor with the correct parameter - assertTrue("Expected output already in data item", + assertTrue("Expecteds output already in data item", !item.containsKey(outputKey)); + SinglePulseGainCalibService igs = new SinglePulseGainCalibService(); + igs.setIntegralGainFile(new SourceURL(FITSStreamTest.class.getResource("/defaultIntegralGains.csv"))); + BasicExtraction extraction = new BasicExtraction(); extraction.setDataKey(key); extraction.setOutputKeyMaxAmplPos(positions); extraction.setOutputKeyPhotonCharge(outputKey); + extraction.setGainService(igs); extraction.process(item); - assertTrue("Expected output not in data item but it should be there", + assertTrue("Expecteds output not in data item but it should be there", item.containsKey(outputKey)); // item.remove(outputKey); } From ba86cd87703c6bc54def63366a8edfca6a89d3d3 Mon Sep 17 00:00:00 2001 From: Dominik Neise Date: Wed, 25 Jan 2017 17:32:59 +0100 Subject: [PATCH 7/7] Revert "Implented a processor, to calibrate Timelines" This reverts commit b52aa7d9e06b864d7fa277528970236b9969ccdf. --- .../SinglePulseGainCalibration.java | 94 ------------------- 1 file changed, 94 deletions(-) delete mode 100644 src/main/java/fact/datacorrection/SinglePulseGainCalibration.java diff --git a/src/main/java/fact/datacorrection/SinglePulseGainCalibration.java b/src/main/java/fact/datacorrection/SinglePulseGainCalibration.java deleted file mode 100644 index 1a2b47f00b..0000000000 --- a/src/main/java/fact/datacorrection/SinglePulseGainCalibration.java +++ /dev/null @@ -1,94 +0,0 @@ -package fact.datacorrection; - -import fact.Constants; -import fact.calibrationservice.SinglePulseGainCalibService; -import fact.features.singlePulse.timeSeriesExtraction.TemplatePulse; -import stream.Data; -import stream.ProcessContext; -import stream.StatefulProcessor; -import stream.annotations.Parameter; - -/** - * Calibrates the unit of TimeLines from mV to SinglePulseAmplitude. - * - * This means, after this calibration the *height* or *amplitude* of a single pulse is 1. - * This means further, that the integral typically performed by BasicExtraction, - * will not be 1. It will instead have the same value, the *TemplatePulse* has, when integrated - * the same way. - */ -public class SinglePulseGainCalibration implements StatefulProcessor{ - - @Parameter( - required = true, - description = "Name of the output array" - ) - private String dataKey = "DataCalibrated"; - - @Parameter( - required = true, - description = "Name of input array" - ) - private String outputKey = "DataCalibrated"; - - @Parameter( - required = true, - description = "The calibration service for the integral single pulse gain" - ) - private SinglePulseGainCalibService gainService; - private double[] gainCorrection; - - @Override - public void init(ProcessContext processContext) throws Exception { - double factSinglePePulseIntegral = TemplatePulse.factSinglePePulseIntegral(); // unit: SinglePulseAmplitude * slices - if(gainService == null) { - gainCorrection = new double[Constants.NUMBEROFPIXEL]; - for(int i=0; i value: ~10 - } - - } - - @Override - public Data process(Data input) { - double[] TimeLines = ((double[]) input.get(dataKey)).clone(); - - final int npix = Constants.NUMBEROFPIXEL; - final int roi = TimeLines.length / npix; - - for (int pix = 0; pix < npix; pix++) { - for (int slice =0; slice < roi; slice++){ - TimeLines[pix*roi + slice] /= gainCorrection[pix]; - // now TimeLines unit: SinglePulseAmplitude, i.e. a single pulse has amplitude 1 in this units. - } - } - - input.put(outputKey, TimeLines); - return input; - } - - - @Override - public void resetState() throws Exception { - // empty on purpose - } - - @Override - public void finish() throws Exception { - // empty on purpose - } - - public void setDataKey(String dataKey) { - this.dataKey = dataKey; - } - - public void setOutputKey(String outputKey) { - this.outputKey = outputKey; - } - - public void setGainService(SinglePulseGainCalibService gainService) { - this.gainService = gainService; - } -}