From 4a508da2ddf8351c4cdfaf824cb89c271674dddb Mon Sep 17 00:00:00 2001 From: Michael Bulinski Date: Fri, 16 Mar 2018 13:12:23 +0100 Subject: [PATCH 01/24] Added the ability to skip to any row within a fits file --- src/main/java/fact/io/hdureader/BinTable.java | 9 +++ .../fact/io/hdureader/BinTableReader.java | 11 ++++ .../java/fact/io/hdureader/FITSStream.java | 4 ++ src/main/java/fact/io/hdureader/Reader.java | 7 +++ .../fact/io/hdureader/ZFITSHeapReader.java | 29 ++++++++++ .../java/fact/io/hdureader/FitsHDUTests.java | 58 +++++++++++++++++++ 6 files changed, 118 insertions(+) diff --git a/src/main/java/fact/io/hdureader/BinTable.java b/src/main/java/fact/io/hdureader/BinTable.java index cad5183d4d..48f5a554ba 100644 --- a/src/main/java/fact/io/hdureader/BinTable.java +++ b/src/main/java/fact/io/hdureader/BinTable.java @@ -35,6 +35,12 @@ public class BinTable { final DataInputStream tableDataStream; DataInputStream heapDataStream = null; + private final Header header; + + public Header getHeader() { + return header; + } + /** * This enum maps the type characters in the header to the fits types. @@ -130,8 +136,10 @@ private void setTypeAndCount(HeaderLine form) throws IOException{ public Integer numberOfRowsInTable = 0; public Integer numberOfColumnsInTable = 0; public final String name; + public Integer numberOfBytesPerRow = 0; // in header value: naxis1 BinTable(Header header, long hduOffset, URL url) throws IllegalArgumentException, IOException { + this.header = header; binTableSanityCheck(header); @@ -163,6 +171,7 @@ private void setTypeAndCount(HeaderLine form) throws IOException{ columns.add(new TableColumn(zform, tform, ttype.getValue())); } + numberOfBytesPerRow = header.getInt("ZNAXIS1").orElse(header.getInt("NAXIS1").orElse(0)); numberOfRowsInTable = header.getInt("ZNAXIS2").orElse(header.getInt("NAXIS2").orElse(0)); numberOfColumnsInTable = columns.size(); diff --git a/src/main/java/fact/io/hdureader/BinTableReader.java b/src/main/java/fact/io/hdureader/BinTableReader.java index 2ac666bfb7..bfe97973ba 100644 --- a/src/main/java/fact/io/hdureader/BinTableReader.java +++ b/src/main/java/fact/io/hdureader/BinTableReader.java @@ -38,11 +38,13 @@ public boolean hasNext() { private final List columns; private int numberOfRowsRead = 0; private final int numberOfRowsInTable; + private final int numberOfBytesPerRow; private BinTableReader(BinTable binTable) { this.stream = binTable.tableDataStream; this.columns = binTable.columns; + this.numberOfBytesPerRow = binTable.numberOfBytesPerRow; this.numberOfRowsInTable = binTable.numberOfRowsInTable; } @@ -141,4 +143,13 @@ private Serializable readArrayFromStream(BinTable.TableColumn c, DataInputStream } return null; } + + @Override + public void skipToRow(int num) throws IOException { + if (num <= numberOfRowsInTable) { + new IndexOutOfBoundsException("Table has not enough rows to access row num: " + num); + } + stream.skipBytes(num * this.numberOfBytesPerRow); + numberOfRowsRead = num; + } } diff --git a/src/main/java/fact/io/hdureader/FITSStream.java b/src/main/java/fact/io/hdureader/FITSStream.java index 26e1e13e2e..262ebb6e01 100644 --- a/src/main/java/fact/io/hdureader/FITSStream.java +++ b/src/main/java/fact/io/hdureader/FITSStream.java @@ -127,4 +127,8 @@ public Data readNext() throws Exception { return item; } + + public Reader getReader() { + return reader; + } } diff --git a/src/main/java/fact/io/hdureader/Reader.java b/src/main/java/fact/io/hdureader/Reader.java index 7c2a1f20b5..c63edaa63b 100644 --- a/src/main/java/fact/io/hdureader/Reader.java +++ b/src/main/java/fact/io/hdureader/Reader.java @@ -72,5 +72,12 @@ default Iterator> iterator() { */ OptionalTypesMap getNextRow() throws IOException; + /** + * Skips the given number of rows. + * + * @param num The amount of rows to skip. + * @throws IOException + */ + void skipToRow(int num) throws IOException; } diff --git a/src/main/java/fact/io/hdureader/ZFITSHeapReader.java b/src/main/java/fact/io/hdureader/ZFITSHeapReader.java index 07dba0ceda..2fd9957167 100644 --- a/src/main/java/fact/io/hdureader/ZFITSHeapReader.java +++ b/src/main/java/fact/io/hdureader/ZFITSHeapReader.java @@ -65,6 +65,9 @@ public final class ZFITSHeapReader implements Reader { private final DataInputStream stream; + private final DataInputStream catalogStream; + private final int zshrink; + private final int zTileLen; private final List columns; private final Integer numberOfRowsInTable; private int numberOfRowsRead = 0; @@ -92,6 +95,9 @@ private ZFITSHeapReader(BinTable binTable) { this.numberOfRowsInTable = binTable.numberOfRowsInTable; this.stream = binTable.heapDataStream; this.columns = binTable.columns; + this.catalogStream = binTable.tableDataStream; + this.zshrink = binTable.getHeader().getInt("ZSHRINK").orElse(1); + this.zTileLen = binTable.getHeader().getInt("ZTILELEN").orElse(1); } @@ -106,6 +112,29 @@ public static ZFITSHeapReader forTable(BinTable binTable) { return new ZFITSHeapReader(binTable); } + @Override + public void skipToRow(int num) throws IOException { + //calc zshrink problem + //num = num+1; + int numRowsSkip = num % (zshrink * zTileLen); // works too, the amount we have to skip with getNext + int numSkip = num - numRowsSkip; // the number of rows that can be skipped + int numTileSkip = numSkip / zshrink / zTileLen; // the amout of tiles we can skip + + int colCount = columns.size(); + long skipBytes = colCount * numTileSkip * (16) + 8;//skip additinal 8 to get directly to the offset + + // the catalog points to the first column so substract 16 to get to the tileheader + long skiped = this.catalogStream.skip(skipBytes); + long tileOffset = this.catalogStream.readLong() - 16; + + this.stream.skip(tileOffset); + this.numberOfRowsRead = numSkip; + + // skip the remaining rows + for (int i = 0; i < numRowsSkip; i++) { + getNextRow(); + } + } /** * Get the data from the next row. The columns in the row can be accessed by their name in the resulting diff --git a/src/test/java/fact/io/hdureader/FitsHDUTests.java b/src/test/java/fact/io/hdureader/FitsHDUTests.java index 5c8cc2ad47..35463a1c84 100644 --- a/src/test/java/fact/io/hdureader/FitsHDUTests.java +++ b/src/test/java/fact/io/hdureader/FitsHDUTests.java @@ -2,6 +2,9 @@ import org.junit.Test; +import stream.io.SourceURL; +import stream.Data; + import java.io.DataInputStream; import java.io.IOException; import java.net.URL; @@ -31,6 +34,61 @@ public void testToOpenInputStream() throws Exception { } + @Test + public void testBinTableReaderSkip() throws Exception { + URL u = FitsHDUTests.class.getResource("/testDataFile.fits.gz"); + FITSStream fits = new FITSStream(new SourceURL(u)); + fits.init(); + FITSStream fits2 = new FITSStream(new SourceURL(u)); + fits2.init(); + + //read 3 + fits.readNext(); + fits.readNext(); + Data item = fits.readNext(); + Reader reader = fits2.getReader(); + reader.skipToRow(2); + Data item2 = fits2.readNext(); + + + int eNr = (int) item.get("EventNum"); + int eNr2 = (int) item2.get("EventNum"); + assertEquals(eNr, eNr2); + + short[] data = (short[]) item.get("Data"); + short[] data2 = (short[]) item2.get("Data"); + assertArrayEquals(data, data2); + } + + @Test + public void testZFitsReaderSkip() throws Exception { + for (int i = 0; i < 5; i++) { + System.out.println("Test skip: " + i); + URL u = FitsHDUTests.class.getResource("/testDataFile.fits.fz"); + FITSStream fits = new FITSStream(new SourceURL(u)); + fits.init(); + FITSStream fits2 = new FITSStream(new SourceURL(u)); + fits2.init(); + //read 3 + for (int j = 0; j < i; j++) { + fits.readNext(); + } + Data item = fits.readNext(); + + Reader reader = fits2.getReader(); + reader.skipToRow(i); + Data item2 = fits2.readNext(); + + + int eNr = (int) item.get("EventNum"); + int eNr2 = (int) item2.get("EventNum"); + assertEquals(eNr, eNr2); + + short[] data = (short[]) item.get("Data"); + short[] data2 = (short[]) item2.get("Data"); + assertArrayEquals(data, data2); + } + } @SuppressWarnings("ResultOfMethodCallIgnored") @Test From 3ee79976404755fcb4528f2e805d3f44ddacb732 Mon Sep 17 00:00:00 2001 From: Michael Bulinski Date: Fri, 16 Mar 2018 13:31:12 +0100 Subject: [PATCH 02/24] added SNAPSHOT to the version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9fa3b79d03..71a9e1bb40 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ de.sfb876 fact-tools fact-tools - 1.0.0 + 1.0.0-SNAPSHOT http://sfb876.de/fact-tools/ From 24a8295d022e7ace8fd2d71b015cd192cc2111f3 Mon Sep 17 00:00:00 2001 From: Michael Bulinski Date: Fri, 16 Mar 2018 13:49:44 +0100 Subject: [PATCH 03/24] small conflict with the version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 71a9e1bb40..b10174dd5a 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ de.sfb876 fact-tools fact-tools - 1.0.0-SNAPSHOT + 1.0.1-SNAPSHOT http://sfb876.de/fact-tools/ From e690fb80ecdf27ac86c4d0f4bbeeb5b3f22d14df Mon Sep 17 00:00:00 2001 From: Michael Bulinski Date: Fri, 16 Mar 2018 16:38:32 +0100 Subject: [PATCH 04/24] changed getReader to skipToRow and fixed bug with ZShrink, also some other small changes --- pom.xml | 2 +- src/main/java/fact/io/hdureader/BinTable.java | 7 +- .../java/fact/io/hdureader/FITSStream.java | 4 +- .../fact/io/hdureader/ZFITSHeapReader.java | 13 ++-- .../java/fact/io/hdureader/FitsHDUTests.java | 66 +++++++++++++++++-- 5 files changed, 71 insertions(+), 21 deletions(-) diff --git a/pom.xml b/pom.xml index b10174dd5a..c6bb21078a 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ de.sfb876 fact-tools fact-tools - 1.0.1-SNAPSHOT + 1.0.2-SNAPSHOT http://sfb876.de/fact-tools/ diff --git a/src/main/java/fact/io/hdureader/BinTable.java b/src/main/java/fact/io/hdureader/BinTable.java index 48f5a554ba..313d4e747b 100644 --- a/src/main/java/fact/io/hdureader/BinTable.java +++ b/src/main/java/fact/io/hdureader/BinTable.java @@ -35,12 +35,7 @@ public class BinTable { final DataInputStream tableDataStream; DataInputStream heapDataStream = null; - private final Header header; - - public Header getHeader() { - return header; - } - + public final Header header; /** * This enum maps the type characters in the header to the fits types. diff --git a/src/main/java/fact/io/hdureader/FITSStream.java b/src/main/java/fact/io/hdureader/FITSStream.java index 262ebb6e01..d396d7580e 100644 --- a/src/main/java/fact/io/hdureader/FITSStream.java +++ b/src/main/java/fact/io/hdureader/FITSStream.java @@ -128,7 +128,7 @@ public Data readNext() throws Exception { return item; } - public Reader getReader() { - return reader; + public void skipToRow(int num) throws IOException{ + reader.skipToRow(num); } } diff --git a/src/main/java/fact/io/hdureader/ZFITSHeapReader.java b/src/main/java/fact/io/hdureader/ZFITSHeapReader.java index 2fd9957167..82ee6a691b 100644 --- a/src/main/java/fact/io/hdureader/ZFITSHeapReader.java +++ b/src/main/java/fact/io/hdureader/ZFITSHeapReader.java @@ -96,8 +96,8 @@ private ZFITSHeapReader(BinTable binTable) { this.stream = binTable.heapDataStream; this.columns = binTable.columns; this.catalogStream = binTable.tableDataStream; - this.zshrink = binTable.getHeader().getInt("ZSHRINK").orElse(1); - this.zTileLen = binTable.getHeader().getInt("ZTILELEN").orElse(1); + this.zshrink = binTable.header.getInt("ZSHRINK").orElse(1); + this.zTileLen = binTable.header.getInt("ZTILELEN").orElse(1); } @@ -114,17 +114,16 @@ public static ZFITSHeapReader forTable(BinTable binTable) { @Override public void skipToRow(int num) throws IOException { - //calc zshrink problem - //num = num+1; - int numRowsSkip = num % (zshrink * zTileLen); // works too, the amount we have to skip with getNext + int numRowsSkip = num % (zTileLen); // works too, the amount we have to skip with getNext int numSkip = num - numRowsSkip; // the number of rows that can be skipped - int numTileSkip = numSkip / zshrink / zTileLen; // the amout of tiles we can skip + int numTileSkip = numSkip / zTileLen; // the amout of tiles we can skip int colCount = columns.size(); long skipBytes = colCount * numTileSkip * (16) + 8;//skip additinal 8 to get directly to the offset - // the catalog points to the first column so substract 16 to get to the tileheader long skiped = this.catalogStream.skip(skipBytes); + if (skiped != skipBytes) + throw new IOException("Couldn't skip to the desired position. The file is broken."); long tileOffset = this.catalogStream.readLong() - 16; this.stream.skip(tileOffset); diff --git a/src/test/java/fact/io/hdureader/FitsHDUTests.java b/src/test/java/fact/io/hdureader/FitsHDUTests.java index 35463a1c84..d192093a4f 100644 --- a/src/test/java/fact/io/hdureader/FitsHDUTests.java +++ b/src/test/java/fact/io/hdureader/FitsHDUTests.java @@ -46,8 +46,7 @@ public void testBinTableReaderSkip() throws Exception { fits.readNext(); fits.readNext(); Data item = fits.readNext(); - Reader reader = fits2.getReader(); - reader.skipToRow(2); + fits2.skipToRow(2); Data item2 = fits2.readNext(); @@ -69,14 +68,71 @@ public void testZFitsReaderSkip() throws Exception { fits.init(); FITSStream fits2 = new FITSStream(new SourceURL(u)); fits2.init(); - //read 3 + + for (int j = 0; j < i; j++) { + fits.readNext(); + } + Data item = fits.readNext(); + + fits2.skipToRow(i); + Data item2 = fits2.readNext(); + + + int eNr = (int) item.get("EventNum"); + int eNr2 = (int) item2.get("EventNum"); + assertEquals(eNr, eNr2); + + short[] data = (short[]) item.get("Data"); + short[] data2 = (short[]) item2.get("Data"); + assertArrayEquals(data, data2); + } + } + + @Test + public void testZFitsReaderSkipWithZShrink() throws Exception { + for (int i = 0; i < 5; i++) { + System.out.println("Test skip: " + i); + URL u = FitsHDUTests.class.getResource("/testDataFileZSHRINK.fits.fz"); + FITSStream fits = new FITSStream(new SourceURL(u)); + fits.init(); + FITSStream fits2 = new FITSStream(new SourceURL(u)); + fits2.init(); + + for (int j = 0; j < i; j++) { + fits.readNext(); + } + Data item = fits.readNext(); + + fits2.skipToRow(i); + Data item2 = fits2.readNext(); + + + int eNr = (int) item.get("EventNum"); + int eNr2 = (int) item2.get("EventNum"); + assertEquals(eNr, eNr2); + + short[] data = (short[]) item.get("Data"); + short[] data2 = (short[]) item2.get("Data"); + assertArrayEquals(data, data2); + } + } + + @Test + public void testZFitsReaderSkipWithZTileLen() throws Exception { + for (int i = 98; i < 102; i++) { + System.out.println("Test skip: " + i); + URL u = FitsHDUTests.class.getResource("/testDataFileZTILELEN.fits.fz"); + FITSStream fits = new FITSStream(new SourceURL(u)); + fits.init(); + FITSStream fits2 = new FITSStream(new SourceURL(u)); + fits2.init(); + for (int j = 0; j < i; j++) { fits.readNext(); } Data item = fits.readNext(); - Reader reader = fits2.getReader(); - reader.skipToRow(i); + fits2.skipToRow(i); Data item2 = fits2.readNext(); From a8c6a42bcf8414d9512ac156240e8adf5fe32316 Mon Sep 17 00:00:00 2001 From: Michael Bulinski Date: Mon, 19 Mar 2018 09:34:06 +0100 Subject: [PATCH 05/24] remodeled the skipToRow function into a skipRows function --- .../fact/io/hdureader/BinTableReader.java | 10 +-- .../java/fact/io/hdureader/FITSStream.java | 10 ++- src/main/java/fact/io/hdureader/Reader.java | 2 +- .../fact/io/hdureader/ZFITSHeapReader.java | 90 +++++++++++++++++-- .../java/fact/io/hdureader/FitsHDUTests.java | 8 +- 5 files changed, 101 insertions(+), 19 deletions(-) diff --git a/src/main/java/fact/io/hdureader/BinTableReader.java b/src/main/java/fact/io/hdureader/BinTableReader.java index bfe97973ba..f2e39ebd76 100644 --- a/src/main/java/fact/io/hdureader/BinTableReader.java +++ b/src/main/java/fact/io/hdureader/BinTableReader.java @@ -145,11 +145,11 @@ private Serializable readArrayFromStream(BinTable.TableColumn c, DataInputStream } @Override - public void skipToRow(int num) throws IOException { - if (num <= numberOfRowsInTable) { - new IndexOutOfBoundsException("Table has not enough rows to access row num: " + num); + public void skipRows(int amount) throws IOException { + if (amount+numberOfRowsRead <= numberOfRowsInTable) { + new IndexOutOfBoundsException("Table has not enough rows to access row num: " + (amount+numberOfRowsRead)); } - stream.skipBytes(num * this.numberOfBytesPerRow); - numberOfRowsRead = num; + stream.skipBytes(amount * this.numberOfBytesPerRow); + numberOfRowsRead += amount; } } diff --git a/src/main/java/fact/io/hdureader/FITSStream.java b/src/main/java/fact/io/hdureader/FITSStream.java index d396d7580e..56aac8138c 100644 --- a/src/main/java/fact/io/hdureader/FITSStream.java +++ b/src/main/java/fact/io/hdureader/FITSStream.java @@ -128,7 +128,13 @@ public Data readNext() throws Exception { return item; } - public void skipToRow(int num) throws IOException{ - reader.skipToRow(num); + /** + * Skips the given amount of rows in the data table. + * + * @param amount The amount of Rows to skip. + * @throws IOException + */ + public void skipRows(int amount) throws IOException{ + reader.skipRows(amount); } } diff --git a/src/main/java/fact/io/hdureader/Reader.java b/src/main/java/fact/io/hdureader/Reader.java index c63edaa63b..255785347c 100644 --- a/src/main/java/fact/io/hdureader/Reader.java +++ b/src/main/java/fact/io/hdureader/Reader.java @@ -78,6 +78,6 @@ default Iterator> iterator() { * @param num The amount of rows to skip. * @throws IOException */ - void skipToRow(int num) throws IOException; + void skipRows(int amount) throws IOException; } diff --git a/src/main/java/fact/io/hdureader/ZFITSHeapReader.java b/src/main/java/fact/io/hdureader/ZFITSHeapReader.java index 82ee6a691b..ba130267d8 100644 --- a/src/main/java/fact/io/hdureader/ZFITSHeapReader.java +++ b/src/main/java/fact/io/hdureader/ZFITSHeapReader.java @@ -66,6 +66,7 @@ public final class ZFITSHeapReader implements Reader { private final DataInputStream stream; private final DataInputStream catalogStream; + private int catalogPosition = 0; private final int zshrink; private final int zTileLen; private final List columns; @@ -113,18 +114,93 @@ public static ZFITSHeapReader forTable(BinTable binTable) { } @Override - public void skipToRow(int num) throws IOException { - int numRowsSkip = num % (zTileLen); // works too, the amount we have to skip with getNext - int numSkip = num - numRowsSkip; // the number of rows that can be skipped + public void skipRows(int amount) throws IOException { + int resultingRow = amount+numberOfRowsRead; + if (resultingRow >= this.numberOfRowsInTable) { + throw new IOException("Not enough rows in table, need "+(amount+numberOfRowsRead)+" have "+numberOfRowsInTable); + } + + int remainer = zTileLen - numberOfRowsRead%zTileLen; + if (amount rowCatalogPosition) { + // align them the difference if always maximal 1 + for (int i = 0; i < zTileLen; i++) { + getNextRow(); + } + rowCatalogPosition += 1; + } + // check if we can't skip using the catalog due to being in the same catalog position + if (rowCatalogPosition == rowCatalogPositionFinished) { + for (int i = 0; i < amount; i++) { + getNextRow(); + } + return; + } + + // move the current catalog position to the rowCatalogPosition + long skipBytes = (rowCatalogPosition - catalogPosition) * columns.size() * (16); + long skiped = this.catalogStream.skip(skipBytes); + this.catalogPosition = rowCatalogPosition; + + // get current row position + this.catalogStream.skip(8); // go directly to the offset + long rowOffset = this.catalogStream.readLong() - 16; // read the offset - 16 for the header + this.catalogStream.skip(columns.size() * (16) - 8); // go to the next catalog start + this.catalogPosition += 1; + + // go to the finishing position catalog + int diffCatalogs = rowCatalogPositionFinished-catalogPosition; + skipBytes = diffCatalogs * columns.size() * (16) + 8; // go directly to the offset + this.catalogStream.skip(skipBytes); + long finalRowOffset = this.catalogStream.readLong() - 16; // read the offset - 16 for the header + + skipBytes = rowOffset - finalRowOffset; + this.stream.skip(skipBytes); + this.numberOfRowsRead += diffCatalogs * this.zTileLen; + + int remainingRows = resultingRow % zTileLen; + for (int i=0; i Date: Mon, 19 Mar 2018 10:21:04 +0100 Subject: [PATCH 06/24] added docs and fixed bug in ZFitsHeapReader#skipRows --- .../fact/io/hdureader/BinTableReader.java | 6 +++ .../fact/io/hdureader/ZFITSHeapReader.java | 50 ++++++++----------- .../java/fact/io/hdureader/FitsHDUTests.java | 5 +- 3 files changed, 28 insertions(+), 33 deletions(-) diff --git a/src/main/java/fact/io/hdureader/BinTableReader.java b/src/main/java/fact/io/hdureader/BinTableReader.java index f2e39ebd76..c2402dc2ca 100644 --- a/src/main/java/fact/io/hdureader/BinTableReader.java +++ b/src/main/java/fact/io/hdureader/BinTableReader.java @@ -144,6 +144,12 @@ private Serializable readArrayFromStream(BinTable.TableColumn c, DataInputStream return null; } + /** + * Skips the given number of rows. + * + * @param num The amount of rows to skip. + * @throws IOException + */ @Override public void skipRows(int amount) throws IOException { if (amount+numberOfRowsRead <= numberOfRowsInTable) { diff --git a/src/main/java/fact/io/hdureader/ZFITSHeapReader.java b/src/main/java/fact/io/hdureader/ZFITSHeapReader.java index ba130267d8..35fec5debe 100644 --- a/src/main/java/fact/io/hdureader/ZFITSHeapReader.java +++ b/src/main/java/fact/io/hdureader/ZFITSHeapReader.java @@ -113,6 +113,12 @@ public static ZFITSHeapReader forTable(BinTable binTable) { return new ZFITSHeapReader(binTable); } + /** + * Skips the given number of rows. + * + * @param num The amount of rows to skip. + * @throws IOException + */ @Override public void skipRows(int amount) throws IOException { int resultingRow = amount+numberOfRowsRead; @@ -120,6 +126,7 @@ public void skipRows(int amount) throws IOException { throw new IOException("Not enough rows in table, need "+(amount+numberOfRowsRead)+" have "+numberOfRowsInTable); } + // check if resulting row is in the current catalog remainer int remainer = zTileLen - numberOfRowsRead%zTileLen; if (amount Date: Thu, 22 Mar 2018 18:10:52 +0100 Subject: [PATCH 07/24] fixed error with skip not skipping entire amount --- .../fact/io/hdureader/ZFITSHeapReader.java | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/main/java/fact/io/hdureader/ZFITSHeapReader.java b/src/main/java/fact/io/hdureader/ZFITSHeapReader.java index 35fec5debe..beb5e30a7c 100644 --- a/src/main/java/fact/io/hdureader/ZFITSHeapReader.java +++ b/src/main/java/fact/io/hdureader/ZFITSHeapReader.java @@ -116,7 +116,7 @@ public static ZFITSHeapReader forTable(BinTable binTable) { /** * Skips the given number of rows. * - * @param num The amount of rows to skip. + * @param amount The amount of rows to skip. * @throws IOException */ @Override @@ -126,6 +126,10 @@ public void skipRows(int amount) throws IOException { throw new IOException("Not enough rows in table, need "+(amount+numberOfRowsRead)+" have "+numberOfRowsInTable); } + /*for (int i=0; i Date: Mon, 30 Apr 2018 22:18:32 +0200 Subject: [PATCH 08/24] add drsfile property as @drsKey to data item, delete drsfile from std calibration XML --- examples/27s.xml | 1 + examples/apply_model.xml | 1 + examples/fellwalker_example.xml | 1 + examples/mc_viewer.xml | 1 + examples/measure_performance.xml | 1 + examples/save_dl1.xml | 1 + examples/stdAnalysis.xml | 1 + examples/studies/extractionTest.xml | 1 + examples/studies/muon_fitting.xml | 1 + examples/studies/muon_identification.xml | 1 + examples/studies/simulations_pass4.xml | 1 + examples/studies/singlePeExtractor/singlePeMinimalExample.xml | 3 ++- .../singlePeExtractor/std_analysis_on_reconstructed_data.xml | 3 ++- examples/viewer.xml | 1 + src/main/resources/analysis/calibration.xml | 1 - 15 files changed, 16 insertions(+), 3 deletions(-) diff --git a/examples/27s.xml b/examples/27s.xml index f79e072c07..8cfc4aba1b 100644 --- a/examples/27s.xml +++ b/examples/27s.xml @@ -20,6 +20,7 @@ + diff --git a/examples/apply_model.xml b/examples/apply_model.xml index 756865626f..1b48e25fec 100644 --- a/examples/apply_model.xml +++ b/examples/apply_model.xml @@ -27,6 +27,7 @@ + diff --git a/examples/fellwalker_example.xml b/examples/fellwalker_example.xml index cd093ffc62..57edbc5840 100644 --- a/examples/fellwalker_example.xml +++ b/examples/fellwalker_example.xml @@ -19,6 +19,7 @@ + diff --git a/examples/mc_viewer.xml b/examples/mc_viewer.xml index 5f1695df5d..7db69a506c 100644 --- a/examples/mc_viewer.xml +++ b/examples/mc_viewer.xml @@ -20,6 +20,7 @@ + diff --git a/examples/measure_performance.xml b/examples/measure_performance.xml index ae8dc2cfcc..3f71d905af 100644 --- a/examples/measure_performance.xml +++ b/examples/measure_performance.xml @@ -20,6 +20,7 @@ + diff --git a/examples/save_dl1.xml b/examples/save_dl1.xml index adebd2e1dc..16fee1d92c 100644 --- a/examples/save_dl1.xml +++ b/examples/save_dl1.xml @@ -20,6 +20,7 @@ + diff --git a/examples/stdAnalysis.xml b/examples/stdAnalysis.xml index ba72995353..bd88e4450d 100644 --- a/examples/stdAnalysis.xml +++ b/examples/stdAnalysis.xml @@ -22,6 +22,7 @@ + diff --git a/examples/studies/extractionTest.xml b/examples/studies/extractionTest.xml index c3c9023fc7..2ab26f3f23 100644 --- a/examples/studies/extractionTest.xml +++ b/examples/studies/extractionTest.xml @@ -13,6 +13,7 @@ + diff --git a/examples/studies/muon_fitting.xml b/examples/studies/muon_fitting.xml index cafb41b84c..8b32265f4c 100644 --- a/examples/studies/muon_fitting.xml +++ b/examples/studies/muon_fitting.xml @@ -19,6 +19,7 @@ + diff --git a/examples/studies/muon_identification.xml b/examples/studies/muon_identification.xml index 2032be0b50..91e7e6174b 100644 --- a/examples/studies/muon_identification.xml +++ b/examples/studies/muon_identification.xml @@ -32,6 +32,7 @@ + diff --git a/examples/studies/simulations_pass4.xml b/examples/studies/simulations_pass4.xml index 03ad25d517..c321c7120a 100644 --- a/examples/studies/simulations_pass4.xml +++ b/examples/studies/simulations_pass4.xml @@ -15,6 +15,7 @@ + diff --git a/examples/studies/singlePeExtractor/singlePeMinimalExample.xml b/examples/studies/singlePeExtractor/singlePeMinimalExample.xml index e819048903..74d72c48df 100644 --- a/examples/studies/singlePeExtractor/singlePeMinimalExample.xml +++ b/examples/studies/singlePeExtractor/singlePeMinimalExample.xml @@ -22,7 +22,8 @@ - + + diff --git a/examples/studies/singlePeExtractor/std_analysis_on_reconstructed_data.xml b/examples/studies/singlePeExtractor/std_analysis_on_reconstructed_data.xml index d70fb53819..a6c3dc41a8 100644 --- a/examples/studies/singlePeExtractor/std_analysis_on_reconstructed_data.xml +++ b/examples/studies/singlePeExtractor/std_analysis_on_reconstructed_data.xml @@ -24,7 +24,8 @@ - + + + diff --git a/src/main/resources/analysis/calibration.xml b/src/main/resources/analysis/calibration.xml index 774b2d4706..5087a2eee0 100644 --- a/src/main/resources/analysis/calibration.xml +++ b/src/main/resources/analysis/calibration.xml @@ -9,7 +9,6 @@ /> From 372ca8a739b76d48885d3f8f4277bbab39881072 Mon Sep 17 00:00:00 2001 From: Jens Buss Date: Mon, 30 Apr 2018 22:20:28 +0200 Subject: [PATCH 09/24] change auxfile service --- examples/forErna/std_analysis_erna_observations.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/forErna/std_analysis_erna_observations.xml b/examples/forErna/std_analysis_erna_observations.xml index d9c0dc0528..5ea2ebd39b 100644 --- a/examples/forErna/std_analysis_erna_observations.xml +++ b/examples/forErna/std_analysis_erna_observations.xml @@ -9,7 +9,7 @@ - + From 03f665e4944b714a5cda5983175c9e017671ab6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Wed, 2 May 2018 10:31:52 +0200 Subject: [PATCH 10/24] Update to streams 1.0.5 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index d8db7cecf3..fd9b300adb 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ de.sfb876 fact-tools fact-tools - 1.0.1 + 1.0.2-SNAPSHOT http://sfb876.de/fact-tools/ @@ -15,7 +15,7 @@ UTF-8 compile - 1.0.0 + 1.0.5 fact-tools-${git.commit.id.describe} false fact-tools-${git.commit.id.describe} From 7e57c8ccdb330cd4009159401d1364dfc9104250 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Wed, 2 May 2018 10:33:36 +0200 Subject: [PATCH 11/24] Set log levels for streams to ERROR --- src/main/resources/log4j.properties | 5 +++-- src/test/resources/log4j.properties | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties index 1ab410878f..f351164eb1 100644 --- a/src/main/resources/log4j.properties +++ b/src/main/resources/log4j.properties @@ -13,9 +13,10 @@ log4j.appender.A1=org.apache.log4j.ConsoleAppender log4j.appender.A1.layout=org.apache.log4j.PatternLayout log4j.appender.A1.layout.ConversionPattern=%d{MMM dd yyyy HH:mm:ss} [%-5p] %C: %m%n -log4j.logger.stream=INFO +log4j.logger.stream=ERROR +log4j.logger.stream.util=ERROR log4j.logger.stream.run=ERROR -log4j.logger.stream.runtime=INFO +log4j.logger.stream.runtime=ERROR log4j.logger.net.scinotes=INFO log4j.logger.net=INFO log4j.logger.org.springframework=INFO diff --git a/src/test/resources/log4j.properties b/src/test/resources/log4j.properties index a2c9a15161..1cbcff9790 100644 --- a/src/test/resources/log4j.properties +++ b/src/test/resources/log4j.properties @@ -14,6 +14,7 @@ log4j.appender.A1.layout=org.apache.log4j.PatternLayout log4j.appender.A1.layout.ConversionPattern=%d{MMM dd yyyy HH:mm:ss} [%-5p] %C: %m%n log4j.logger.fact=INFO -#log4j.logger.fact.io=DEBUG -#log4j.logger.org.apache=DEBUG -#log4j.logger.com=DEBUG +log4j.logger.stream=ERROR +log4j.logger.stream.util=ERROR +log4j.logger.stream.run=ERROR +log4j.logger.stream.runtime=ERROR From afc2a38072c697e68f3473296dc174093464960c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Wed, 2 May 2018 10:57:42 +0200 Subject: [PATCH 12/24] Delete AUTHORS.md This file is horribly outdated and the same information is in the pom, the zenodo.json and the git history. --- AUTHORS.md | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 AUTHORS.md diff --git a/AUTHORS.md b/AUTHORS.md deleted file mode 100644 index cff9a2ef0a..0000000000 --- a/AUTHORS.md +++ /dev/null @@ -1,15 +0,0 @@ -Authors -======= - -The following people have contributed to the *fact-tools* project: - - * Christian Bockermann - * Kai Bruegge - * Fabian Temme - * Jens Buss - * Dominik Neise - * Jan Freiwald - * Maximilian Nöthe - * Katie Gray - * Michael Bulinski - * Dominik Baack \ No newline at end of file From 1f70555d6af4a5696260dfd523c509fed03a2cd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Wed, 2 May 2018 11:28:38 +0200 Subject: [PATCH 13/24] Fix studies test for streams 1.0.5 --- src/test/java/fact/FunctionalTest.java | 13 ++++++++--- src/test/java/fact/RunFACTTools.java | 30 +++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/src/test/java/fact/FunctionalTest.java b/src/test/java/fact/FunctionalTest.java index 5f9611fc30..050fcc0559 100644 --- a/src/test/java/fact/FunctionalTest.java +++ b/src/test/java/fact/FunctionalTest.java @@ -18,6 +18,7 @@ import static fact.RunFACTTools.runFACTTools; import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; @@ -58,6 +59,12 @@ public void studiesXMLs() throws IOException { String[] args = {f.toAbsolutePath().toString()}; try { runFACTTools(args); + } catch (RunFACTTools.SystemExitException e) { + if (e.status != 0) { + log.error("Error executing xml: " + f, e); + failedFilesList.add(f.toString()); + counter++; + } } catch (Exception e) { log.error("Error executing xml: " + f, e); failedFilesList.add(f.toString()); @@ -65,9 +72,9 @@ public void studiesXMLs() throws IOException { } } - log.debug("\n\n" + counter + " of " + size + " files in 'examples/studies' failed to execute"); - log.debug(Arrays.toString(failedFilesList.toArray())); + String msg = counter + " of " + size + " files in 'examples/studies' failed to execute: "; + msg += Arrays.toString(failedFilesList.toArray()); - assertThat(failedFilesList.size(), is(0)); + assertEquals(msg, 0, failedFilesList.size()); } } diff --git a/src/test/java/fact/RunFACTTools.java b/src/test/java/fact/RunFACTTools.java index 13122af2ae..b893bf2c15 100644 --- a/src/test/java/fact/RunFACTTools.java +++ b/src/test/java/fact/RunFACTTools.java @@ -1,6 +1,26 @@ package fact; +import java.security.Permission; + public class RunFACTTools { + + static class SystemExitException extends SecurityException { + final public int status; + + SystemExitException(int status) { + super("SystemExit called with status code " + status); + this.status = status; + } + } + + static class CatchSystemExitSecurityManager extends SecurityManager { + @Override public void checkExit(int status) { + throw new SystemExitException(status); + } + + @Override public void checkPermission(Permission perm) {} + } + public static void runFACTTools(String[] args) throws Exception { System.clearProperty("drsfile"); System.clearProperty("infile"); @@ -10,6 +30,14 @@ public static void runFACTTools(String[] args) throws Exception { System.clearProperty("integralGainFile"); System.clearProperty("aux_source"); System.clearProperty("output"); - fact.run.main(args); + + // Catch the system exit stream calls when an error happens + SecurityManager before = System.getSecurityManager(); + System.setSecurityManager(new CatchSystemExitSecurityManager()); + try { + fact.run.main(args); + } finally { + System.setSecurityManager(before); + } } } From 06199ddcd5d4e1930d8eb0a0ddae129985f80254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Wed, 2 May 2018 12:29:41 +0200 Subject: [PATCH 14/24] Also interpolate non-finite values in InterPolatePixelArray --- .../datacorrection/InterpolatePixelArray.java | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/src/main/java/fact/datacorrection/InterpolatePixelArray.java b/src/main/java/fact/datacorrection/InterpolatePixelArray.java index 15e0566aa4..cf29b8133e 100644 --- a/src/main/java/fact/datacorrection/InterpolatePixelArray.java +++ b/src/main/java/fact/datacorrection/InterpolatePixelArray.java @@ -1,5 +1,6 @@ package fact.datacorrection; +import fact.Constants; import fact.Utils; import fact.calibrationservice.CalibrationService; import fact.container.PixelSet; @@ -18,7 +19,10 @@ /** * * This Processor interpolates all values for a broken Pixel by the average values of its neighboring Pixels. - * @author Kai Bruegge <kai.bruegge@tu-dortmund.de> + * It will interpolate the pixels from the GainService.badPixels pixelset as well as all pixels having not-finite + * values in the input array. + * + * @author Kai Bruegge <kai.bruegge@tu-dortmund.de> * */ public class InterpolatePixelArray implements Processor { @@ -48,7 +52,7 @@ public Data process(Data item) { ZonedDateTime timeStamp; - if (item.containsKey("UnixTimeUTC") == true) { + if (item.containsKey("UnixTimeUTC")) { Utils.isKeyValid(item, "UnixTimeUTC", int[].class); int[] eventTime = (int[]) item.get("UnixTimeUTC"); timeStamp = Utils.unixTimeUTCToZonedDateTime(eventTime); @@ -57,7 +61,6 @@ public Data process(Data item) { // => The 12 bad pixels we have from the beginning on are used. timeStamp = ZonedDateTime.of(2000, 1, 1, 0, 0,0,0, ZoneOffset.UTC); } - PixelSet badPixelsSet = calibService.getBadPixels(timeStamp); double[] output; @@ -73,19 +76,22 @@ public Data process(Data item) { } double[] interpolatePixelArray(double[] pixelArray, PixelSet badPixels) { - for (CameraPixel pixel: badPixels){ - CameraPixel[] currentNeighbors = pixelMap.getNeighborsForPixel(pixel); - double avg = 0.0; - int numNeighbours = 0; - for (CameraPixel neighbor: currentNeighbors){ - if (badPixels.contains(neighbor)) { - continue; + for (int chid = 0; chid < Constants.N_PIXELS; chid++){ + if (badPixels.containsID(chid) || (!Double.isFinite(pixelArray[chid]))) { + CameraPixel[] currentNeighbors = pixelMap.getNeighborsFromID(chid); + + double avg = 0.0; + int numNeighbours = 0; + for (CameraPixel neighbor: currentNeighbors){ + if (badPixels.contains(neighbor) || (!Double.isFinite(pixelArray[neighbor.id]))) { + continue; + } + avg += pixelArray[neighbor.id]; + numNeighbours++; } - avg += pixelArray[neighbor.id]; - numNeighbours++; + checkNumNeighbours(numNeighbours, chid); + pixelArray[chid] = avg / numNeighbours; } - checkNumNeighbours(numNeighbours, pixel.id); - pixelArray[pixel.id] = avg / numNeighbours; } return pixelArray; } From afc88d770e9452627171430ef3d2ce4687176c99 Mon Sep 17 00:00:00 2001 From: Michael Bulinski Date: Wed, 2 May 2018 12:42:40 +0200 Subject: [PATCH 15/24] removed uncommented code lines --- src/main/java/fact/io/hdureader/ZFITSHeapReader.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/main/java/fact/io/hdureader/ZFITSHeapReader.java b/src/main/java/fact/io/hdureader/ZFITSHeapReader.java index beb5e30a7c..597af7c84d 100644 --- a/src/main/java/fact/io/hdureader/ZFITSHeapReader.java +++ b/src/main/java/fact/io/hdureader/ZFITSHeapReader.java @@ -125,11 +125,7 @@ public void skipRows(int amount) throws IOException { if (resultingRow >= this.numberOfRowsInTable) { throw new IOException("Not enough rows in table, need "+(amount+numberOfRowsRead)+" have "+numberOfRowsInTable); } - - /*for (int i=0; i Date: Wed, 2 May 2018 13:12:09 +0200 Subject: [PATCH 16/24] fixe errors in the testKeys function and the call of it in the fitsWriter --- src/main/java/fact/io/FITSWriter.java | 2 +- src/main/java/fact/io/Writer.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/fact/io/FITSWriter.java b/src/main/java/fact/io/FITSWriter.java index 4c49f11b1f..56aa2d1f58 100644 --- a/src/main/java/fact/io/FITSWriter.java +++ b/src/main/java/fact/io/FITSWriter.java @@ -82,7 +82,7 @@ public Data process(Data item) { for (String key : keys.select(item)) { outputItem.put(key, item.get(key)); } - testKeys(item, keys, allowNullKeys); + testKeys(outputItem, keys, allowNullKeys); if (!initialized) { try { diff --git a/src/main/java/fact/io/Writer.java b/src/main/java/fact/io/Writer.java index 3038d7212a..3707589c7c 100644 --- a/src/main/java/fact/io/Writer.java +++ b/src/main/java/fact/io/Writer.java @@ -71,7 +71,8 @@ public void testKeys(Data item, Keys keys, boolean allowNullKeys) { previousKeySet = item.keySet(); } else { if (!previousKeySet.equals(item.keySet())) { - Set diff1 = item.keySet(); + Set diff1 = new HashSet(); + diff1.addAll(item.keySet()); diff1.removeAll(previousKeySet); Set diff2 = previousKeySet; diff2.removeAll(item.keySet()); From 605c66fe4c81ccf8be5a97387d90f9b2532bef5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Wed, 2 May 2018 15:33:20 +0200 Subject: [PATCH 17/24] Update author metadata files --- .mailmap | 80 ++++++++++++++++++++++++++++++++++++++++++ .zenodo.json | 82 +++++++++++++++++++++++++------------------ pom.xml | 98 ++++++++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 213 insertions(+), 47 deletions(-) create mode 100644 .mailmap diff --git a/.mailmap b/.mailmap new file mode 100644 index 0000000000..0f50062082 --- /dev/null +++ b/.mailmap @@ -0,0 +1,80 @@ +# Kai Brügge +Kai Brügge Kai +Kai Brügge Kai B +Kai Brügge Kai Bruegge +Kai Brügge Kai Bruegge +Kai Brügge Kai Brügge +Kai Brügge Kai Brügge +Kai Brügge Kai MacGyver +Kai Brügge MacKaiver +Kai Brügge kai +Kai Brügge kai +Kai Brügge kai +Kai Brügge kai +Kai Brügge kai +Kai Brügge mackaiver +Kai Brügge mackaiver +Kai Brügge kai + +# Maximilian Nöthe +Maximilian Nöthe Maximilian Noethe +Maximilian Nöthe Maximilian Nöthe + +# Jens Buß +Jens Buß jebuss +Jens Buß Jens Buss +Jens Buß jens buß +Jens Buß jens buß +Jens Buß jebuss +Jens Buß Jens Buss +Jens Buß Jens Buss +Jens Buß jbuss + + +# Michael Bulinski +Michael Bulinski fact_opr +Michael Bulinski fact_opr + +# Dominik Baack +Dominik Baack Dominik Baack +Dominik Baack Matyro +Dominik Baack Dominik +Dominik Baack Dominik +Dominik Baack Dominik B + +# Dominik Neise +Dominik Neise Dominik Neise +Dominik Neise Dominik Neise + +# Maximilian Nöthe +Maximilian Nöthe Maximilian Noethe +Maximilian Nöthe Maximilian Noethe + +# Max Ludwig Ahnen +Max L. Ahnen Max +Max L. Ahnen Max Ahnen +Max L. Ahnen mahnen +Max L. Ahnen mknoetig + +# Sebastian Achim Mueller +Sebastian A. Mueller sebastian + +# Fabian Temme +Fabian Temme ftemme + +# Kai Schenetten +Kai Schenetten kschennetten + +# Lena Linhoff +Lena Linhoff Lena +Lena Linhoff Lena Linhoff +Lena Linhoff Lena Linhoff +Lena Linhoff lena +Lena Linhoff lena-lin + +# Alexey Egoroov +Alexey Egorov Alexey Egorov +Alexey Egorov Alexey + +# Christian Bockermann +Christian Bockermann Christian Bockermann diff --git a/.zenodo.json b/.zenodo.json index d5f05ec4a2..2ca071c6a8 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -1,6 +1,6 @@ { - "description": "The FACT-Tools are an extension of the streams framework to analyse the data of the First G-APD Cherenkov Telescope", - "license": "lgpl-3.0", + "description": "The FACT-Tools are an extension of the streams framework to analyse the data of the First G-APD Cherenkov Telescope", + "license": "lgpl-3.0", "keywords": [ "gamma-ray astronomy", "streams", @@ -11,57 +11,73 @@ "upload_type": "software", "creators": [ { - "affiliation": "TU Dortmund", - "name": "Maximilian N\u00f6the" - }, + "affiliation": "TU Dortmund", + "name": "Kai Br\u00fcgge" + }, { - "affiliation": "TU Dortmund", - "name": "Jens Buss" - }, + "affiliation": "TU Dortmund", + "name": "Maximilian N\u00f6the" + }, { - "affiliation": "TU Dortmund", - "name": "Kai Br\u00fcgge" - }, + "affiliation": "TU Dortmund", + "name": "Jens Bu\u00df" + }, { - "affiliation": "TU Dortmund", + "affiliation": "TU Dortmund", "name": "T. Fabian Temme" - }, + }, + { + "affiliation": "TU Dortmund", + "name": "Michael Bulinski" + }, { - "affiliation": "ETH Zurich", + "affiliation": "ETH Zurich", "name": "Sebastian A. Mueller" - }, + }, { - "affiliation": "TU Dortmund", + "affiliation": "TU Dortmund", "name": "Christian Bockermann" - }, + }, { - "affiliation": "TU Dortmund", + "affiliation": "TU Dortmund", "name": "Lena Linhoff" - }, + }, + { + "affiliation": "TU Dortmund", + "name": "Jan Freiwald" + }, { - "affiliation": "ETH Zurich", + "affiliation": "ETH Zurich", "name": "Dominik Neise" - }, + }, { - "affiliation": "TU Dortmund", + "affiliation": "TU Dortmund", "name": "Dominik Baack" - }, + }, { - "affiliation": "TU Dortmund", + "affiliation": "TU Dortmund", "name": "Hendrik Hildebrandt" - }, + }, { - "affiliation": "ETH Zurich", - "name": "Max Ahnen" - }, + "affiliation": "TU Dortmund", + "name": "Katie Gray" + }, { - "affiliation": "TU Dortmund", - "name": "Niklas Wulf" - }, + "affiliation": "ETH Zurich", + "name": "Max L. Ahnen" + }, { - "affiliation": "TU Dortmund", + "affiliation": "TU Dortmund", "name": "Alexey Egorov" + }, + { + "affiliation": "TU Dortmund", + "name": "Kai Schenetten" + }, + { + "affiliation": "TU Dortmund", + "name": "Niklas Wulf" } - ], + ], "access_right": "open" } diff --git a/pom.xml b/pom.xml index d8db7cecf3..bc53ec9c55 100644 --- a/pom.xml +++ b/pom.xml @@ -29,6 +29,48 @@ + + Kai Brügge + kai.bruegge@tu-dortmund.de + Physik E5b, TU-Dortmund + http://app.tu-dortmund.de + +1 + + + Maximilian Nöthe + maximilian.noethe@tu-dortmund.de + Physik E5b, TU Dortmund + http://app.tu-dortmund.de + +1 + + + Jens Buß + jens.buss@tu-dortmund.de + Physik E5b, TU-Dortmund + http://app.tu-dortmund.de + +1 + + + Fabian Temme + fabian.temme@tu-dortmund.de + Physik E5b, TU Dortmund + http://app.tu-dortmund.de + +1 + + + Michael Bulinski + michael.bulinski@tu-dortmund.de + Physik E5b, TU Dortmund + http://app.tu-dortmund.de + +1 + + + Sebastian A. Mueller + sebmuell@phys.ethz.ch + Departement of physics, ETH Zurich + http://www.ipp.phys.ethz.ch/ + +1 + Christian Bockermann christian.bockermann@cs.uni-dortmund.de @@ -37,16 +79,16 @@ +1 - Kai Brügge - kai.bruegge@tu-dortmund.de - Physik E5b, TU-Dortmund + Lena Linhoff + lena.linhoff@tu-dortmund.de + Physik E5b, TU Dortmund http://app.tu-dortmund.de +1 - Jens Buss - jens.buss@tu-dortmund.de - Physik E5b, TU-Dortmund + Jan Freiwald + jan.freiwald@tu-dortmund.de + Physik E5b, TU Dortmund http://app.tu-dortmund.de +1 @@ -58,26 +100,54 @@ +1 - Sebastian Müller - sebmuell@phys.ethz.ch - Departement of physics, ETH Zurich - http://www.ipp.phys.ethz.ch/ + Dominik Baack + dominik.baack@tu-dortmund.de + Physik E5b, TU Dortmund + http://app.tu-dortmund.de +1 - Fabian Temme - fabian.temme@tu-dortmund.de + Hendrik Hildebrandt + hendrik.hildebrandt@tu-dortmund.de Physik E5b, TU Dortmund http://app.tu-dortmund.de +1 - Maximilian Nöthe - maximilian.noethe@tu-dortmund.de + Katie Gray + grayka@whitman.edu Physik E5b, TU Dortmund http://app.tu-dortmund.de +1 + + Max L. Ahnen + m.knoetig@gmail.com + +1 + Departement of physics, ETH Zurich + http://www.ipp.phys.ethz.ch/ + + + Alexey Egorov + alexey.egorov@tu-dortmund.de + Informatik LS8, TU-Dortmund + http://www-ai.cs.uni-dortmund.de + +1 + + + Kai Schenetten + kai.schenetten@tu-dortmund.de + Physik E5b, TU Dortmund + http://app.tu-dortmund.de + +1 + + + Niklas Wulf + niklas.wulf@tu-dortmund.de + Informatik LS8, TU-Dortmund + http://www-ai.cs.uni-dortmund.de + +1 + From d33df68b83bdca50af26d29eff478ac86ad2bfbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20N=C3=B6the?= Date: Thu, 3 May 2018 13:27:10 +0200 Subject: [PATCH 18/24] Update version and changelog for 1.0.2 --- CHANGELOG.md | 8 ++++++++ pom.xml | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b0ac71329..f0d8f82e2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog for the fact-tools +# Version 1.0.2 -- 03.05.2018 + +* Fix exit code being 0 even if an error occured by updating to streams 1.0.5 +* `InterpolatePixelArray` now also interpolates pixels containing `NaN` / `Infinity` values +* Bug fix for `Writer.testKeys`, that resultet in a wrong error message +* Implement skipping rows in the `HDUReader` +* Update all author metadata + # Version 1.0.1 -- 16.03.2018 * Add doi via zenodo diff --git a/pom.xml b/pom.xml index 4703516e8b..7b5a957da4 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ de.sfb876 fact-tools fact-tools - 1.0.2-SNAPSHOT + 1.0.2 http://sfb876.de/fact-tools/ From 17834d9504cd55bbb5d2627bf3d8f4488a779816 Mon Sep 17 00:00:00 2001 From: Jens Buss Date: Wed, 9 May 2018 12:21:29 +0200 Subject: [PATCH 19/24] Revert "add drsfile property as @drsKey to data item, delete drsfile from std calibration XML" This reverts commit 3d3feb79ac270f43555396b2ee21bcc588674951. --- examples/27s.xml | 1 - examples/apply_model.xml | 1 - examples/fellwalker_example.xml | 1 - examples/mc_viewer.xml | 1 - examples/measure_performance.xml | 1 - examples/save_dl1.xml | 1 - examples/stdAnalysis.xml | 1 - examples/studies/extractionTest.xml | 1 - examples/studies/muon_fitting.xml | 1 - examples/studies/muon_identification.xml | 1 - examples/studies/simulations_pass4.xml | 1 - examples/studies/singlePeExtractor/singlePeMinimalExample.xml | 3 +-- .../singlePeExtractor/std_analysis_on_reconstructed_data.xml | 3 +-- examples/viewer.xml | 1 - src/main/resources/analysis/calibration.xml | 1 + 15 files changed, 3 insertions(+), 16 deletions(-) diff --git a/examples/27s.xml b/examples/27s.xml index 8cfc4aba1b..f79e072c07 100644 --- a/examples/27s.xml +++ b/examples/27s.xml @@ -20,7 +20,6 @@ - diff --git a/examples/apply_model.xml b/examples/apply_model.xml index 1b48e25fec..756865626f 100644 --- a/examples/apply_model.xml +++ b/examples/apply_model.xml @@ -27,7 +27,6 @@ - diff --git a/examples/fellwalker_example.xml b/examples/fellwalker_example.xml index 57edbc5840..cd093ffc62 100644 --- a/examples/fellwalker_example.xml +++ b/examples/fellwalker_example.xml @@ -19,7 +19,6 @@ - diff --git a/examples/mc_viewer.xml b/examples/mc_viewer.xml index 7db69a506c..5f1695df5d 100644 --- a/examples/mc_viewer.xml +++ b/examples/mc_viewer.xml @@ -20,7 +20,6 @@ - diff --git a/examples/measure_performance.xml b/examples/measure_performance.xml index 3f71d905af..ae8dc2cfcc 100644 --- a/examples/measure_performance.xml +++ b/examples/measure_performance.xml @@ -20,7 +20,6 @@ - diff --git a/examples/save_dl1.xml b/examples/save_dl1.xml index 16fee1d92c..adebd2e1dc 100644 --- a/examples/save_dl1.xml +++ b/examples/save_dl1.xml @@ -20,7 +20,6 @@ - diff --git a/examples/stdAnalysis.xml b/examples/stdAnalysis.xml index bd88e4450d..ba72995353 100644 --- a/examples/stdAnalysis.xml +++ b/examples/stdAnalysis.xml @@ -22,7 +22,6 @@ - diff --git a/examples/studies/extractionTest.xml b/examples/studies/extractionTest.xml index 2ab26f3f23..c3c9023fc7 100644 --- a/examples/studies/extractionTest.xml +++ b/examples/studies/extractionTest.xml @@ -13,7 +13,6 @@ - diff --git a/examples/studies/muon_fitting.xml b/examples/studies/muon_fitting.xml index 8b32265f4c..cafb41b84c 100644 --- a/examples/studies/muon_fitting.xml +++ b/examples/studies/muon_fitting.xml @@ -19,7 +19,6 @@ - diff --git a/examples/studies/muon_identification.xml b/examples/studies/muon_identification.xml index 91e7e6174b..2032be0b50 100644 --- a/examples/studies/muon_identification.xml +++ b/examples/studies/muon_identification.xml @@ -32,7 +32,6 @@ - diff --git a/examples/studies/simulations_pass4.xml b/examples/studies/simulations_pass4.xml index c321c7120a..03ad25d517 100644 --- a/examples/studies/simulations_pass4.xml +++ b/examples/studies/simulations_pass4.xml @@ -15,7 +15,6 @@ - diff --git a/examples/studies/singlePeExtractor/singlePeMinimalExample.xml b/examples/studies/singlePeExtractor/singlePeMinimalExample.xml index 74d72c48df..e819048903 100644 --- a/examples/studies/singlePeExtractor/singlePeMinimalExample.xml +++ b/examples/studies/singlePeExtractor/singlePeMinimalExample.xml @@ -22,8 +22,7 @@ - - + diff --git a/examples/studies/singlePeExtractor/std_analysis_on_reconstructed_data.xml b/examples/studies/singlePeExtractor/std_analysis_on_reconstructed_data.xml index a6c3dc41a8..d70fb53819 100644 --- a/examples/studies/singlePeExtractor/std_analysis_on_reconstructed_data.xml +++ b/examples/studies/singlePeExtractor/std_analysis_on_reconstructed_data.xml @@ -24,8 +24,7 @@ - - + - diff --git a/src/main/resources/analysis/calibration.xml b/src/main/resources/analysis/calibration.xml index 5087a2eee0..774b2d4706 100644 --- a/src/main/resources/analysis/calibration.xml +++ b/src/main/resources/analysis/calibration.xml @@ -9,6 +9,7 @@ /> From c651a81d2298e65b1742e4314c82406c1ba67c12 Mon Sep 17 00:00:00 2001 From: Jens Buss Date: Wed, 9 May 2018 15:32:41 +0200 Subject: [PATCH 20/24] delete SqliteService sinces it is not used anywhere anymore --- .../java/fact/auxservice/SqliteService.java | 270 ------------------ 1 file changed, 270 deletions(-) delete mode 100644 src/main/java/fact/auxservice/SqliteService.java diff --git a/src/main/java/fact/auxservice/SqliteService.java b/src/main/java/fact/auxservice/SqliteService.java deleted file mode 100644 index d649257ad1..0000000000 --- a/src/main/java/fact/auxservice/SqliteService.java +++ /dev/null @@ -1,270 +0,0 @@ -package fact.auxservice; - -import com.google.common.cache.*; -import fact.auxservice.strategies.AuxPointStrategy; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.tmatesoft.sqljet.core.SqlJetException; -import org.tmatesoft.sqljet.core.SqlJetTransactionMode; -import org.tmatesoft.sqljet.core.table.ISqlJetCursor; -import org.tmatesoft.sqljet.core.table.ISqlJetTable; -import org.tmatesoft.sqljet.core.table.SqlJetDb; -import stream.annotations.Parameter; -import stream.io.SourceURL; - -import java.io.File; -import java.io.IOException; -import java.io.Serializable; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; -import java.util.HashMap; -import java.util.Map; -import java.util.TreeSet; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; - - -/** - * This implements an AuxiliaryService {@link AuxiliaryService} providing data from a sqlite file. - * Can handle TRACKING and SOURCE information. - *

- * Created by kai on 01.03.15. - */ -public class SqliteService implements AuxiliaryService { - - static Logger log = LoggerFactory.getLogger(SqliteService.class); - - @Parameter(required = true, description = "The URL to the sqlite database file. ") - private SourceURL url; - - @Parameter(required = false, description = "Expiry time for entries in the cache.", defaultValue = "10 Minutes") - private int window = 10; - - private LoadingCache> cache = CacheBuilder.newBuilder() - .maximumSize(100) - .expireAfterAccess(window, TimeUnit.MINUTES) - .removalListener(new RemovalListener() { - @Override - public void onRemoval(RemovalNotification notification) { - log.info("Removing Data {} from cache for cause {}", notification.toString(), notification.getCause()); - } - }) - .build(new CacheLoader>() { - @Override - public TreeSet load(AuxDataCacheKey key) throws Exception { - log.info("Building entry for service {} and key: {}", key.service, key.roundedTimeStamp); - return loadDataFromDataBase(key.service, key.roundedTimeStamp); - } - }); - - public static ZonedDateTime floorToQuarterHour(ZonedDateTime time) { - ZonedDateTime t = time.withZoneSameInstant(ZoneOffset.UTC).withSecond(0); - - int oldMinute = t.getMinute(); - int newMinute = 15 * (int) Math.floor(oldMinute / 15.0); - - return t.withMinute(newMinute); - } - - - public class AuxDataCacheKey { - private final AuxiliaryServiceName service; - private final ZonedDateTime roundedTimeStamp; - - public AuxDataCacheKey(AuxiliaryServiceName service, ZonedDateTime timeStamp) { - this.service = service; - this.roundedTimeStamp = floorToQuarterHour(timeStamp); - } - - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - AuxDataCacheKey cacheKey = (AuxDataCacheKey) o; - - if (!roundedTimeStamp.equals(cacheKey.roundedTimeStamp)) return false; - - return service == cacheKey.service; - - } - - @Override - public int hashCode() { - int result = service.hashCode(); - result = 31 * result + roundedTimeStamp.hashCode(); - return result; - } - } - - - /** - * public for unit testing purposes - * - * @param service the name of the aux data to fetch - * @param time the time stamp of the data event - * @return the TreeSet containing the AuxPoints - */ - public TreeSet loadDataFromDataBase(AuxiliaryServiceName service, ZonedDateTime time) { - TreeSet result = new TreeSet<>(); - - try { -// log.info("Querying Database for " + service.toString() + " data for time " + time.toString()); - - SqlJetDb db = SqlJetDb.open(new File(url.getFile()), true); - ISqlJetTable table = db.getTable(service.toString()); - ISqlJetCursor cursor = null; - try { - - db.beginTransaction(SqlJetTransactionMode.READ_ONLY); - try { - if (service == AuxiliaryServiceName.DRIVE_CONTROL_TRACKING_POSITION) { - - - String earlierTime = time.minusHours(4).toString(); - String[] dateAndTime = earlierTime.split("T"); - String[] timeSplitZone = dateAndTime[1].split("Z"); - String earlier = dateAndTime[0] + " " + timeSplitZone[0].substring(0, 8); - - String laterTime = time.plusMinutes(window).toString(); - dateAndTime = laterTime.split("T"); - timeSplitZone = dateAndTime[1].split("Z"); - String later = dateAndTime[0] + " " + timeSplitZone[0].substring(0, 8); - - cursor = table.scope("ix_DRIVE_CONTROL_TRACKING_POSITION_Time", new Object[]{earlier}, new Object[]{later}); - result = getTrackingDataFromCursor(cursor); - } else if (service == AuxiliaryServiceName.DRIVE_CONTROL_SOURCE_POSITION) { - //source position is slower. get data from several hours ago. - String earlierTime = time.minusHours(4).toString(); - String[] dateAndTime = earlierTime.split("T"); - String[] timeSplitZone = dateAndTime[1].split("Z"); - String earlier = dateAndTime[0] + " " + timeSplitZone[0].substring(0, 8); - - String laterTime = time.plusMinutes(window).toString(); - dateAndTime = laterTime.split("T"); - timeSplitZone = dateAndTime[1].split("Z"); - String later = dateAndTime[0] + " " + timeSplitZone[0].substring(0, 8); - - cursor = table.scope("ix_DRIVE_CONTROL_SOURCE_POSITION_Time", new Object[]{earlier}, new Object[]{later}); - result = getSourceDataFromCursor(cursor); - } else { - throw new RuntimeException("This service only provides data from DRIVE_CONTROL_SOURCE_POSITION and TRACKING files"); - } - } finally { - if (cursor != null) { - cursor.close(); - } - } - - } finally { - db.commit(); - db.close(); - } - - } catch (SqlJetException e) { - e.printStackTrace(); - } - return result; - } - - private TreeSet getSourceDataFromCursor(ISqlJetCursor cursor) throws SqlJetException { - TreeSet result = new TreeSet<>(); - if (!cursor.eof()) { - do { - Map m = new HashMap<>(); - m.put("QoS", cursor.getInteger("QoS")); - //getFloat actually returns a double. WTF - m.put("Name", cursor.getString("Name")); - m.put("Angle", cursor.getFloat("Angle")); - m.put("Ra_src", cursor.getFloat("Ra_src")); - m.put("Dec_src", cursor.getFloat("Dec_src")); - // m.put("Ra_cmd", cursor.getFloat("Ra_cmd")); - // m.put("Dec_cmd", cursor.getFloat("Dec_cmd")); - m.put("Offset", cursor.getFloat("Offset")); - - String tempTime = cursor.getString("Time"); - tempTime = tempTime.concat("+00:00"); - tempTime = tempTime.replace(" ", "T"); - ZonedDateTime t = ZonedDateTime.parse(tempTime); - result.add(new AuxPoint(t, m)); - } while (cursor.next()); - } - return result; - } - - private TreeSet getTrackingDataFromCursor(ISqlJetCursor cursor) throws SqlJetException { - TreeSet result = new TreeSet<>(); - if (!cursor.eof()) { - do { - Map m = new HashMap<>(); - m.put("QoS", cursor.getInteger("QoS")); - //getFloat actually returns a double. WTF - m.put("Zd", cursor.getFloat("Zd")); - m.put("Az", cursor.getFloat("Az")); - m.put("Ra", cursor.getFloat("Ra")); - m.put("Dec", cursor.getFloat("Dec")); - m.put("dZd", cursor.getFloat("dZd")); - m.put("dAz", cursor.getFloat("dAz")); - m.put("dev", cursor.getFloat("dev")); - - - String tempTime = cursor.getString("Time"); - tempTime = tempTime.concat("+00:00"); - tempTime = tempTime.replace(" ", "T"); - - - ZonedDateTime t = ZonedDateTime.parse(tempTime); - - result.add(new AuxPoint(t, m)); - } while (cursor.next()); - } - return result; - } - - - /** - * Get auxiliary data from the cache connected to the sqlite database holding drive information. - * - * @param service The service to query. - * @param eventTimeStamp The timestamp of your current event. - * @return the data closest to the eventtimestamp which is found in the database according to the strategy. - */ - @Override - public AuxPoint getAuxiliaryData(AuxiliaryServiceName service, ZonedDateTime eventTimeStamp, AuxPointStrategy strategy) throws IOException { - try { - AuxDataCacheKey key = new AuxDataCacheKey(service, eventTimeStamp); - //this set might not contain the data we need - TreeSet set = cache.get(key); - AuxPoint a = strategy.getPointFromTreeSet(set, eventTimeStamp); - if (a == null) { - throw new IOException("No auxpoint found for the given timestamp " + eventTimeStamp); - } - TimeUnit tu = TimeUnit.SECONDS; - long difference = eventTimeStamp.toEpochSecond() - a.getTimeStamp().toEpochSecond(); - long seconds = tu.toSeconds(difference); - log.debug("Seconds between event and {} auxpoint : {}", service, seconds); - return strategy.getPointFromTreeSet(set, eventTimeStamp); - - } catch (ExecutionException e) { - throw new IOException("Could not load data from Database. Is the database readable?"); -// e.printStackTrace(); - } - } - - - @Override - public void reset() throws Exception { - log.info("Reset called on {}", this.getClass()); - } - - public void setUrl(SourceURL url) { - this.url = url; - } - - public void setWindow(int window) { - this.window = window; - } - - -} From c54d2b7de65128a790856fabf0b9aac625c2cc15 Mon Sep 17 00:00:00 2001 From: Jens Buss Date: Wed, 9 May 2018 15:34:27 +0200 Subject: [PATCH 21/24] bump version number --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d8db7cecf3..8e842e11ec 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ de.sfb876 fact-tools fact-tools - 1.0.1 + 1.0.3 http://sfb876.de/fact-tools/ From 19b68a5e54a8fdccc31259f726ecb926046a39ff Mon Sep 17 00:00:00 2001 From: Jens Buss Date: Wed, 9 May 2018 15:38:46 +0200 Subject: [PATCH 22/24] also delete the test --- src/test/java/fact/services/SqliteTest.java | 97 --------------------- 1 file changed, 97 deletions(-) delete mode 100644 src/test/java/fact/services/SqliteTest.java diff --git a/src/test/java/fact/services/SqliteTest.java b/src/test/java/fact/services/SqliteTest.java deleted file mode 100644 index 0769e4ee7f..0000000000 --- a/src/test/java/fact/services/SqliteTest.java +++ /dev/null @@ -1,97 +0,0 @@ -package fact.services; - -import fact.auxservice.AuxPoint; -import fact.auxservice.AuxiliaryServiceName; -import fact.auxservice.SqliteService; -import fact.auxservice.strategies.Closest; -import org.junit.Test; -import stream.io.SourceURL; - -import java.io.IOException; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; -import java.util.TreeSet; - -import static org.hamcrest.core.Is.is; -import static org.hamcrest.core.IsNot.not; -import static org.hamcrest.core.IsNull.nullValue; -import static org.junit.Assert.assertThat; - -/** - * Test the implementation of the sqliteservice test by querying the test files. - * Created by kai on 06.12.15. - */ -public class SqliteTest { - - @Test - public void testSourcePosition() throws Exception { - SqliteService s = new SqliteService(); - s.setUrl(new SourceURL(SqliteTest.class.getResource("/drive_control_unittest_20140118_19.sqlite"))); - - ZonedDateTime t = ZonedDateTime.parse("2014-01-19T01:40:33+00:00"); - - TreeSet r = s.loadDataFromDataBase(AuxiliaryServiceName.DRIVE_CONTROL_SOURCE_POSITION, t); - - assertThat(r, is(not(nullValue()))); - - ZonedDateTime first = r.first().getTimeStamp(); - ZonedDateTime last = r.last().getTimeStamp(); - - - assertThat(first, is(ZonedDateTime.parse("2014-01-19T01:22:18.045+00:00"))); - assertThat(last, is(ZonedDateTime.parse("2014-01-19T01:42:58.391+00:00"))); - assertThat(r.size(), is(9)); - } - - @Test - public void testSourcePositionInMay() throws Exception { - SqliteService s = new SqliteService(); - s.setUrl(new SourceURL(SqliteTest.class.getResource("/drive_control_5_20.sqlite"))); - ZonedDateTime t = ZonedDateTime.parse("2014-05-20T23:46:14.560Z"); - AuxPoint auxiliaryData = s.getAuxiliaryData(AuxiliaryServiceName.DRIVE_CONTROL_SOURCE_POSITION, t, new Closest()); - assertThat(auxiliaryData, is(not(nullValue()))); - } - - /** - * Should deliver the same result as - * SELECT * FROM DRIVE_CONTROL_TRACKING_POSITION WHERE Time BETWEEN "2014-01-19 01:33:00" AND "2014-01-19 01:44:00" - * on the test DB - * - * @throws Exception - */ - @Test - public void testTrackingPosition() throws Exception { - SqliteService s = new SqliteService(); - s.setUrl(new SourceURL(SqliteTest.class.getResource("/drive_control_unittest_20140118_19.sqlite"))); - - ZonedDateTime t = ZonedDateTime.parse("2014-01-19T01:34:00.04+00:00"); - TreeSet r = s.loadDataFromDataBase(AuxiliaryServiceName.DRIVE_CONTROL_TRACKING_POSITION, t); - - assertThat(r, is(not(nullValue()))); - - ZonedDateTime first = r.first().getTimeStamp(); - ZonedDateTime last = r.last().getTimeStamp(); - - assertThat(first, is(ZonedDateTime.parse("2014-01-19T01:33:17.164+00:00"))); - assertThat(last, is(ZonedDateTime.parse("2014-01-19T01:43:58.684+00:00"))); - assertThat(r.size(), is(480)); - } - - @Test - public void testTimeFlooring() throws IOException { - ZonedDateTime time = ZonedDateTime.of(1987, 9, 20, 12, 40, 34, 0, ZoneOffset.UTC); - ZonedDateTime roundedTime = SqliteService.floorToQuarterHour(time); - assertThat(roundedTime, is(ZonedDateTime.of(1987, 9, 20, 12, 30, 00, 0, ZoneOffset.UTC))); - - time = ZonedDateTime.of(1987, 9, 20, 23, 59, 59, 0, ZoneOffset.UTC); - roundedTime = SqliteService.floorToQuarterHour(time); - assertThat(roundedTime, is(ZonedDateTime.of(1987, 9, 20, 23, 45, 00, 0, ZoneOffset.UTC))); - - - time = ZonedDateTime.of(1987, 9, 20, 00, 00, 01, 0, ZoneOffset.UTC); - roundedTime = SqliteService.floorToQuarterHour(time); - assertThat(roundedTime, is(ZonedDateTime.of(1987, 9, 20, 00, 00, 00, 0, ZoneOffset.UTC))); - } - - -} From 4d138263db5d23384007adf0f63570de755d7b6d Mon Sep 17 00:00:00 2001 From: Jens Buss Date: Wed, 9 May 2018 15:59:54 +0200 Subject: [PATCH 23/24] addapt default aux file path --- examples/forErna/std_analysis_erna_observations.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/forErna/std_analysis_erna_observations.xml b/examples/forErna/std_analysis_erna_observations.xml index 5ea2ebd39b..a0e5857aa1 100644 --- a/examples/forErna/std_analysis_erna_observations.xml +++ b/examples/forErna/std_analysis_erna_observations.xml @@ -5,7 +5,7 @@ --> - + From 1c28b057e154336a6ebdeeb6723c0a9408080d87 Mon Sep 17 00:00:00 2001 From: Jens Buss Date: Wed, 9 May 2018 16:03:31 +0200 Subject: [PATCH 24/24] makr as SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3cc4966284..b5c807bf31 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ de.sfb876 fact-tools fact-tools - 1.0.3 + 1.0.3-SNAPSHOT http://sfb876.de/fact-tools/