diff --git a/pom.xml b/pom.xml index 9fa3b79d03..c6bb21078a 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ de.sfb876 fact-tools fact-tools - 1.0.0 + 1.0.2-SNAPSHOT http://sfb876.de/fact-tools/ diff --git a/src/main/java/fact/io/hdureader/BinTable.java b/src/main/java/fact/io/hdureader/BinTable.java index cad5183d4d..313d4e747b 100644 --- a/src/main/java/fact/io/hdureader/BinTable.java +++ b/src/main/java/fact/io/hdureader/BinTable.java @@ -35,6 +35,7 @@ public class BinTable { final DataInputStream tableDataStream; DataInputStream heapDataStream = null; + public final Header header; /** * This enum maps the type characters in the header to the fits types. @@ -130,8 +131,10 @@ private void setTypeAndCount(HeaderLine form) throws IOException{ public Integer numberOfRowsInTable = 0; public Integer numberOfColumnsInTable = 0; public final String name; + public Integer numberOfBytesPerRow = 0; // in header value: naxis1 BinTable(Header header, long hduOffset, URL url) throws IllegalArgumentException, IOException { + this.header = header; binTableSanityCheck(header); @@ -163,6 +166,7 @@ private void setTypeAndCount(HeaderLine form) throws IOException{ columns.add(new TableColumn(zform, tform, ttype.getValue())); } + numberOfBytesPerRow = header.getInt("ZNAXIS1").orElse(header.getInt("NAXIS1").orElse(0)); numberOfRowsInTable = header.getInt("ZNAXIS2").orElse(header.getInt("NAXIS2").orElse(0)); numberOfColumnsInTable = columns.size(); diff --git a/src/main/java/fact/io/hdureader/BinTableReader.java b/src/main/java/fact/io/hdureader/BinTableReader.java index 2ac666bfb7..c2402dc2ca 100644 --- a/src/main/java/fact/io/hdureader/BinTableReader.java +++ b/src/main/java/fact/io/hdureader/BinTableReader.java @@ -38,11 +38,13 @@ public boolean hasNext() { private final List columns; private int numberOfRowsRead = 0; private final int numberOfRowsInTable; + private final int numberOfBytesPerRow; private BinTableReader(BinTable binTable) { this.stream = binTable.tableDataStream; this.columns = binTable.columns; + this.numberOfBytesPerRow = binTable.numberOfBytesPerRow; this.numberOfRowsInTable = binTable.numberOfRowsInTable; } @@ -141,4 +143,19 @@ private Serializable readArrayFromStream(BinTable.TableColumn c, DataInputStream } return null; } + + /** + * Skips the given number of rows. + * + * @param num The amount of rows to skip. + * @throws IOException + */ + @Override + public void skipRows(int amount) throws IOException { + if (amount+numberOfRowsRead <= numberOfRowsInTable) { + new IndexOutOfBoundsException("Table has not enough rows to access row num: " + (amount+numberOfRowsRead)); + } + stream.skipBytes(amount * this.numberOfBytesPerRow); + numberOfRowsRead += amount; + } } diff --git a/src/main/java/fact/io/hdureader/FITSStream.java b/src/main/java/fact/io/hdureader/FITSStream.java index 26e1e13e2e..56aac8138c 100644 --- a/src/main/java/fact/io/hdureader/FITSStream.java +++ b/src/main/java/fact/io/hdureader/FITSStream.java @@ -127,4 +127,14 @@ public Data readNext() throws Exception { return item; } + + /** + * Skips the given amount of rows in the data table. + * + * @param amount The amount of Rows to skip. + * @throws IOException + */ + public void skipRows(int amount) throws IOException{ + reader.skipRows(amount); + } } diff --git a/src/main/java/fact/io/hdureader/Reader.java b/src/main/java/fact/io/hdureader/Reader.java index 7c2a1f20b5..255785347c 100644 --- a/src/main/java/fact/io/hdureader/Reader.java +++ b/src/main/java/fact/io/hdureader/Reader.java @@ -72,5 +72,12 @@ default Iterator> iterator() { */ OptionalTypesMap getNextRow() throws IOException; + /** + * Skips the given number of rows. + * + * @param num The amount of rows to skip. + * @throws IOException + */ + void skipRows(int amount) throws IOException; } diff --git a/src/main/java/fact/io/hdureader/ZFITSHeapReader.java b/src/main/java/fact/io/hdureader/ZFITSHeapReader.java index 07dba0ceda..597af7c84d 100644 --- a/src/main/java/fact/io/hdureader/ZFITSHeapReader.java +++ b/src/main/java/fact/io/hdureader/ZFITSHeapReader.java @@ -65,6 +65,10 @@ public final class ZFITSHeapReader implements Reader { private final DataInputStream stream; + private final DataInputStream catalogStream; + private int catalogPosition = 0; + private final int zshrink; + private final int zTileLen; private final List columns; private final Integer numberOfRowsInTable; private int numberOfRowsRead = 0; @@ -92,6 +96,9 @@ private ZFITSHeapReader(BinTable binTable) { this.numberOfRowsInTable = binTable.numberOfRowsInTable; this.stream = binTable.heapDataStream; this.columns = binTable.columns; + this.catalogStream = binTable.tableDataStream; + this.zshrink = binTable.header.getInt("ZSHRINK").orElse(1); + this.zTileLen = binTable.header.getInt("ZTILELEN").orElse(1); } @@ -106,6 +113,96 @@ public static ZFITSHeapReader forTable(BinTable binTable) { return new ZFITSHeapReader(binTable); } + /** + * Skips the given number of rows. + * + * @param amount The amount of rows to skip. + * @throws IOException + */ + @Override + public void skipRows(int amount) throws IOException { + int resultingRow = amount+numberOfRowsRead; + if (resultingRow >= this.numberOfRowsInTable) { + throw new IOException("Not enough rows in table, need "+(amount+numberOfRowsRead)+" have "+numberOfRowsInTable); + } + + // check if resulting row is in the current catalog remainer + int remainer = zTileLen - numberOfRowsRead%zTileLen; + if (amount rowCatalogPosition) { + // align them the difference if always maximal 1 + for (int i = 0; i < zTileLen; i++) { + getNextRow(); + } + rowCatalogPosition += 1; + } + // check if we can't skip using the catalog due to being in the same catalog position + if (rowCatalogPosition == rowCatalogPositionFinished) { + for (int i = 0; i < amount; i++) { + getNextRow(); + } + return; + } + + // move the current catalog position to the rowCatalogPosition + int skipBytes = (rowCatalogPosition - catalogPosition) * columns.size() * (16); + this.catalogStream.skipBytes(skipBytes); + this.catalogPosition = rowCatalogPosition; + + // get current row position + this.catalogStream.skipBytes(8); // go directly to the offset + long rowOffset = this.catalogStream.readLong() - 16; // read the offset - 16 for the header + this.catalogStream.skipBytes(columns.size() * (16) - 8 - 8); // go to the next catalog start + this.catalogPosition += 1; + + // go to the finishing position catalog + int diffCatalogs = rowCatalogPositionFinished-catalogPosition; + skipBytes = diffCatalogs * columns.size() * (16) + 8; // go directly to the offset + this.catalogStream.skipBytes(skipBytes); + long finalRowOffset = this.catalogStream.readLong() - 16; // read the offset - 16 for the header + + // readjust the catalogStream + this.catalogStream.skipBytes(columns.size() * (16) - 8 - 8); // go to the next catalog start + this.catalogPosition += diffCatalogs+1; + + // skip the bytes in the data stream + long skipManyBytes = finalRowOffset - rowOffset; + while(skipManyBytes!=0) { + long skipped = this.stream.skip(skipManyBytes); + skipManyBytes -= skipped; + } + this.numberOfRowsRead += diffCatalogs * this.zTileLen; + + int remainingRows = resultingRow % zTileLen; + for (int i=0; i