Hdu reader skip rows

pom.xml

@@ -4,7 +4,7 @@
   <groupId>de.sfb876</groupId>
   <artifactId>fact-tools</artifactId>
   <name>fact-tools</name>
-  <version>1.0.0</version>
+  <version>1.0.2-SNAPSHOT</version>
   <url>http://sfb876.de/fact-tools/</url>
 
   <description>

src/main/java/fact/io/hdureader/BinTable.java

@@ -35,6 +35,7 @@ public class BinTable {
     final DataInputStream tableDataStream;
     DataInputStream heapDataStream = null;
 
+    public final Header header;
 
     /**
      * This enum maps the type characters in the header to the fits types.
@@ -130,8 +131,10 @@ private void setTypeAndCount(HeaderLine form) throws IOException{
     public Integer numberOfRowsInTable = 0;
     public Integer numberOfColumnsInTable = 0;
     public final String name;
+    public Integer numberOfBytesPerRow = 0; // in header value: naxis1
 
     BinTable(Header header, long hduOffset, URL url) throws IllegalArgumentException, IOException {
+        this.header = header;
 
         binTableSanityCheck(header);
 
@@ -163,6 +166,7 @@ private void setTypeAndCount(HeaderLine form) throws IOException{
             columns.add(new TableColumn(zform, tform, ttype.getValue()));
         }
 
+        numberOfBytesPerRow = header.getInt("ZNAXIS1").orElse(header.getInt("NAXIS1").orElse(0));
         numberOfRowsInTable = header.getInt("ZNAXIS2").orElse(header.getInt("NAXIS2").orElse(0));
         numberOfColumnsInTable = columns.size();
 

src/main/java/fact/io/hdureader/BinTableReader.java

@@ -38,11 +38,13 @@ public boolean hasNext() {
     private final List<BinTable.TableColumn> columns;
     private int numberOfRowsRead = 0;
     private final int numberOfRowsInTable;
+    private final int numberOfBytesPerRow;
 
 
     private BinTableReader(BinTable binTable) {
         this.stream = binTable.tableDataStream;
         this.columns = binTable.columns;
+        this.numberOfBytesPerRow = binTable.numberOfBytesPerRow;
         this.numberOfRowsInTable = binTable.numberOfRowsInTable;
     }
 
@@ -141,4 +143,19 @@ private Serializable readArrayFromStream(BinTable.TableColumn c, DataInputStream
         }
         return null;
     }
+
+    /**
+     * Skips the given number of rows.
+     *
+     * @param num The amount of rows to skip.
+     * @throws IOException
+     */
+    @Override
+    public void skipRows(int amount) throws IOException {
+        if (amount+numberOfRowsRead <= numberOfRowsInTable) {
+            new IndexOutOfBoundsException("Table has not enough rows to access row num: " + (amount+numberOfRowsRead));
+        }
+        stream.skipBytes(amount * this.numberOfBytesPerRow);
+        numberOfRowsRead += amount;
+    }
 }

src/main/java/fact/io/hdureader/FITSStream.java

@@ -127,4 +127,14 @@ public Data readNext() throws Exception {
 
         return item;
     }
+
+    /**
+     * Skips the given amount of rows in the data table.
+     *
+     * @param amount The amount of Rows to skip.
+     * @throws IOException
+     */
+    public void skipRows(int amount) throws IOException{
+        reader.skipRows(amount);
+    }
 }

src/main/java/fact/io/hdureader/Reader.java

@@ -72,5 +72,12 @@ default Iterator<OptionalTypesMap<String, Serializable>> iterator() {
      */
     OptionalTypesMap<String, Serializable> getNextRow() throws IOException;
 
+    /**
+     * Skips the given number of rows.
+     *
+     * @param num The amount of rows to skip.
+     * @throws IOException
+     */
+    void skipRows(int amount) throws IOException;
 
 }

src/main/java/fact/io/hdureader/ZFITSHeapReader.java

@@ -65,6 +65,10 @@
 public final class ZFITSHeapReader implements Reader {
 
     private final DataInputStream stream;
+    private final DataInputStream catalogStream;
+    private int catalogPosition = 0;
+    private final int zshrink;
+    private final int zTileLen;
     private final List<BinTable.TableColumn> columns;
     private final Integer numberOfRowsInTable;
     private int numberOfRowsRead = 0;
@@ -92,6 +96,9 @@ private ZFITSHeapReader(BinTable binTable) {
         this.numberOfRowsInTable = binTable.numberOfRowsInTable;
         this.stream = binTable.heapDataStream;
         this.columns = binTable.columns;
+        this.catalogStream = binTable.tableDataStream;
+        this.zshrink = binTable.header.getInt("ZSHRINK").orElse(1);
+        this.zTileLen = binTable.header.getInt("ZTILELEN").orElse(1);
     }
 
 
@@ -106,6 +113,96 @@ public static ZFITSHeapReader forTable(BinTable binTable) {
         return new ZFITSHeapReader(binTable);
     }
 
+    /**
+     * Skips the given number of rows.
+     *
+     * @param amount The amount of rows to skip.
+     * @throws IOException
+     */
+    @Override
+    public void skipRows(int amount) throws IOException {
+        int resultingRow = amount+numberOfRowsRead;
+        if (resultingRow >= this.numberOfRowsInTable) {
+            throw new IOException("Not enough rows in table, need "+(amount+numberOfRowsRead)+" have "+numberOfRowsInTable);
+        }
+
+        // check if resulting row is in the current catalog remainer
+        int remainer = zTileLen - numberOfRowsRead%zTileLen;
+        if (amount<remainer) { // no need to deal with the catalog as we just work within one
+            for (int i = 0; i < amount; i++) {
+                getNextRow();
+            }
+            return;
+        }
+        if (remainer!=zTileLen) {
+            // go the the start of the next catalog and adjust the amount to skip left
+            for (int i = 0; i < remainer; i++) {
+                getNextRow();
+            }
+            amount -= remainer;
+        }
+
+
+        int rowCatalogPosition = numberOfRowsRead / zTileLen;
+        int rowCatalogPositionFinished = resultingRow / zTileLen;
+        // check if we can't skip using the catalog due to being in the same catalog position
+        if (rowCatalogPosition == rowCatalogPositionFinished) {
+            for (int i = 0; i < amount; i++) {
+                getNextRow();
+            }
+            return;
+        }
+
+        // check if the current catalog position is further then the rowCatalogPosition
+        if (catalogPosition > rowCatalogPosition) {
+            // align them the difference if always maximal 1
+            for (int i = 0; i < zTileLen; i++) {
+                getNextRow();
+            }
+            rowCatalogPosition += 1;
+        }
+        // check if we can't skip using the catalog due to being in the same catalog position
+        if (rowCatalogPosition == rowCatalogPositionFinished) {
+            for (int i = 0; i < amount; i++) {
+                getNextRow();
+            }
+            return;
+        }
+
+        // move the current catalog position to the rowCatalogPosition
+        int skipBytes = (rowCatalogPosition - catalogPosition) * columns.size() * (16);
+        this.catalogStream.skipBytes(skipBytes);
+        this.catalogPosition = rowCatalogPosition;
+
+        // get current row position
+        this.catalogStream.skipBytes(8); // go directly to the offset
+        long rowOffset = this.catalogStream.readLong() - 16; // read the offset - 16 for the header
+        this.catalogStream.skipBytes(columns.size() * (16) - 8 - 8); // go to the next catalog start
+        this.catalogPosition += 1;
+
+        // go to the finishing position catalog
+        int diffCatalogs = rowCatalogPositionFinished-catalogPosition;
+        skipBytes = diffCatalogs * columns.size() * (16) + 8; // go directly to the offset
+        this.catalogStream.skipBytes(skipBytes);
+        long finalRowOffset = this.catalogStream.readLong() - 16; // read the offset - 16 for the header
+
+        // readjust the catalogStream
+        this.catalogStream.skipBytes(columns.size() * (16) - 8 - 8); // go to the next catalog start
+        this.catalogPosition += diffCatalogs+1;
+
+        // skip the bytes in the data stream
+        long skipManyBytes = finalRowOffset - rowOffset;
+        while(skipManyBytes!=0) {
+            long skipped = this.stream.skip(skipManyBytes);
+            skipManyBytes -= skipped;
+        }
+        this.numberOfRowsRead += diffCatalogs * this.zTileLen;
+
+        int remainingRows = resultingRow % zTileLen;
+        for (int i=0; i<remainingRows; i++) {
+            getNextRow();
+        }
+    }
 
     /**
      * Get the data from the next row. The columns in the row can be accessed by their name in the resulting

src/test/java/fact/io/hdureader/FitsHDUTests.java

@@ -2,6 +2,9 @@
 
 import org.junit.Test;
 
+import stream.io.SourceURL;
+import stream.Data;
+
 import java.io.DataInputStream;
 import java.io.IOException;
 import java.net.URL;
@@ -31,6 +34,116 @@ public void testToOpenInputStream() throws Exception {
 
     }
 
+    @Test
+    public void testBinTableReaderSkip() throws Exception {
+        URL u = FitsHDUTests.class.getResource("/testDataFile.fits.gz");
+        FITSStream fits = new FITSStream(new SourceURL(u));
+        fits.init();
+        FITSStream fits2 = new FITSStream(new SourceURL(u));
+        fits2.init();
+
+        //read 3
+        fits.readNext();
+        fits.readNext();
+        Data item = fits.readNext();
+        fits2.skipRows(2);
+        Data item2 = fits2.readNext();
+
+
+        int eNr = (int) item.get("EventNum");
+        int eNr2 = (int) item2.get("EventNum");
+        assertEquals(eNr, eNr2);
+
+        short[] data = (short[]) item.get("Data");
+        short[] data2 = (short[]) item2.get("Data");
+        assertArrayEquals(data, data2);
+    }
+
+    @Test
+    public void testZFitsReaderSkip() throws Exception {
+        for (int i = 2; i < 5; i++) {
+            System.out.println("Test skip: " + i);
+            URL u = FitsHDUTests.class.getResource("/testDataFile.fits.fz");
+            FITSStream fits = new FITSStream(new SourceURL(u));
+            fits.init();
+            FITSStream fits2 = new FITSStream(new SourceURL(u));
+            fits2.init();
+
+            for (int j = 0; j < i; j++) {
+                fits.readNext();
+            }
+            Data item = fits.readNext();
+
+            fits2.skipRows(i);
+            Data item2 = fits2.readNext();
+
+            int eNr = (int) item.get("EventNum");
+            int eNr2 = (int) item2.get("EventNum");
+            assertEquals(eNr, eNr2);
+
+            short[] data = (short[]) item.get("Data");
+            short[] data2 = (short[]) item2.get("Data");
+            assertArrayEquals(data, data2);
+        }
+    }
+
+    @Test
+    public void testZFitsReaderSkipWithZShrink() throws Exception {
+        for (int i = 0; i < 5; i++) {
+            System.out.println("Test skip: " + i);
+            URL u = FitsHDUTests.class.getResource("/testDataFileZSHRINK.fits.fz");
+            FITSStream fits = new FITSStream(new SourceURL(u));
+            fits.init();
+            FITSStream fits2 = new FITSStream(new SourceURL(u));
+            fits2.init();
+
+            for (int j = 0; j < i; j++) {
+                fits.readNext();
+            }
+            Data item = fits.readNext();
+
+            fits2.skipRows(i);
+            Data item2 = fits2.readNext();
+
+
+            int eNr = (int) item.get("EventNum");
+            int eNr2 = (int) item2.get("EventNum");
+            assertEquals(eNr, eNr2);
+
+            short[] data = (short[]) item.get("Data");
+            short[] data2 = (short[]) item2.get("Data");
+            assertArrayEquals(data, data2);
+        }
+    }
+
+    @Test
+    public void testZFitsReaderSkipWithZTileLen() throws Exception {
+        for (int i = 98; i < 102; i++) {
+            System.out.println("Test skip: " + i);
+            URL u = FitsHDUTests.class.getResource("/testDataFileZTILELEN.fits.fz");
+            FITSStream fits = new FITSStream(new SourceURL(u));
+            fits.init();
+            FITSStream fits2 = new FITSStream(new SourceURL(u));
+            fits2.init();
+
+            for (int j = 0; j < i; j++) {
+                fits.readNext();
+            }
+            Data item = fits.readNext();
+
+            fits2.skipRows(i);
+            Data item2 = fits2.readNext();
+
+
+            int eNr = (int) item.get("EventNum");
+            int eNr2 = (int) item2.get("EventNum");
+            assertEquals(eNr, eNr2);
+
+            short[] data = (short[]) item.get("Data");
+            short[] data2 = (short[]) item2.get("Data");
+            assertArrayEquals(data, data2);
+        }
+    }
 
     @SuppressWarnings("ResultOfMethodCallIgnored")
     @Test