Merge pull request #390 from fact-project/fix_zfits_boardtime

Fix zfits boardtime

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog for the fact-tools
 
+# Version 1.1.2 -- 17.12.2018
+
+* Fix reading of uncompressed values from zfits heap tiles (Issue #389)
+  Before, the same value was repeated for every row in one tile.
+  This affects e.g. `UnixTimeUTC` and `BoardTime` and booleans.
+
 # Version 1.1.1 -- 14.11.2018
 
 * Make Writer keys case sensitive by default (#384)

pom.xml

@@ -4,7 +4,7 @@
   <groupId>de.sfb876</groupId>
   <artifactId>fact-tools</artifactId>
   <name>fact-tools</name>
-  <version>1.1.1</version>
+  <version>1.1.2</version>
   <url>http://sfb876.de/fact-tools/</url>
 
   <description>

src/main/java/fact/io/hdureader/ZFITSHeapReader.java

@@ -125,7 +125,7 @@ public void skipRows(int amount) throws IOException {
         if (resultingRow >= this.numberOfRowsInTable) {
             throw new IOException("Not enough rows in table, need "+(amount+numberOfRowsRead)+" have "+numberOfRowsInTable);
         }
-        
+
         // check if resulting row is in the current catalog remainer
         int remainer = zTileLen - numberOfRowsRead%zTileLen;
         if (amount<remainer) { // no need to deal with the catalog as we just work within one
@@ -257,11 +257,10 @@ public OptionalTypesMap<String, Serializable> getNextRow(boolean ignoreWrongTile
                 shortBuffer.get(data);
                 map.put(column.name, data);
             } else {
-                ByteBuffer byteBuffer = (ByteBuffer) curBuffer;
                 if (column.repeatCount == 1) {
-                    map.put(column.name, readSingleValueFromBuffer(column, byteBuffer));
+                    map.put(column.name, readSingleValueFromBuffer(column, (ByteBuffer) curBuffer));
                 } else {
-                    map.put(column.name, readArrayFromBuffer(column, byteBuffer));
+                    map.put(column.name, readArrayFromBuffer(column, curBuffer));
                 }
             }
         }
@@ -324,10 +323,40 @@ private void getNextTileIntoCache(boolean ignoreWrongTileHeader) throws IOExcept
                 // convert to int, no block is bigger than 4GB
                 int numberBytes = Math.toIntExact(block.getDataSize());
                 byte[] cache = new byte[numberBytes];
-                for (int i = 0; i < numberBytes; i++) {
-                    cache[i] = tileBuffer.get();
+                tileBuffer.get(cache);
+
+                ByteBuffer byteBuffer = ByteBuffer.wrap(cache).order(ByteOrder.LITTLE_ENDIAN);
+
+                if (column.repeatCount > 1) {
+                    switch (column.type) {
+                        case BYTE:
+                            tileCache.put(column.name, byteBuffer);
+                            break;
+                        case BOOLEAN:
+                            tileCache.put(column.name, byteBuffer);
+                            break;
+                        case CHAR:
+                            tileCache.put(column.name, byteBuffer.asCharBuffer());
+                            break;
+                        case SHORT:
+                            tileCache.put(column.name, byteBuffer.asShortBuffer());
+                            break;
+                        case INT:
+                            tileCache.put(column.name, byteBuffer.asIntBuffer());
+                            break;
+                        case LONG:
+                            tileCache.put(column.name, byteBuffer.asLongBuffer());
+                            break;
+                        case FLOAT:
+                            tileCache.put(column.name, byteBuffer.asFloatBuffer());
+                            break;
+                        case DOUBLE:
+                            tileCache.put(column.name, byteBuffer.asDoubleBuffer());
+                            break;
+                    }
+                } else {
+                    tileCache.put(column.name, byteBuffer);
                 }
-                tileCache.put(column.name, ByteBuffer.wrap(cache).order(ByteOrder.LITTLE_ENDIAN));
             } else {
                 throw new NotImplementedException("Compression: '" + block.compression.name() + "' is not implemented");
             }
@@ -646,10 +675,10 @@ private Serializable readSingleValueFromBuffer(BinTable.TableColumn c, ByteBuffe
         Serializable b = null;
         switch (c.type) {
             case BOOLEAN:
-                b = buffer.get() > 0;
+                b = buffer.get() == 'T';
                 break;
             case CHAR:
-                b = buffer.asCharBuffer().toString();
+                b = buffer.getChar();
                 break;
             case BYTE:
                 b = buffer.get();
@@ -673,44 +702,42 @@ private Serializable readSingleValueFromBuffer(BinTable.TableColumn c, ByteBuffe
         return b;
     }
 
-    private Serializable readArrayFromBuffer(BinTable.TableColumn c, ByteBuffer buffer) throws IOException {
-
-        if (c.type == BinTable.ColumnType.BOOLEAN) {
-            boolean[] bools = new boolean[c.repeatCount];
-            for (int i = 0; i < c.repeatCount; i++) {
-                bools[i] = buffer.get() > 0;
-            }
-            return bools;
-        }
-
+    private Serializable readArrayFromBuffer(BinTable.TableColumn c, Buffer buffer) throws IOException {
         switch (c.type) {
-            case CHAR:
-                char[] chars = new char[c.repeatCount];
-                buffer.asCharBuffer().get(chars);
-                return chars;
             case BYTE:
                 byte[] b = new byte[c.repeatCount];
-                buffer.get(b);
+                ((ByteBuffer) buffer).get(b);
                 return b;
+            case BOOLEAN:
+                boolean[] bools = new boolean[c.repeatCount];
+                ByteBuffer byteBuffer = (ByteBuffer) buffer;
+                for (int i = 0; i < c.repeatCount; i++) {
+                    bools[i] = byteBuffer.get() == 'T';
+                }
+                return bools;
+            case CHAR:
+                char[] chars = new char[c.repeatCount];
+                ((CharBuffer) buffer).get(chars);
+                return chars;
             case SHORT:
                 short[] shorts = new short[c.repeatCount];
-                buffer.asShortBuffer().get(shorts);
+                ((ShortBuffer) buffer).get(shorts);
                 return shorts;
             case INT:
                 int[] ints = new int[c.repeatCount];
-                buffer.asIntBuffer().get(ints);
+                ((IntBuffer) buffer).get(ints);
                 return ints;
             case LONG:
                 long[] longs = new long[c.repeatCount];
-                buffer.asLongBuffer().get(longs);
+                ((LongBuffer) buffer).get(longs);
                 return longs;
             case FLOAT:
                 float[] floats = new float[c.repeatCount];
-                buffer.asFloatBuffer().get(floats);
+                ((FloatBuffer) buffer).get(floats);
                 return floats;
             case DOUBLE:
                 double[] doubles = new double[c.repeatCount];
-                buffer.asDoubleBuffer().get(doubles);
+                ((DoubleBuffer) buffer).get(doubles);
                 return doubles;
         }
         return null;

src/test/java/fact/io/hdureader/ZFitsReaderTests.java

@@ -50,6 +50,29 @@ public void testZSHRINK() throws Exception {
         assertTrue(num == 6);
     }
 
+
+    @Test
+    public void testUnixTimeUTCIncreases() throws Exception {
+        URL u = ZFitsReaderTests.class.getResource("/testDataFileZTILELEN.fits.fz");
+
+        FITS f = new FITS(u);
+        HDU events = f.getHDU("Events").orElseThrow(() -> new RuntimeException("File did not contain HDU 'Events'"));
+        BinTable binTable = events.getBinTable();
+        ZFITSHeapReader heapReader = ZFITSHeapReader.forTable(binTable);
+
+        long lastTimestamp = -1;
+        for (OptionalTypesMap<String, Serializable> p : heapReader) {
+            assertTrue(p.containsKey("Data"));
+
+            int[] unixTimeUTC = p.getIntArray("UnixTimeUTC").orElseThrow(RuntimeException::new);
+            long timestamp = ((long) unixTimeUTC[0]) * 1000000 + unixTimeUTC[1];
+
+            assertTrue(timestamp > lastTimestamp);
+            lastTimestamp = timestamp;
+        }
+    }
+
+
     @Test
     public void testZTileLen() throws Exception {
         //URL u =  CompareOldAndNewReaders.class.getResource("/testDataFile.fits.fz");