From 68626981be6fa37e111de6cd474ce43550203a96 Mon Sep 17 00:00:00 2001 From: Roman Chernyatchik Date: Fri, 27 Apr 2018 23:47:01 +0200 Subject: [PATCH] Introduced consts for prefetch levels --- CHANGES | 8 +++++--- src/main/kotlin/org/jetbrains/bio/big/BigBed.kt | 2 +- src/main/kotlin/org/jetbrains/bio/big/BigFile.kt | 9 ++++++--- src/main/kotlin/org/jetbrains/bio/big/BigWig.kt | 2 +- src/test/kotlin/org/jetbrains/bio/TestSupport.kt | 6 +++--- .../kotlin/org/jetbrains/bio/big/BigBedFileTest.kt | 10 +++++----- .../kotlin/org/jetbrains/bio/big/BigWigFileTest.kt | 10 +++++----- .../kotlin/org/jetbrains/bio/big/RTreeIndexTest.kt | 6 +++--- 8 files changed, 29 insertions(+), 24 deletions(-) diff --git a/CHANGES b/CHANGES index 18d8d2d..70741ef 100644 --- a/CHANGES +++ b/CHANGES @@ -27,9 +27,11 @@ Changed - Use `RomBufferFactory` to detect file endianness - Changed `prefetch` option to int value in `BigFile.read()`, `BigWigFile.read()`, `BigBedFile.read()`. Supported values are: - * `0` : do not prefetch - * `1` : prefetch zoom level indexes tree, chromosomes list - * `2` : prefetch w/o zoom level data tree indexes up to chromosomes level + * `BigFile.PREFETCH_LEVEL_OFF` : do not prefetch + * `BigFile.PREFETCH_LEVEL_FAST` : prefetch zoom level indexes tree, + chromosomes list + * `BigFile.PREFETCH_LEVEL_DETAILED` : prefetch w/o zoom level data tree + indexes up to chromosomes level Larger prefetch values increase big file opening time and memory consumption but provides faster data access with fewer i/o operations diff --git a/src/main/kotlin/org/jetbrains/bio/big/BigBed.kt b/src/main/kotlin/org/jetbrains/bio/big/BigBed.kt index 14aa2a6..08104f1 100644 --- a/src/main/kotlin/org/jetbrains/bio/big/BigBed.kt +++ b/src/main/kotlin/org/jetbrains/bio/big/BigBed.kt @@ -94,7 +94,7 @@ class BigBedFile private constructor( @Throws(IOException::class) @JvmStatic - fun read(src: String, prefetch: Int = 2, + fun read(src: String, prefetch: Int = BigFile.PREFETCH_LEVEL_DETAILED, cancelledChecker: (() -> Unit)? = null, factoryProvider: RomBufferFactoryProvider = defaultFactory() ): BigBedFile { diff --git a/src/main/kotlin/org/jetbrains/bio/big/BigFile.kt b/src/main/kotlin/org/jetbrains/bio/big/BigFile.kt index dfea210..0b2873a 100644 --- a/src/main/kotlin/org/jetbrains/bio/big/BigFile.kt +++ b/src/main/kotlin/org/jetbrains/bio/big/BigFile.kt @@ -84,7 +84,7 @@ abstract class BigFile internal constructor( bPlusTree = BPlusTree.read(input, header.chromTreeOffset) // if do prefetch input and file not empty: - if (prefetch > 0) { + if (prefetch >= BigFile.PREFETCH_LEVEL_FAST) { // stored in the beginning of file near header cancelledChecker?.invoke() prefetechedTotalSummary = BigSummary.read(input, header.totalSummaryOffset) @@ -132,7 +132,7 @@ abstract class BigFile internal constructor( // in buffered stream cancelledChecker?.invoke() rTree = RTreeIndex.read(input, header.unzoomedIndexOffset) - if (prefetch > 1) { + if (prefetch >= BigFile.PREFETCH_LEVEL_DETAILED) { rTree.prefetchBlocksIndex( input, false, true, header.uncompressBufSize, cancelledChecker ) @@ -467,6 +467,9 @@ abstract class BigFile internal constructor( companion object { private val LOG = LogManager.getLogger(BigFile::class.java) + const val PREFETCH_LEVEL_OFF = 0 + const val PREFETCH_LEVEL_FAST = 1 + const val PREFETCH_LEVEL_DETAILED = 2 private val lastCachedBlockInfo: ThreadLocal> = ThreadLocal.withInitial { RomBufferState(null, 0L, 0L, "") to null @@ -522,7 +525,7 @@ abstract class BigFile internal constructor( @Throws(IOException::class) @JvmStatic - fun read(src: String, prefetch: Int = 2, + fun read(src: String, prefetch: Int = BigFile.PREFETCH_LEVEL_DETAILED, cancelledChecker: (() -> Unit)? = null, factoryProvider: RomBufferFactoryProvider = defaultFactory() ): BigFile> = factoryProvider(src, ByteOrder.LITTLE_ENDIAN).use { factory -> diff --git a/src/main/kotlin/org/jetbrains/bio/big/BigWig.kt b/src/main/kotlin/org/jetbrains/bio/big/BigWig.kt index e347a1d..4395bdf 100644 --- a/src/main/kotlin/org/jetbrains/bio/big/BigWig.kt +++ b/src/main/kotlin/org/jetbrains/bio/big/BigWig.kt @@ -170,7 +170,7 @@ class BigWigFile private constructor( @Throws(IOException::class) @JvmStatic - fun read(src: String, prefetch: Int = 2, + fun read(src: String, prefetch: Int = BigFile.PREFETCH_LEVEL_DETAILED, cancelledChecker: (() -> Unit)? = null, factoryProvider: RomBufferFactoryProvider = defaultFactory() ): BigWigFile { diff --git a/src/test/kotlin/org/jetbrains/bio/TestSupport.kt b/src/test/kotlin/org/jetbrains/bio/TestSupport.kt index 9b4fd9a..c9a95f6 100644 --- a/src/test/kotlin/org/jetbrains/bio/TestSupport.kt +++ b/src/test/kotlin/org/jetbrains/bio/TestSupport.kt @@ -89,7 +89,7 @@ fun threadSafeRomFactoryProvidersAndPrefetchParams(): List> { }) } return providers.flatMap { fp -> - intArrayOf(0, 1, 2).map { prefetch -> + intArrayOf(BigFile.PREFETCH_LEVEL_OFF, BigFile.PREFETCH_LEVEL_FAST, BigFile.PREFETCH_LEVEL_DETAILED).map { prefetch -> arrayOf(fp, prefetch) } } @@ -98,7 +98,7 @@ fun threadSafeRomFactoryProvidersAndPrefetchParams(): List> { fun romFactoryProviderParams(): List> = romFactoryProviders().map { arrayOf(it) } fun romFactoryProviderAndPrefetchParams(): List> = romFactoryProviders().flatMap { fp -> - intArrayOf(0, 1, 2).map { prefetch -> + intArrayOf(BigFile.PREFETCH_LEVEL_OFF, BigFile.PREFETCH_LEVEL_FAST, BigFile.PREFETCH_LEVEL_DETAILED).map { prefetch -> arrayOf(fp, prefetch) } } @@ -132,7 +132,7 @@ fun romFactoryByteOrderCompressionParamsSets(): Iterable> { } fun allBigFileParams(): List> = romFactoryByteOrderCompressionParamsSets().flatMap { - intArrayOf(0, 1, 2).map { prefetch -> + intArrayOf(BigFile.PREFETCH_LEVEL_OFF, BigFile.PREFETCH_LEVEL_FAST, BigFile.PREFETCH_LEVEL_DETAILED).map { prefetch -> it + prefetch } } diff --git a/src/test/kotlin/org/jetbrains/bio/big/BigBedFileTest.kt b/src/test/kotlin/org/jetbrains/bio/big/BigBedFileTest.kt index f0b1582..9a196c0 100644 --- a/src/test/kotlin/org/jetbrains/bio/big/BigBedFileTest.kt +++ b/src/test/kotlin/org/jetbrains/bio/big/BigBedFileTest.kt @@ -26,8 +26,8 @@ class BigBedFileTest( BigBedFile.read(Examples["example1.bb"], bfProvider, prefetch).use { bf -> assertEquals(5, bf.zoomLevels.size) assertEquals(39110000, bf.zoomLevels[4].reduction) - assertEquals(prefetch > 0, bf.prefetchedLevel2RTreeIndex != null) - if (prefetch > 0) { + assertEquals(prefetch == BigFile.PREFETCH_LEVEL_OFF, bf.prefetchedLevel2RTreeIndex == null) + if (prefetch >= BigFile.PREFETCH_LEVEL_FAST) { assertEquals(5, bf.prefetchedLevel2RTreeIndex!!.size) val zRTree0 = bf.prefetchedLevel2RTreeIndex!![bf.zoomLevels[0]]!! @@ -41,8 +41,8 @@ class BigBedFileTest( (zRTree4.rootNode!! as RTReeNodeLeaf).leaves[0].interval.toString()) } - assertEquals(prefetch > 0, bf.prefetchedChr2Leaf != null) - if (prefetch > 0) { + assertEquals(prefetch == BigFile.PREFETCH_LEVEL_OFF, bf.prefetchedChr2Leaf == null) + if (prefetch >= BigFile.PREFETCH_LEVEL_FAST) { assertEquals(1, bf.prefetchedChr2Leaf!!.size) assertEquals("chr21", bf.prefetchedChr2Leaf!!["chr21"]!!.key) assertEquals(0, bf.prefetchedChr2Leaf!!["chr21"]!!.id) @@ -58,7 +58,7 @@ class BigBedFileTest( assertEquals(192819, bf.rTree.header.rootOffset) assertEquals(14810, bf.rTree.header.itemCount) - if (prefetch > 1) { + if (prefetch >= BigFile.PREFETCH_LEVEL_DETAILED) { assertNotNull(bf.rTree.rootNode) assertTrue(bf.rTree.rootNode!! is RTReeNodeLeaf) assertEquals(232, (bf.rTree.rootNode!! as RTReeNodeLeaf).leaves.size) diff --git a/src/test/kotlin/org/jetbrains/bio/big/BigWigFileTest.kt b/src/test/kotlin/org/jetbrains/bio/big/BigWigFileTest.kt index d7735b1..c588413 100644 --- a/src/test/kotlin/org/jetbrains/bio/big/BigWigFileTest.kt +++ b/src/test/kotlin/org/jetbrains/bio/big/BigWigFileTest.kt @@ -23,8 +23,8 @@ class BigWigFileTest( BigWigFile.read(Examples["example2.bw"], bfProvider, prefetch).use { bf -> assertEquals(10, bf.zoomLevels.size) assertEquals(25600, bf.zoomLevels[4].reduction) - assertEquals(prefetch > 0, bf.prefetchedLevel2RTreeIndex != null) - if (prefetch > 0) { + assertEquals(prefetch == BigFile.PREFETCH_LEVEL_OFF, bf.prefetchedLevel2RTreeIndex == null) + if (prefetch >= BigFile.PREFETCH_LEVEL_FAST) { assertEquals(10, bf.prefetchedLevel2RTreeIndex!!.size) val zRTree0 = bf.prefetchedLevel2RTreeIndex!![bf.zoomLevels[0]]!! @@ -42,8 +42,8 @@ class BigWigFileTest( (zRTree4.rootNode!! as RTReeNodeLeaf).leaves[1].interval) } - assertEquals(prefetch > 0, bf.prefetchedChr2Leaf != null) - if (prefetch > 0) { + assertEquals(prefetch == BigFile.PREFETCH_LEVEL_OFF, bf.prefetchedChr2Leaf == null) + if (prefetch >= BigFile.PREFETCH_LEVEL_FAST) { assertEquals(1, bf.prefetchedChr2Leaf!!.size) assertEquals("chr21", bf.prefetchedChr2Leaf!!["chr21"]!!.key) assertEquals(0, bf.prefetchedChr2Leaf!!["chr21"]!!.id) @@ -59,7 +59,7 @@ class BigWigFileTest( assertEquals(15751097, bf.rTree.header.rootOffset) assertEquals(6857, bf.rTree.header.itemCount) - if (prefetch > 1) { + if (prefetch >= BigFile.PREFETCH_LEVEL_DETAILED) { assertNotNull(bf.rTree.rootNode) assertTrue(bf.rTree.rootNode!! is RTReeNodeIntermediate) assertEquals(27, (bf.rTree.rootNode!! as RTReeNodeIntermediate).children.size) diff --git a/src/test/kotlin/org/jetbrains/bio/big/RTreeIndexTest.kt b/src/test/kotlin/org/jetbrains/bio/big/RTreeIndexTest.kt index a522a97..d043f43 100644 --- a/src/test/kotlin/org/jetbrains/bio/big/RTreeIndexTest.kt +++ b/src/test/kotlin/org/jetbrains/bio/big/RTreeIndexTest.kt @@ -24,7 +24,7 @@ class RTreeIndexTest( assertEquals(192771L, rti.header.endDataOffset) assertEquals(64, rti.header.itemsPerSlot) assertEquals(192819L, rti.header.rootOffset) - if (prefetch > 1) { + if (prefetch >= BigFile.PREFETCH_LEVEL_DETAILED) { assertNotNull(bbf.rTree.rootNode) assertTrue(bbf.rTree.rootNode is RTReeNodeLeaf) assertEquals(232, (bbf.rTree.rootNode as RTReeNodeLeaf).leaves.size) @@ -49,7 +49,7 @@ class RTreeIndexTest( f.create().use { input -> val rti = RTreeIndex.read(input, 0L) - if (prefetch > 0) { + if (prefetch >= BigFile.PREFETCH_LEVEL_FAST) { rti.prefetchBlocksIndex(input, true, false, 0, null) } @@ -108,7 +108,7 @@ class RTreeIndexTest( bfProvider(path.toString(), ByteOrder.nativeOrder()).use { f -> f.create().use { input -> val rti = RTreeIndex.read(input, 0) - if (prefetch > 0) { + if (prefetch >= BigFile.PREFETCH_LEVEL_FAST) { rti.prefetchBlocksIndex(input, true, false, 0, null) } for (leaf in leaves) {