#2: implement reader for big files

big/pom.xml

@@ -29,5 +29,16 @@
 			<artifactId>kotlin-test</artifactId>
 			<scope>test</scope>
 		</dependency>
+		<dependency>
+			<groupId>org.jetbrains.kotlin</groupId>
+			<artifactId>kotlin-test-junit5</artifactId>
+			<scope>test</scope>
+		</dependency>
+
+		<dependency>
+			<groupId>org.assertj</groupId>
+			<artifactId>assertj-core</artifactId>
+			<scope>test</scope>
+		</dependency>
 	</dependencies>
 </project>

big/src/main/kotlin/de/darkatra/bfme2/big/BigArchive.kt

@@ -1,42 +1,139 @@
 package de.darkatra.bfme2.big
 
+import de.darkatra.bfme2.readNullTerminatedString
 import de.darkatra.bfme2.toBigEndianBytes
+import de.darkatra.bfme2.toBigEndianUInt
 import de.darkatra.bfme2.toLittleEndianBytes
 import java.io.OutputStream
+import java.nio.charset.StandardCharsets
 import java.nio.file.Path
 import kotlin.io.path.exists
 import kotlin.io.path.inputStream
+import kotlin.io.path.outputStream
 
 /**
- * Allows writing data as BIG archive.
+ * Allows reading and writing data from and to big archives.
+ *
+ * @param version The version of the big archive.
+ * @param path The path to the big archive.
  *
  * Heavily inspired by https://github.com/OpenSAGE/OpenSAGE/blob/master/src/OpenSage.FileFormats.Big/BigArchive.cs
  */
 class BigArchive(
-	private val version: BigArchiveVersion
+	@Suppress("MemberVisibilityCanBePrivate")
+	val version: BigArchiveVersion,
+	val path: Path
 ) {
 
 	companion object {
-		const val HEADER_SIZE = 16
+		const val HEADER_SIZE = 16u
+
+		fun from(path: Path): BigArchive {
+			if (!path.exists()) {
+				throw IllegalArgumentException("The specified path does not exist.")
+			}
+
+			val fourCCBytes = path.inputStream().use { it.readNBytes(4) }
+			if (fourCCBytes.size < 4) {
+				throw IllegalStateException("Big archive is too small")
+			}
+
+			val version = when (val fourCC = fourCCBytes.toString(StandardCharsets.UTF_8)) {
+				"BIGF" -> BigArchiveVersion.BIG_F
+				"BIG4" -> BigArchiveVersion.BIG_4
+				else -> throw IllegalStateException("Unknown big archive version: '$fourCC'")
+			}
+
+			return BigArchive(version, path)
+		}
+	}
+
+	private val _entries: MutableList<BigArchiveEntry> = arrayListOf()
+
+	@Suppress("MemberVisibilityCanBePrivate")
+	val entries
+		get() = _entries.sortedWith(Comparator.comparing(BigArchiveEntry::name))
+
+	init {
+		readFromDisk()
 	}
 
-	private val entries = arrayListOf<BigArchiveEntry>()
+	/**
+	 * Adds a new entry to the archive.
+	 *
+	 * @param name The name of the entry to add.
+	 */
+	fun createEntry(name: String): BigArchiveEntry {
+		if (name.isBlank()) {
+			throw IllegalArgumentException("Name must not be blank")
+		}
+
+		val entry = BigArchiveEntry(
+			name = name,
+			archive = this,
+			hasPendingChanges = true
+		)
+		_entries.add(entry)
+		return entry
+	}
 
-	fun addFile(file: Path, name: String) {
-		if (!file.exists()) {
-			throw IllegalArgumentException("File does not exist: $file")
+	/**
+	 * Deletes an entry from the archive and writes changes to disk.
+	 *
+	 * @param name The name of the entry to delete.
+	 */
+	@Suppress("unused")
+	fun deleteEntry(name: String) {
+		if (name.isBlank()) {
+			throw IllegalArgumentException("Name must not be blank")
 		}
 
-		entries.add(BigArchiveEntry(file, name))
+		_entries.removeIf { it.name == name }
+		writeToDisk()
 	}
 
-	fun write(output: OutputStream) {
-		entries.sortWith(Comparator.comparing(BigArchiveEntry::name))
+	/**
+	 * Reads the archive from disk.
+	 */
+	@Suppress("MemberVisibilityCanBePrivate")
+	fun readFromDisk() {
+		if (!path.exists()) {
+			return
+		}
+
+		path.inputStream().use {
+			it.skip(4) // skip fourCC
+			it.readNBytes(4).toBigEndianUInt() // archive size
+			val numberOfEntries = it.readNBytes(4).toBigEndianUInt()
+			it.readNBytes(4).toBigEndianUInt() // data start
+
+			for (i in 0u until numberOfEntries) {
+				val entryOffset = it.readNBytes(4).toBigEndianUInt()
+				val entrySize = it.readNBytes(4).toBigEndianUInt()
+				val entryName = it.readNullTerminatedString()
+
+				val bigArchiveEntry = BigArchiveEntry(
+					name = entryName,
+					archive = this,
+					offset = entryOffset,
+					size = entrySize,
+					hasPendingChanges = false
+				)
+
+				_entries.add(bigArchiveEntry)
+			}
+		}
+	}
 
+	/**
+	 * Writes changes to disk.
+	 */
+	fun writeToDisk() {
+		val output = path.outputStream()
 		val tableSize = calculateTableSize()
 		val contentSize = calculateContentSize()
-		val archiveSize: Long = HEADER_SIZE + tableSize + contentSize
-		val dataStart: Int = HEADER_SIZE + tableSize
+		val archiveSize: UInt = HEADER_SIZE + tableSize + contentSize
+		val dataStart: UInt = HEADER_SIZE + tableSize
 
 		output.use {
 			writeHeader(output, archiveSize, dataStart)
@@ -47,44 +144,46 @@ class BigArchive(
 		}
 	}
 
-	private fun writeHeader(output: OutputStream, archiveSize: Long, dataStart: Int) {
+	private fun writeHeader(output: OutputStream, archiveSize: UInt, dataStart: UInt) {
 		output.write(
 			when (version) {
 				BigArchiveVersion.BIG_F -> "BIGF".toByteArray()
 				BigArchiveVersion.BIG_4 -> "BIG4".toByteArray()
 			}
 		)
 
-		output.write(archiveSize.toInt().toLittleEndianBytes())
+		output.write(archiveSize.toLittleEndianBytes())
 		output.write(entries.size.toBigEndianBytes())
 		output.write(dataStart.toBigEndianBytes())
 	}
 
-	private fun writeFileTable(output: OutputStream, dataStart: Int) {
-		var entryOffset: Long = dataStart.toLong()
+	private fun writeFileTable(output: OutputStream, dataStart: UInt) {
+		var entryOffset: UInt = dataStart
 
 		entries.forEach { entry ->
-			output.write(entryOffset.toInt().toBigEndianBytes())
-			output.write(entry.size.toInt().toBigEndianBytes())
-			output.write(entry.name.toByteArray())
-			output.write(byteArrayOf(0.toByte()))
+			output.write(entryOffset.toBigEndianBytes())
+			output.write(entry.size.toBigEndianBytes())
+			// write the entry name as null terminated string
+			output.write(entry.name.toByteArray() + byteArrayOf(0))
 
+			entry.offset = entryOffset
 			entryOffset += entry.size
 		}
 	}
 
 	private fun writeFileContent(output: OutputStream) {
 		entries.forEach { entry ->
-			entry.file.inputStream().transferTo(output)
+			output.write(entry.inputStream().use { it.readAllBytes() })
+			entry.hasPendingChanges = false
 		}
 	}
 
-	private fun calculateTableSize(): Int {
+	private fun calculateTableSize(): UInt {
 		// Each entry has 4 bytes for the offset + 4 for size and a null-terminated string
-		return entries.fold(0) { acc, entry -> acc + 8 + entry.name.length + 1 }
+		return entries.fold(0u) { acc, entry -> acc + 8u + entry.name.length.toUInt() + 1u }
 	}
 
-	private fun calculateContentSize(): Long {
-		return entries.fold(0) { acc, entry -> acc + entry.size }
+	private fun calculateContentSize(): UInt {
+		return entries.fold(0u) { acc, entry -> acc + entry.size }
 	}
 }

big/src/main/kotlin/de/darkatra/bfme2/big/BigArchiveEntry.kt

@@ -1,11 +1,38 @@
 package de.darkatra.bfme2.big
 
-import java.nio.file.Path
+import java.io.ByteArrayOutputStream
+import java.io.InputStream
+import java.io.OutputStream
 
+/**
+ * Represents a single entry in a [BigArchive].
+ *
+ * @param name The name of the entry.
+ */
 class BigArchiveEntry(
-	val file: Path,
-	val name: String
+	val name: String,
+	internal val archive: BigArchive,
+	internal var offset: UInt = 0u,
+	internal var size: UInt = 0u,
+	internal var hasPendingChanges: Boolean,
+	internal val pendingOutputStream: ByteArrayOutputStream = ByteArrayOutputStream()
 ) {
-	val size: Long
-		get() = file.toFile().length()
+
+	/**
+	 * Obtains an input stream for this entry.
+	 */
+	fun inputStream(): InputStream {
+		if (hasPendingChanges) {
+			return pendingOutputStream.toByteArray().inputStream()
+		}
+
+		return BigArchiveEntryInputStream(this)
+	}
+
+	/**
+	 * Obtains an output stream for this entry.
+	 */
+	fun outputStream(): OutputStream {
+		return BigArchiveEntryOutputStream(this, pendingOutputStream)
+	}
 }

big/src/main/kotlin/de/darkatra/bfme2/big/BigArchiveEntryInputStream.kt

@@ -0,0 +1,16 @@
+package de.darkatra.bfme2.big
+
+import de.darkatra.bfme2.SkippingInputStream
+import java.io.InputStream
+import kotlin.io.path.inputStream
+
+internal class BigArchiveEntryInputStream(
+	bigArchiveEntry: BigArchiveEntry
+) : InputStream() {
+
+	private val inputStream = SkippingInputStream(bigArchiveEntry.archive.path.inputStream(), bigArchiveEntry.offset.toLong())
+
+	override fun read(): Int {
+		return inputStream.read()
+	}
+}

big/src/main/kotlin/de/darkatra/bfme2/big/BigArchiveEntryOutputStream.kt

@@ -0,0 +1,16 @@
+package de.darkatra.bfme2.big
+
+import java.io.ByteArrayOutputStream
+import java.io.FilterOutputStream
+
+internal class BigArchiveEntryOutputStream(
+	private val bigArchiveEntry: BigArchiveEntry,
+	private val outputStream: ByteArrayOutputStream
+) : FilterOutputStream(outputStream) {
+
+	override fun flush() {
+		bigArchiveEntry.hasPendingChanges = true
+		bigArchiveEntry.size = outputStream.size().toUInt()
+		outputStream.reset()
+	}
+}

big/src/test/kotlin/de/darkatra/bfme2/big/BigArchiveTest.kt

@@ -0,0 +1,81 @@
+package de.darkatra.bfme2.big
+
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.io.TempDir
+import java.nio.file.Path
+import kotlin.io.path.inputStream
+import kotlin.io.path.toPath
+
+internal class BigArchiveTest {
+
+	private val testFile: Path = javaClass.getResource("/test/hello.txt")!!.toURI().toPath()
+	private val testArchive: Path = javaClass.getResource("/test/hello.big")!!.toURI().toPath()
+
+	@Test
+	internal fun shouldWriteBigArchiveWithFilesToDisk(@TempDir tempDir: Path) {
+
+		val tempFile = tempDir.resolve("out.big")
+		val bigArchive = BigArchive(BigArchiveVersion.BIG_F, tempFile)
+
+		val entry = bigArchive.createEntry("/test/hello.txt")
+
+		assertThat(bigArchive.entries).hasSize(1)
+		assertThat(bigArchive.entries[0]).isNotNull
+		assertThat(bigArchive.entries[0].name).isEqualTo("/test/hello.txt")
+		assertThat(bigArchive.entries[0].archive).isEqualTo(bigArchive)
+		assertThat(bigArchive.entries[0].offset).isEqualTo(0u)
+		assertThat(bigArchive.entries[0].size).isEqualTo(0u)
+		assertThat(bigArchive.entries[0].hasPendingChanges).isTrue
+		assertThat(bigArchive.entries[0].pendingOutputStream).isNotNull
+
+		// write to the entry
+		testFile.inputStream().use { input ->
+			entry.outputStream().use { output ->
+				input.transferTo(output)
+			}
+		}
+		bigArchive.writeToDisk()
+
+		assertThatExpectedEntryForTestFileExists(bigArchive)
+
+		// write to the entry again (should override and produce the same result)
+		testFile.inputStream().use { input ->
+			entry.outputStream().use { output ->
+				input.transferTo(output)
+			}
+		}
+		bigArchive.writeToDisk()
+
+		assertThatExpectedEntryForTestFileExists(bigArchive)
+	}
+
+	@Test
+	internal fun shouldReadBigArchiveWithFiles() {
+
+		val bigArchive = BigArchive.from(testArchive)
+
+		assertThat(bigArchive.entries).hasSize(1)
+		assertThat(bigArchive.entries[0]).isNotNull
+		assertThat(bigArchive.entries[0].name).isEqualTo("/test/hello.txt")
+		assertThat(bigArchive.entries[0].archive).isEqualTo(bigArchive)
+		assertThat(bigArchive.entries[0].offset).isEqualTo(48u)
+		assertThat(bigArchive.entries[0].size).isEqualTo(592u)
+		assertThat(bigArchive.entries[0].hasPendingChanges).isFalse
+		assertThat(bigArchive.entries[0].pendingOutputStream).isNotNull
+
+		val entryBytes = bigArchive.entries[0].inputStream().use { it.readAllBytes() }
+		assertThat(entryBytes).isEqualTo(testFile.inputStream().use { it.readAllBytes() })
+	}
+
+	private fun assertThatExpectedEntryForTestFileExists(bigArchive: BigArchive) {
+		assertThat(bigArchive.entries).hasSize(1)
+		assertThat(bigArchive.entries[0]).isNotNull
+		assertThat(bigArchive.entries[0].name).isEqualTo("/test/hello.txt")
+		assertThat(bigArchive.entries[0].archive).isEqualTo(bigArchive)
+		assertThat(bigArchive.entries[0].offset).isEqualTo(40u)
+		assertThat(bigArchive.entries[0].size).isEqualTo(592u)
+		assertThat(bigArchive.entries[0].hasPendingChanges).isFalse
+		assertThat(bigArchive.entries[0].pendingOutputStream).isNotNull
+	}
+}

big/src/test/resources/test/hello.txt

@@ -0,0 +1 @@
+Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.