-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Support for incremental DCX decoding
- Loading branch information
1 parent
ce2d597
commit 96e2d2c
Showing
11 changed files
with
280 additions
and
160 deletions.
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,65 +1,146 @@ | ||
| use std::io::{Cursor, Error, ErrorKind, Read, Result}; | ||
| use std::{ | ||
| cmp::min, | ||
| io::{BufRead, BufReader, Error, Read, Result, Take}, | ||
| ptr::null_mut, | ||
| }; | ||
|
|
||
| use byteorder::BE; | ||
| use oodle_sys::{ | ||
| OodleLZ_Decode_ThreadPhase_OodleLZ_Decode_ThreadPhaseAll, OodleLZ_Decompress, OODLELZ_FAILED, | ||
| OodleLZDecoder, OodleLZDecoder_Create, OodleLZDecoder_DecodeSome, | ||
| OodleLZ_CheckCRC_OodleLZ_CheckCRC_Yes, OodleLZ_Compressor_OodleLZ_Compressor_Kraken, | ||
| OodleLZ_DecodeSome_Out, OodleLZ_Decode_ThreadPhase_OodleLZ_Decode_Unthreaded, | ||
| OodleLZ_FuzzSafe_OodleLZ_FuzzSafe_Yes, OodleLZ_Verbosity_OodleLZ_Verbosity_Lots, | ||
| OODLELZ_BLOCK_LEN, | ||
| }; | ||
| use zerocopy::U32; | ||
|
|
||
| pub struct DcxDecoderKraken<'a> { | ||
| compressed: &'a [u8], | ||
| uncompressed_size: U32<BE>, | ||
| inner_cursor: Option<Cursor<Vec<u8>>>, | ||
| pub struct DcxDecoderKraken<R: Read> { | ||
| reader: BufReader<Take<R>>, | ||
|
|
||
| /// The total size of the raw data expected to be read from the underlying stream. | ||
| uncompressed_size: u32, | ||
|
|
||
| /// The Oodle decoder instance created for this buffer. | ||
| decoder: *mut OodleLZDecoder, | ||
|
|
||
| /// A sliding window of bytes decoded by the compressor, large enough to keep the past block in | ||
| /// memory while the next block is decoded. | ||
| sliding_window: Box<[u8]>, | ||
|
|
||
| /// The decoders position into the sliding window. | ||
| sliding_window_pos: usize, | ||
|
|
||
| /// The number of bytes that the consuming reader is lagging behind the decoder. | ||
| sliding_window_lag: usize, | ||
| } | ||
|
|
||
| impl<'a> DcxDecoderKraken<'a> { | ||
| pub fn from_buffer(buf: &'a [u8], uncompressed_size: U32<BE>) -> Self { | ||
| impl<R: Read> DcxDecoderKraken<R> { | ||
| // TODO: fix vfs reader so it isn't producing padding | ||
| pub fn new(reader: Take<R>, uncompressed_size: u32) -> Self { | ||
| let compressor = OodleLZ_Compressor_OodleLZ_Compressor_Kraken; | ||
| let decoder = unsafe { | ||
| OodleLZDecoder_Create(compressor, uncompressed_size as i64, null_mut(), 0isize) | ||
| }; | ||
|
|
||
| if decoder.is_null() { | ||
| panic!("return error here: failed to create decoder, check oodle error"); | ||
| } | ||
|
|
||
| let sliding_window = Box::new([0u8; (OODLELZ_BLOCK_LEN * 2) as usize]); | ||
|
|
||
| Self { | ||
| compressed: buf, | ||
| decoder, | ||
| reader: BufReader::with_capacity(OODLELZ_BLOCK_LEN as usize, reader), | ||
| sliding_window, | ||
| sliding_window_pos: 0, | ||
| sliding_window_lag: 0, | ||
| uncompressed_size, | ||
| inner_cursor: None, | ||
| } | ||
| } | ||
| } | ||
| impl<'a> Read for DcxDecoderKraken<'a> { | ||
| // TODO: implement somewhat incremental reading by working with oodle's | ||
| // blocks per example in docs. | ||
| // It currently just decompresses the entire input one go and then | ||
| // operates a Cursor wrapping the decompressed bytes. | ||
|
|
||
| impl<R: Read> Read for DcxDecoderKraken<R> { | ||
| fn read(&mut self, buf: &mut [u8]) -> Result<usize> { | ||
| if self.inner_cursor.is_none() { | ||
| let mut inner_buffer = vec![0u8; self.uncompressed_size.get() as usize]; | ||
| let compressed_len = | ||
| isize::try_from(inner_buffer.len()).map_err(|e| Error::new(ErrorKind::Other, e))?; | ||
| let inner_buffer_len = | ||
| isize::try_from(inner_buffer.len()).map_err(|e| Error::new(ErrorKind::Other, e))?; | ||
| if buf.is_empty() { | ||
| return Ok(0); | ||
| } | ||
|
|
||
| let mut total_written = 0usize; | ||
| while total_written < buf.len() { | ||
| let wpos = self.sliding_window_pos; | ||
|
|
||
| // Check if there's data to be written from the sliding window first | ||
| if self.sliding_window_lag > 0 { | ||
| let bytes_to_copy = min(self.sliding_window_lag, buf.len() - total_written); | ||
| let start = self.sliding_window_pos - self.sliding_window_lag; | ||
| let end = start + bytes_to_copy; | ||
|
|
||
| let src = &self.sliding_window[start..end]; | ||
| let dest = &mut buf[total_written..total_written + bytes_to_copy]; | ||
|
|
||
| dest.copy_from_slice(src); | ||
|
|
||
| self.sliding_window_lag -= bytes_to_copy; | ||
| total_written += bytes_to_copy; | ||
|
|
||
| continue; | ||
| } | ||
|
|
||
| // Read and decode new data | ||
| let input_data = self.reader.fill_buf()?; | ||
| if input_data.is_empty() { | ||
| break; // EOF reached | ||
| } | ||
|
|
||
| let mut out: OodleLZ_DecodeSome_Out = unsafe { std::mem::zeroed() }; | ||
| let result = unsafe { | ||
| OodleLZ_Decompress( | ||
| self.compressed.as_ptr() as *const _, | ||
| compressed_len, | ||
| inner_buffer.as_mut_ptr() as *mut _, | ||
| inner_buffer_len, | ||
| oodle_sys::OodleLZ_FuzzSafe_OodleLZ_FuzzSafe_Yes, | ||
| 0, | ||
| 0, | ||
| std::ptr::null_mut(), | ||
| 0, | ||
| None, | ||
| std::ptr::null_mut(), | ||
| std::ptr::null_mut(), | ||
| 0, | ||
| OodleLZ_Decode_ThreadPhase_OodleLZ_Decode_ThreadPhaseAll, | ||
| ) as usize | ||
| // EXTREMELY unlikely, however unsound otherwise. | ||
| let input_data_len = isize::try_from(input_data.len()).unwrap_or(isize::MAX); | ||
|
|
||
| // SAFETY: | ||
| // - Signedness conversions of offsets are all valid, given that | ||
| // `sliding_window.len() <= i32::MAX` and `self.uncompressed_size < isize::MAX`. | ||
| // - Consumed `input_data_len` is caped at i32::MAX | ||
| OodleLZDecoder_DecodeSome( | ||
| self.decoder, | ||
| &mut out as *mut _, | ||
| self.sliding_window.as_mut_ptr() as *mut _, | ||
| wpos as isize, | ||
| self.uncompressed_size as _, | ||
| (self.sliding_window.len() - wpos) as isize, | ||
| input_data.as_ptr() as *const _, | ||
| input_data_len, | ||
| OodleLZ_FuzzSafe_OodleLZ_FuzzSafe_Yes, | ||
| OodleLZ_CheckCRC_OodleLZ_CheckCRC_Yes, | ||
| OodleLZ_Verbosity_OodleLZ_Verbosity_Lots, | ||
| OodleLZ_Decode_ThreadPhase_OodleLZ_Decode_Unthreaded, | ||
| ) | ||
| }; | ||
|
|
||
| if result == OODLELZ_FAILED as usize { | ||
| return Err(Error::from(ErrorKind::Other)); | ||
| if result == 0 { | ||
| return Err(Error::other("Oodle decoder failed")); | ||
| } | ||
|
|
||
| self.inner_cursor = Some(Cursor::new(inner_buffer)); | ||
| let decoded_bytes = out.decodedCount as usize; | ||
| let consumed_bytes = out.compBufUsed as usize; | ||
|
|
||
| self.reader.consume(consumed_bytes); | ||
|
|
||
| let bytes_to_copy = min(decoded_bytes, buf.len() - total_written); | ||
| let dest = &mut buf[total_written..total_written + bytes_to_copy]; | ||
| let src = &self.sliding_window[wpos..wpos + bytes_to_copy]; | ||
|
|
||
| dest.copy_from_slice(src); | ||
|
|
||
| self.sliding_window_pos += decoded_bytes; | ||
| self.sliding_window_lag = decoded_bytes - bytes_to_copy; | ||
| total_written += bytes_to_copy; | ||
|
|
||
| // Manage sliding window | ||
| if self.sliding_window_pos >= self.sliding_window.len() { | ||
| self.sliding_window.rotate_left(self.sliding_window_pos); | ||
| self.sliding_window_pos = 0; | ||
| } | ||
| } | ||
|
|
||
| self.inner_cursor.as_mut().unwrap().read(buf) | ||
| Ok(total_written) | ||
| } | ||
| } |
Oops, something went wrong.