From a17e2d413d17c42a9f656ad3e01e4b956260fc86 Mon Sep 17 00:00:00 2001 From: Vincent Swarte Date: Sun, 10 Mar 2024 20:17:26 +0100 Subject: [PATCH 01/11] Initial implementation of the less-copy Matbin parser --- Cargo.lock | 1 + crates/cli/src/bin/matbin-test.rs | 83 ++++++++ crates/formats/Cargo.toml | 2 +- crates/formats/src/matbin.rs | 141 -------------- crates/formats/src/matbin/mod.rs | 309 ++++++++++++++++++++++++++++++ 5 files changed, 394 insertions(+), 142 deletions(-) create mode 100644 crates/cli/src/bin/matbin-test.rs delete mode 100644 crates/formats/src/matbin.rs create mode 100644 crates/formats/src/matbin/mod.rs diff --git a/Cargo.lock b/Cargo.lock index bed3e9f..412ad5d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2159,6 +2159,7 @@ dependencies = [ "rsa", "rug", "thiserror", + "widestring", "zerocopy", ] diff --git a/crates/cli/src/bin/matbin-test.rs b/crates/cli/src/bin/matbin-test.rs new file mode 100644 index 0000000..f1d553b --- /dev/null +++ b/crates/cli/src/bin/matbin-test.rs @@ -0,0 +1,83 @@ +use std::{error::Error, io::Read, path::PathBuf}; + +use clap::Parser; +use fstools_formats::{bnd4::BND4, dcx::DcxHeader, matbin::Matbin}; +use fstools_vfs::{FileKeyProvider, Vfs}; + +#[derive(Parser, Debug)] +#[command(version, about, long_about = None)] +struct Args { + #[arg(long)] + erpath: PathBuf, +} + +fn main() -> Result<(), Box> { + let args = Args::parse(); + + let er_path = args.erpath; + + let keys = FileKeyProvider::new("keys"); + let archives = [ + er_path.join("Data0"), + er_path.join("Data1"), + er_path.join("Data2"), + er_path.join("Data3"), + er_path.join("sd/sd"), + ]; + + let vfs = Vfs::create(archives.clone(), &keys) + .expect("unable to create vfs"); + let matbinbnd = vfs.open("/material/allmaterial.matbinbnd.dcx") + .unwrap(); + + let (_, mut decoder) = DcxHeader::read(matbinbnd)?; + + let mut decompressed = Vec::with_capacity(decoder.hint_size()); + decoder.read_to_end(&mut decompressed)?; + + let mut cursor = std::io::Cursor::new(decompressed); + let bnd4 = BND4::from_reader(&mut cursor)?; + + for file in bnd4.files.iter() { + println!(" + Walking file {}", file.path); + + let start = file.data_offset as usize; + let end = start + file.compressed_size as usize; + + let bytes = &bnd4.data[start..end]; + let matbin = Matbin::parse(bytes).unwrap(); + + println!( + " - Source path: {}", + matbin.source_path().unwrap().to_string().unwrap() + ); + println!( + " - Shader path: {}", + matbin.shader_path().unwrap().to_string().unwrap() + ); + + let parameters = matbin.parameters().collect::, _>>() + .expect("Could not collect samplers"); + for parameter in parameters.iter(){ + println!( + " - Parameter: {} = {:?}", + parameter.name.to_string().unwrap(), + parameter.value, + ); + } + + let samplers = matbin.samplers().collect::, _>>() + .expect("Could not collect samplers"); + for sampler in samplers.iter() { + println!( + " - Sampler: {} = {}", + sampler.sampler_type.to_string().unwrap(), + sampler.path.to_string().unwrap(), + ); + } + + println!(""); + } + + Ok(()) +} diff --git a/crates/formats/Cargo.toml b/crates/formats/Cargo.toml index 64e93a5..59b4bb7 100644 --- a/crates/formats/Cargo.toml +++ b/crates/formats/Cargo.toml @@ -20,4 +20,4 @@ rsa = "0.9" rug = "1.24" zerocopy = { version = "0.7.32", features = ["derive"] } thiserror.workspace = true - +widestring = "1" diff --git a/crates/formats/src/matbin.rs b/crates/formats/src/matbin.rs deleted file mode 100644 index 51c9475..0000000 --- a/crates/formats/src/matbin.rs +++ /dev/null @@ -1,141 +0,0 @@ -use std::io::{self, SeekFrom}; - -use byteorder::{ReadBytesExt, LE}; - -use crate::io_ext::ReadFormatsExt; - -#[derive(Debug)] -pub enum MatbinError { - IO(io::Error), -} - -#[derive(Debug)] -pub struct Matbin { - pub unk04: u32, - pub shader_path: String, - pub source_path: String, - pub key: u32, - pub param_count: u32, - pub sampler_count: u32, - pub params: Vec, - pub samplers: Vec, -} - -impl Matbin { - pub fn from_reader(r: &mut (impl io::Read + io::Seek)) -> Result { - let _magic = r.read_u32::()?; - // assert!(magic == 0x42414d, "Matbin was not of expected format"); - - let unk04 = r.read_u32::()?; - let shader_path_offset = r.read_u64::()?; - let source_path_offset = r.read_u64::()?; - let key = r.read_u32::()?; - let param_count = r.read_u32::()?; - let sampler_count = r.read_u32::()?; - - let current_pos = r.stream_position()?; - r.seek(SeekFrom::Start(shader_path_offset))?; - let shader_path = r.read_utf16::()?; - r.seek(SeekFrom::Start(source_path_offset))?; - let source_path = r.read_utf16::()?; - r.seek(SeekFrom::Start(current_pos))?; - - assert!(r.read_u64::()? == 0x0); - assert!(r.read_u64::()? == 0x0); - assert!(r.read_u32::()? == 0x0); - - let mut params = vec![]; - for _ in 0..param_count { - params.push(MatbinParam::from_reader(r)?); - } - - let mut samplers = vec![]; - for _ in 0..sampler_count { - samplers.push(MatbinSampler::from_reader(r)?); - } - - Ok(Self { - unk04, - shader_path, - source_path, - key, - param_count, - sampler_count, - params, - samplers, - }) - } -} - -#[derive(Debug)] -pub struct MatbinParam { - pub name: String, - pub value: u32, - pub key: u32, - pub value_type: u32, -} - -impl MatbinParam { - pub fn from_reader(r: &mut (impl io::Read + io::Seek)) -> Result { - let name_offset = r.read_u64::()?; - - // TODO: read values - let _value_offset = r.read_u64::()?; - let key = r.read_u32::()?; - let value_type = r.read_u32::()?; - - assert!(r.read_u64::()? == 0x0); - assert!(r.read_u64::()? == 0x0); - - let current_pos = r.stream_position()?; - r.seek(SeekFrom::Start(name_offset))?; - let name = r.read_utf16::()?; - r.seek(SeekFrom::Start(current_pos))?; - - Ok(Self { - name, - value: 0x0, - key, - value_type, - }) - } -} - -#[derive(Debug)] -pub struct MatbinSampler { - pub sampler_type: String, - pub path: String, - pub key: u32, - pub unkx: f32, - pub unky: f32, -} - -impl MatbinSampler { - pub fn from_reader(r: &mut (impl io::Read + io::Seek)) -> Result { - let type_offset = r.read_u64::()?; - let path_offset = r.read_u64::()?; - let key = r.read_u32::()?; - - let unkx = r.read_f32::()?; - let unky = r.read_f32::()?; - - assert!(r.read_u64::()? == 0x0); - assert!(r.read_u64::()? == 0x0); - assert!(r.read_u32::()? == 0x0); - - let current_pos = r.stream_position()?; - r.seek(SeekFrom::Start(type_offset))?; - let sampler_type = r.read_utf16::()?; - r.seek(SeekFrom::Start(path_offset))?; - let path = r.read_utf16::()?; - r.seek(SeekFrom::Start(current_pos))?; - - Ok(Self { - sampler_type, - path, - key, - unkx, - unky, - }) - } -} diff --git a/crates/formats/src/matbin/mod.rs b/crates/formats/src/matbin/mod.rs new file mode 100644 index 0000000..ab8fe5d --- /dev/null +++ b/crates/formats/src/matbin/mod.rs @@ -0,0 +1,309 @@ +use std::{borrow::Cow, io}; + +use bytemuck::PodCastError; +use byteorder::LE; +use thiserror::Error; +use widestring::{U16Str, U16String}; +use zerocopy::{FromBytes, FromZeroes, Ref, F32, U32, U64}; + +use crate::io_ext::zerocopy::Padding; + +#[derive(Debug, Error)] +pub enum MatbinError { + #[error("Could not copy bytes {0}")] + IO(#[from] io::Error), + + #[error("Could not read string")] + String(#[from] ReadUtf16StringError), + + #[error("Got unknown parameter type {0}")] + UnknownParameterType(u32), + + #[error("Could not create reference to value")] + ParameterValue, +} + +#[derive(FromZeroes, FromBytes)] +#[repr(C)] +#[allow(unused)] +// Defines a material for instancing in FLVERs and such. +// It does so by pointing at a shader and specifying the parameter/sampler +// setup. +pub struct Matbin<'buffer> { + bytes: &'buffer [u8], + + header: &'buffer Header, + + parameters: &'buffer [Parameter], + + samplers: &'buffer [Sampler], +} + +impl<'buffer> Matbin<'buffer> { + pub fn parse(bytes: &'buffer [u8]) -> Option { + let (header, next) = Ref::<_, Header>::new_from_prefix(bytes)?; + let (parameters, next) = Parameter::slice_from_prefix( + next, + header.parameter_count.get() as usize, + )?; + + let (samplers, _) = Sampler::slice_from_prefix( + next, + header.sampler_count.get() as usize, + )?; + + Some(Self { + bytes, + header: header.into_ref(), + parameters, + samplers, + }) + } + + pub fn shader_path(&self) -> Result, MatbinError> { + let offset = self.header.shader_path_offset.get() as usize; + let bytes = &self.bytes[offset..]; + + Ok(read_utf16_string(bytes)?) + } + + pub fn source_path(&self) -> Result, MatbinError> { + let offset = self.header.source_path_offset.get() as usize; + let bytes = &self.bytes[offset..]; + + Ok(read_utf16_string(bytes)?) + } + + pub fn samplers( + &self, + ) -> impl Iterator> { + self.samplers.iter() + .map(|e| { + let sampler_type = { + let offset = e.type_offset.get() as usize; + let bytes = &self.bytes[offset..]; + read_utf16_string(bytes) + }?; + + let path = { + let offset = e.path_offset.get() as usize; + let bytes = &self.bytes[offset..]; + read_utf16_string(bytes) + }?; + + Ok(SamplerIterElement { + sampler_type, + path, + }) + }) + } + + pub fn parameters( + &self, + ) -> impl Iterator> { + self.parameters.iter() + .map(|e| { + let name = { + let offset = e.name_offset.get() as usize; + let bytes = &self.bytes[offset..]; + read_utf16_string(bytes) + }?; + + let value_slice = &self.bytes[e.value_offset.get() as usize..]; + let value = ParameterValue::from_type_and_slice( + e.value_type.get(), + value_slice, + )?; + + Ok(ParameterIterElement { + name, + value, + }) + }) + } +} + +pub struct ParameterIterElement<'buffer> { + pub name: Cow<'buffer, U16Str>, + pub value: ParameterValue<'buffer>, +} + +pub struct SamplerIterElement<'buffer> { + pub sampler_type: Cow<'buffer, U16Str>, + pub path: Cow<'buffer, U16Str>, +} + +impl<'buffer> std::fmt::Debug for Matbin<'buffer> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Matbin") + .field("shader_path", &self.shader_path()) + .field("source_path", &self.source_path()) + .field("header", self.header) + .field("parameters", &self.parameters) + .field("samplers", &self.samplers) + .finish() + } +} + +#[derive(Debug)] +pub enum ParameterValue<'buffer> { + Bool(bool), + Int(&'buffer U32), + IntVec2(&'buffer [U32]), + Float(&'buffer F32), + FloatVec2(&'buffer [F32]), + FloatVec3(&'buffer [F32]), + FloatVec4(&'buffer [F32]), + FloatVec5(&'buffer [F32]), +} + +impl<'buffer> ParameterValue<'buffer> { + pub fn from_type_and_slice( + value_type: u32, + value_slice: &'buffer [u8], + ) -> Result { + Ok(match value_type { + 0x0 => ParameterValue::Bool( + value_slice[0] != 0x0 + ), + 0x4 => ParameterValue::Int( + U32::::ref_from_prefix(value_slice) + .ok_or(MatbinError::ParameterValue)?, + ), + 0x5 => ParameterValue::IntVec2( + U32::::slice_from_prefix(value_slice, 2) + .ok_or(MatbinError::ParameterValue)?.0, + ), + 0x8 => ParameterValue::Float( + F32::::ref_from_prefix(value_slice) + .ok_or(MatbinError::ParameterValue)?, + ), + 0x9 => ParameterValue::FloatVec2( + F32::::slice_from_prefix(value_slice, 2) + .ok_or(MatbinError::ParameterValue)?.0, + ), + 0xA => ParameterValue::FloatVec3( + F32::::slice_from_prefix(value_slice, 3) + .ok_or(MatbinError::ParameterValue)?.0, + ), + 0xB => ParameterValue::FloatVec4( + F32::::slice_from_prefix(value_slice, 4) + .ok_or(MatbinError::ParameterValue)?.0, + ), + 0xC => ParameterValue::FloatVec5( + F32::::slice_from_prefix(value_slice, 5) + .ok_or(MatbinError::ParameterValue)?.0, + ), + _ => { + return Err(MatbinError::UnknownParameterType(value_type)) + } + }) + } +} + +#[derive(FromZeroes, FromBytes, Debug)] +#[repr(C)] +#[allow(unused)] +pub struct Header { + chunk_magic: [u8; 4], + + // Seems to be 2? Might be some version number. Couldn't easily find the + /// parser with Ghidra so :shrug:. + unk04: U32, + + /// Offset to the shader path + shader_path_offset: U64, + + /// Offset to the source path as a wstring. Seems to reference the source + /// for the current matbin file. + source_path_offset: U64, + + /// Adler32 hash of the source path string without the string terminator + source_path_hash: U32, + + /// Amount of parameters for this material + parameter_count: U32, + + /// Amount of samples for this material + sampler_count: U32, + + _padding24: Padding<20>, +} + +#[derive(FromZeroes, FromBytes, Debug)] +#[repr(C)] +#[allow(unused)] +pub struct Parameter { + /// Offset to name of the parameter + name_offset: U64, + + /// Offset to value of the parameter + value_offset: U64, + + /// Adler32 hash of the name string without the string terminator + name_hash: U32, + + /// Type of the value pointed at by value_offset + value_type: U32, + + _padding18: Padding<16>, +} + +#[derive(FromZeroes, FromBytes, Debug)] +#[repr(C)] +#[allow(unused)] +pub struct Sampler { + // Offset to the samplers type + type_offset: U64, + + // Offset to the samplers path + path_offset: U64, + + // Adler32 hash of the type string without the string terminator + type_hash: U32, + + // ??? + unkxy: [F32; 2], + + _padding1c: Padding<20>, +} + +#[derive(Debug, Error)] +pub enum ReadUtf16StringError { + #[error("Could not find end of string")] + NoEndFound, + + #[error("Bytemuck could not cast the pod")] + Bytemuck, +} + +fn read_utf16_string( + input: &[u8] +) -> Result, ReadUtf16StringError> { + // Find the end of the input string + let length = input.chunks_exact(2) + .position(|bytes| bytes[0] == 0x0 && bytes[1] == 0x0) + .ok_or(ReadUtf16StringError::NoEndFound)?; + + // Create a view that has a proper end so we don't copy + // the entire input slice if required and so we don't have to deal with + // bytemuck freaking out over the end not being aligned + let string_bytes = &input[..length * 2]; + + Ok(match bytemuck::try_cast_slice::(string_bytes) { + Ok(s) => Cow::Borrowed(U16Str::from_slice(s)), + Err(e) => { + // We should probably return the error if it isn't strictly + // about the alignment of the input. + if e != PodCastError::TargetAlignmentGreaterAndInputNotAligned { + return Err(ReadUtf16StringError::Bytemuck); + } + + let aligned_copy = string_bytes.chunks(2) + .map(|a| u16::from_le_bytes([a[0], a[1]])) + .collect::>(); + + Cow::Owned(U16String::from_vec(aligned_copy)) + } + }) +} From f8ecf02d3d602cce65538df48def9046dd3255fc Mon Sep 17 00:00:00 2001 From: Vincent Swarte Date: Sun, 10 Mar 2024 20:39:50 +0100 Subject: [PATCH 02/11] Renamed MatbinError::ValueZerocopy to MatbinError::UnalignedValue --- crates/formats/src/matbin/mod.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/formats/src/matbin/mod.rs b/crates/formats/src/matbin/mod.rs index ab8fe5d..ffe7310 100644 --- a/crates/formats/src/matbin/mod.rs +++ b/crates/formats/src/matbin/mod.rs @@ -20,7 +20,7 @@ pub enum MatbinError { UnknownParameterType(u32), #[error("Could not create reference to value")] - ParameterValue, + UnalignedValue, } #[derive(FromZeroes, FromBytes)] @@ -168,31 +168,31 @@ impl<'buffer> ParameterValue<'buffer> { ), 0x4 => ParameterValue::Int( U32::::ref_from_prefix(value_slice) - .ok_or(MatbinError::ParameterValue)?, + .ok_or(MatbinError::UnalignedValue)?, ), 0x5 => ParameterValue::IntVec2( U32::::slice_from_prefix(value_slice, 2) - .ok_or(MatbinError::ParameterValue)?.0, + .ok_or(MatbinError::UnalignedValue)?.0, ), 0x8 => ParameterValue::Float( F32::::ref_from_prefix(value_slice) - .ok_or(MatbinError::ParameterValue)?, + .ok_or(MatbinError::UnalignedValue)?, ), 0x9 => ParameterValue::FloatVec2( F32::::slice_from_prefix(value_slice, 2) - .ok_or(MatbinError::ParameterValue)?.0, + .ok_or(MatbinError::UnalignedValue)?.0, ), 0xA => ParameterValue::FloatVec3( F32::::slice_from_prefix(value_slice, 3) - .ok_or(MatbinError::ParameterValue)?.0, + .ok_or(MatbinError::UnalignedValue)?.0, ), 0xB => ParameterValue::FloatVec4( F32::::slice_from_prefix(value_slice, 4) - .ok_or(MatbinError::ParameterValue)?.0, + .ok_or(MatbinError::UnalignedValue)?.0, ), 0xC => ParameterValue::FloatVec5( F32::::slice_from_prefix(value_slice, 5) - .ok_or(MatbinError::ParameterValue)?.0, + .ok_or(MatbinError::UnalignedValue)?.0, ), _ => { return Err(MatbinError::UnknownParameterType(value_type)) @@ -273,7 +273,7 @@ pub enum ReadUtf16StringError { #[error("Could not find end of string")] NoEndFound, - #[error("Bytemuck could not cast the pod")] + #[error("Bytemuck could not cast pod")] Bytemuck, } From 574b93d216ec8d05173f6aeb66908c934b48df2f Mon Sep 17 00:00:00 2001 From: Vincent Swarte Date: Sun, 10 Mar 2024 20:44:04 +0100 Subject: [PATCH 03/11] Remove unnecessary derives and repr from Matbin --- crates/formats/src/matbin/mod.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/formats/src/matbin/mod.rs b/crates/formats/src/matbin/mod.rs index ffe7310..acc1940 100644 --- a/crates/formats/src/matbin/mod.rs +++ b/crates/formats/src/matbin/mod.rs @@ -23,12 +23,10 @@ pub enum MatbinError { UnalignedValue, } -#[derive(FromZeroes, FromBytes)] -#[repr(C)] -#[allow(unused)] // Defines a material for instancing in FLVERs and such. // It does so by pointing at a shader and specifying the parameter/sampler // setup. +#[allow(unused)] pub struct Matbin<'buffer> { bytes: &'buffer [u8], From 5a8254ca826fda4d5c084f43e91449126f675289 Mon Sep 17 00:00:00 2001 From: Vincent Swarte Date: Sun, 10 Mar 2024 20:45:20 +0100 Subject: [PATCH 04/11] Rename IO error invariant to Io --- crates/formats/src/matbin/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/formats/src/matbin/mod.rs b/crates/formats/src/matbin/mod.rs index acc1940..52c5d21 100644 --- a/crates/formats/src/matbin/mod.rs +++ b/crates/formats/src/matbin/mod.rs @@ -11,7 +11,7 @@ use crate::io_ext::zerocopy::Padding; #[derive(Debug, Error)] pub enum MatbinError { #[error("Could not copy bytes {0}")] - IO(#[from] io::Error), + Io(#[from] io::Error), #[error("Could not read string")] String(#[from] ReadUtf16StringError), From 1fd01fd1a40e296400e33e7fe691ef86ef74c3d6 Mon Sep 17 00:00:00 2001 From: Vincent Swarte Date: Sun, 10 Mar 2024 20:51:43 +0100 Subject: [PATCH 05/11] repr(C) to repr(packed) on the FromBytes structs --- crates/formats/src/matbin/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/formats/src/matbin/mod.rs b/crates/formats/src/matbin/mod.rs index 52c5d21..fd1cf0e 100644 --- a/crates/formats/src/matbin/mod.rs +++ b/crates/formats/src/matbin/mod.rs @@ -200,7 +200,7 @@ impl<'buffer> ParameterValue<'buffer> { } #[derive(FromZeroes, FromBytes, Debug)] -#[repr(C)] +#[repr(packed)] #[allow(unused)] pub struct Header { chunk_magic: [u8; 4], @@ -229,7 +229,7 @@ pub struct Header { } #[derive(FromZeroes, FromBytes, Debug)] -#[repr(C)] +#[repr(packed)] #[allow(unused)] pub struct Parameter { /// Offset to name of the parameter @@ -248,7 +248,7 @@ pub struct Parameter { } #[derive(FromZeroes, FromBytes, Debug)] -#[repr(C)] +#[repr(packed)] #[allow(unused)] pub struct Sampler { // Offset to the samplers type From 808b7a96a9af4984a4ac229712b30be1cc015e48 Mon Sep 17 00:00:00 2001 From: Vincent Swarte Date: Sun, 10 Mar 2024 20:53:01 +0100 Subject: [PATCH 06/11] Renamed lifetime 'buffer to 'a --- crates/formats/src/matbin/mod.rs | 48 ++++++++++++++++---------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/crates/formats/src/matbin/mod.rs b/crates/formats/src/matbin/mod.rs index fd1cf0e..db24be4 100644 --- a/crates/formats/src/matbin/mod.rs +++ b/crates/formats/src/matbin/mod.rs @@ -27,18 +27,18 @@ pub enum MatbinError { // It does so by pointing at a shader and specifying the parameter/sampler // setup. #[allow(unused)] -pub struct Matbin<'buffer> { - bytes: &'buffer [u8], +pub struct Matbin<'a> { + bytes: &'a [u8], - header: &'buffer Header, + header: &'a Header, - parameters: &'buffer [Parameter], + parameters: &'a [Parameter], - samplers: &'buffer [Sampler], + samplers: &'a [Sampler], } -impl<'buffer> Matbin<'buffer> { - pub fn parse(bytes: &'buffer [u8]) -> Option { +impl<'a> Matbin<'a> { + pub fn parse(bytes: &'a [u8]) -> Option { let (header, next) = Ref::<_, Header>::new_from_prefix(bytes)?; let (parameters, next) = Parameter::slice_from_prefix( next, @@ -121,17 +121,17 @@ impl<'buffer> Matbin<'buffer> { } } -pub struct ParameterIterElement<'buffer> { - pub name: Cow<'buffer, U16Str>, - pub value: ParameterValue<'buffer>, +pub struct ParameterIterElement<'a> { + pub name: Cow<'a, U16Str>, + pub value: ParameterValue<'a>, } -pub struct SamplerIterElement<'buffer> { - pub sampler_type: Cow<'buffer, U16Str>, - pub path: Cow<'buffer, U16Str>, +pub struct SamplerIterElement<'a> { + pub sampler_type: Cow<'a, U16Str>, + pub path: Cow<'a, U16Str>, } -impl<'buffer> std::fmt::Debug for Matbin<'buffer> { +impl<'a> std::fmt::Debug for Matbin<'a> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Matbin") .field("shader_path", &self.shader_path()) @@ -144,21 +144,21 @@ impl<'buffer> std::fmt::Debug for Matbin<'buffer> { } #[derive(Debug)] -pub enum ParameterValue<'buffer> { +pub enum ParameterValue<'a> { Bool(bool), - Int(&'buffer U32), - IntVec2(&'buffer [U32]), - Float(&'buffer F32), - FloatVec2(&'buffer [F32]), - FloatVec3(&'buffer [F32]), - FloatVec4(&'buffer [F32]), - FloatVec5(&'buffer [F32]), + Int(&'a U32), + IntVec2(&'a [U32]), + Float(&'a F32), + FloatVec2(&'a [F32]), + FloatVec3(&'a [F32]), + FloatVec4(&'a [F32]), + FloatVec5(&'a [F32]), } -impl<'buffer> ParameterValue<'buffer> { +impl<'a> ParameterValue<'a> { pub fn from_type_and_slice( value_type: u32, - value_slice: &'buffer [u8], + value_slice: &'a [u8], ) -> Result { Ok(match value_type { 0x0 => ParameterValue::Bool( From 4d5f4758d37071fa7a8a8c41f15bd9216614a935 Mon Sep 17 00:00:00 2001 From: Vincent Swarte Date: Sun, 10 Mar 2024 21:48:28 +0100 Subject: [PATCH 07/11] Renames and some tweaks to comments --- crates/cli/src/bin/matbin-test.rs | 2 +- crates/formats/src/matbin/mod.rs | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/crates/cli/src/bin/matbin-test.rs b/crates/cli/src/bin/matbin-test.rs index f1d553b..aef5ef3 100644 --- a/crates/cli/src/bin/matbin-test.rs +++ b/crates/cli/src/bin/matbin-test.rs @@ -71,7 +71,7 @@ fn main() -> Result<(), Box> { for sampler in samplers.iter() { println!( " - Sampler: {} = {}", - sampler.sampler_type.to_string().unwrap(), + sampler.name.to_string().unwrap(), sampler.path.to_string().unwrap(), ); } diff --git a/crates/formats/src/matbin/mod.rs b/crates/formats/src/matbin/mod.rs index db24be4..c3695ba 100644 --- a/crates/formats/src/matbin/mod.rs +++ b/crates/formats/src/matbin/mod.rs @@ -77,8 +77,8 @@ impl<'a> Matbin<'a> { ) -> impl Iterator> { self.samplers.iter() .map(|e| { - let sampler_type = { - let offset = e.type_offset.get() as usize; + let name = { + let offset = e.name_offset.get() as usize; let bytes = &self.bytes[offset..]; read_utf16_string(bytes) }?; @@ -90,7 +90,7 @@ impl<'a> Matbin<'a> { }?; Ok(SamplerIterElement { - sampler_type, + name, path, }) }) @@ -206,7 +206,7 @@ pub struct Header { chunk_magic: [u8; 4], // Seems to be 2? Might be some version number. Couldn't easily find the - /// parser with Ghidra so :shrug:. + // parser with Ghidra so :shrug:. unk04: U32, /// Offset to the shader path @@ -251,16 +251,16 @@ pub struct Parameter { #[repr(packed)] #[allow(unused)] pub struct Sampler { - // Offset to the samplers type - type_offset: U64, + /// Offset to the samplers name + name_offset: U64, - // Offset to the samplers path + /// Offset to the samplers path path_offset: U64, - // Adler32 hash of the type string without the string terminator - type_hash: U32, + /// Adler32 hash of the name string without the string terminator + name_hash: U32, - // ??? + /// ??? unkxy: [F32; 2], _padding1c: Padding<20>, From 042418143a2f0747da2b22b94c531ea4a0fddb4c Mon Sep 17 00:00:00 2001 From: Vincent Swarte Date: Sun, 10 Mar 2024 21:49:14 +0100 Subject: [PATCH 08/11] Moving the iter element structs to after the Matbin Debug impl --- crates/formats/src/matbin/mod.rs | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/crates/formats/src/matbin/mod.rs b/crates/formats/src/matbin/mod.rs index c3695ba..979cc96 100644 --- a/crates/formats/src/matbin/mod.rs +++ b/crates/formats/src/matbin/mod.rs @@ -121,16 +121,6 @@ impl<'a> Matbin<'a> { } } -pub struct ParameterIterElement<'a> { - pub name: Cow<'a, U16Str>, - pub value: ParameterValue<'a>, -} - -pub struct SamplerIterElement<'a> { - pub sampler_type: Cow<'a, U16Str>, - pub path: Cow<'a, U16Str>, -} - impl<'a> std::fmt::Debug for Matbin<'a> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Matbin") @@ -143,7 +133,16 @@ impl<'a> std::fmt::Debug for Matbin<'a> { } } -#[derive(Debug)] +pub struct ParameterIterElement<'a> { + pub name: Cow<'a, U16Str>, + pub value: ParameterValue<'a>, +} + +pub struct SamplerIterElement<'a> { + pub name: Cow<'a, U16Str>, + pub path: Cow<'a, U16Str>, +} + pub enum ParameterValue<'a> { Bool(bool), Int(&'a U32), From 2c94b738df8f4792e482d1170c567fe733c286af Mon Sep 17 00:00:00 2001 From: Vincent Swarte Date: Sun, 10 Mar 2024 21:49:52 +0100 Subject: [PATCH 09/11] Implement std::fmt::Debug manually on ParameterValue --- crates/formats/src/matbin/mod.rs | 41 ++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/crates/formats/src/matbin/mod.rs b/crates/formats/src/matbin/mod.rs index 979cc96..71f6cc4 100644 --- a/crates/formats/src/matbin/mod.rs +++ b/crates/formats/src/matbin/mod.rs @@ -198,6 +198,47 @@ impl<'a> ParameterValue<'a> { } } +impl<'a> std::fmt::Debug for ParameterValue<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&match self { + ParameterValue::Bool(v) => format!("Bool({})", v), + ParameterValue::Int(v) => format!("Int({})", v.get()), + ParameterValue::IntVec2(v) => format!( + "IntVec2([{}, {}])", + v[0].get(), + v[1].get(), + ), + ParameterValue::Float(v) => format!("Float({})", v.get()), + ParameterValue::FloatVec2(v) => format!( + "FloatVec2([{}, {}])", + v[0].get(), + v[1].get(), + ), + ParameterValue::FloatVec3(v) => format!( + "FloatVec3([{}, {}, {}])", + v[0].get(), + v[1].get(), + v[2].get(), + ), + ParameterValue::FloatVec4(v) => format!( + "FloatVec4([{}, {}, {}, {}])", + v[0].get(), + v[1].get(), + v[2].get(), + v[3].get(), + ), + ParameterValue::FloatVec5(v) => format!( + "FloatVec5([{}, {}, {}, {}, {}])", + v[0].get(), + v[1].get(), + v[2].get(), + v[3].get(), + v[4].get(), + ), + }) + } +} + #[derive(FromZeroes, FromBytes, Debug)] #[repr(packed)] #[allow(unused)] From cb3b215c0b934072c6f5c60434ef0639a98718fa Mon Sep 17 00:00:00 2001 From: Vincent Swarte Date: Sun, 10 Mar 2024 21:51:15 +0100 Subject: [PATCH 10/11] Fix clippy feedback --- crates/cli/src/bin/matbin-test.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/cli/src/bin/matbin-test.rs b/crates/cli/src/bin/matbin-test.rs index aef5ef3..b7c08f6 100644 --- a/crates/cli/src/bin/matbin-test.rs +++ b/crates/cli/src/bin/matbin-test.rs @@ -75,8 +75,6 @@ fn main() -> Result<(), Box> { sampler.path.to_string().unwrap(), ); } - - println!(""); } Ok(()) From fc91e91a3d80a150eac5c6ce81efa3c37c7cc863 Mon Sep 17 00:00:00 2001 From: Vincent Swarte Date: Sun, 10 Mar 2024 21:52:59 +0100 Subject: [PATCH 11/11] Run rustfmt --- crates/cli/src/bin/matbin-test.rs | 16 ++-- crates/formats/src/matbin/mod.rs | 137 ++++++++++++------------------ 2 files changed, 63 insertions(+), 90 deletions(-) diff --git a/crates/cli/src/bin/matbin-test.rs b/crates/cli/src/bin/matbin-test.rs index b7c08f6..eacd12d 100644 --- a/crates/cli/src/bin/matbin-test.rs +++ b/crates/cli/src/bin/matbin-test.rs @@ -25,10 +25,8 @@ fn main() -> Result<(), Box> { er_path.join("sd/sd"), ]; - let vfs = Vfs::create(archives.clone(), &keys) - .expect("unable to create vfs"); - let matbinbnd = vfs.open("/material/allmaterial.matbinbnd.dcx") - .unwrap(); + let vfs = Vfs::create(archives.clone(), &keys).expect("unable to create vfs"); + let matbinbnd = vfs.open("/material/allmaterial.matbinbnd.dcx").unwrap(); let (_, mut decoder) = DcxHeader::read(matbinbnd)?; @@ -56,9 +54,11 @@ fn main() -> Result<(), Box> { matbin.shader_path().unwrap().to_string().unwrap() ); - let parameters = matbin.parameters().collect::, _>>() + let parameters = matbin + .parameters() + .collect::, _>>() .expect("Could not collect samplers"); - for parameter in parameters.iter(){ + for parameter in parameters.iter() { println!( " - Parameter: {} = {:?}", parameter.name.to_string().unwrap(), @@ -66,7 +66,9 @@ fn main() -> Result<(), Box> { ); } - let samplers = matbin.samplers().collect::, _>>() + let samplers = matbin + .samplers() + .collect::, _>>() .expect("Could not collect samplers"); for sampler in samplers.iter() { println!( diff --git a/crates/formats/src/matbin/mod.rs b/crates/formats/src/matbin/mod.rs index 71f6cc4..8e31b77 100644 --- a/crates/formats/src/matbin/mod.rs +++ b/crates/formats/src/matbin/mod.rs @@ -31,7 +31,7 @@ pub struct Matbin<'a> { bytes: &'a [u8], header: &'a Header, - + parameters: &'a [Parameter], samplers: &'a [Sampler], @@ -40,15 +40,10 @@ pub struct Matbin<'a> { impl<'a> Matbin<'a> { pub fn parse(bytes: &'a [u8]) -> Option { let (header, next) = Ref::<_, Header>::new_from_prefix(bytes)?; - let (parameters, next) = Parameter::slice_from_prefix( - next, - header.parameter_count.get() as usize, - )?; + let (parameters, next) = + Parameter::slice_from_prefix(next, header.parameter_count.get() as usize)?; - let (samplers, _) = Sampler::slice_from_prefix( - next, - header.sampler_count.get() as usize, - )?; + let (samplers, _) = Sampler::slice_from_prefix(next, header.sampler_count.get() as usize)?; Some(Self { bytes, @@ -72,52 +67,37 @@ impl<'a> Matbin<'a> { Ok(read_utf16_string(bytes)?) } - pub fn samplers( - &self, - ) -> impl Iterator> { - self.samplers.iter() - .map(|e| { - let name = { - let offset = e.name_offset.get() as usize; - let bytes = &self.bytes[offset..]; - read_utf16_string(bytes) - }?; - - let path = { - let offset = e.path_offset.get() as usize; - let bytes = &self.bytes[offset..]; - read_utf16_string(bytes) - }?; - - Ok(SamplerIterElement { - name, - path, - }) - }) + pub fn samplers(&self) -> impl Iterator> { + self.samplers.iter().map(|e| { + let name = { + let offset = e.name_offset.get() as usize; + let bytes = &self.bytes[offset..]; + read_utf16_string(bytes) + }?; + + let path = { + let offset = e.path_offset.get() as usize; + let bytes = &self.bytes[offset..]; + read_utf16_string(bytes) + }?; + + Ok(SamplerIterElement { name, path }) + }) } - pub fn parameters( - &self, - ) -> impl Iterator> { - self.parameters.iter() - .map(|e| { - let name = { - let offset = e.name_offset.get() as usize; - let bytes = &self.bytes[offset..]; - read_utf16_string(bytes) - }?; - - let value_slice = &self.bytes[e.value_offset.get() as usize..]; - let value = ParameterValue::from_type_and_slice( - e.value_type.get(), - value_slice, - )?; - - Ok(ParameterIterElement { - name, - value, - }) - }) + pub fn parameters(&self) -> impl Iterator> { + self.parameters.iter().map(|e| { + let name = { + let offset = e.name_offset.get() as usize; + let bytes = &self.bytes[offset..]; + read_utf16_string(bytes) + }?; + + let value_slice = &self.bytes[e.value_offset.get() as usize..]; + let value = ParameterValue::from_type_and_slice(e.value_type.get(), value_slice)?; + + Ok(ParameterIterElement { name, value }) + }) } } @@ -160,40 +140,39 @@ impl<'a> ParameterValue<'a> { value_slice: &'a [u8], ) -> Result { Ok(match value_type { - 0x0 => ParameterValue::Bool( - value_slice[0] != 0x0 - ), + 0x0 => ParameterValue::Bool(value_slice[0] != 0x0), 0x4 => ParameterValue::Int( - U32::::ref_from_prefix(value_slice) - .ok_or(MatbinError::UnalignedValue)?, + U32::::ref_from_prefix(value_slice).ok_or(MatbinError::UnalignedValue)?, ), 0x5 => ParameterValue::IntVec2( U32::::slice_from_prefix(value_slice, 2) - .ok_or(MatbinError::UnalignedValue)?.0, + .ok_or(MatbinError::UnalignedValue)? + .0, ), 0x8 => ParameterValue::Float( - F32::::ref_from_prefix(value_slice) - .ok_or(MatbinError::UnalignedValue)?, + F32::::ref_from_prefix(value_slice).ok_or(MatbinError::UnalignedValue)?, ), 0x9 => ParameterValue::FloatVec2( F32::::slice_from_prefix(value_slice, 2) - .ok_or(MatbinError::UnalignedValue)?.0, + .ok_or(MatbinError::UnalignedValue)? + .0, ), 0xA => ParameterValue::FloatVec3( F32::::slice_from_prefix(value_slice, 3) - .ok_or(MatbinError::UnalignedValue)?.0, + .ok_or(MatbinError::UnalignedValue)? + .0, ), 0xB => ParameterValue::FloatVec4( F32::::slice_from_prefix(value_slice, 4) - .ok_or(MatbinError::UnalignedValue)?.0, + .ok_or(MatbinError::UnalignedValue)? + .0, ), 0xC => ParameterValue::FloatVec5( F32::::slice_from_prefix(value_slice, 5) - .ok_or(MatbinError::UnalignedValue)?.0, + .ok_or(MatbinError::UnalignedValue)? + .0, ), - _ => { - return Err(MatbinError::UnknownParameterType(value_type)) - } + _ => return Err(MatbinError::UnknownParameterType(value_type)), }) } } @@ -203,17 +182,9 @@ impl<'a> std::fmt::Debug for ParameterValue<'a> { f.write_str(&match self { ParameterValue::Bool(v) => format!("Bool({})", v), ParameterValue::Int(v) => format!("Int({})", v.get()), - ParameterValue::IntVec2(v) => format!( - "IntVec2([{}, {}])", - v[0].get(), - v[1].get(), - ), + ParameterValue::IntVec2(v) => format!("IntVec2([{}, {}])", v[0].get(), v[1].get(),), ParameterValue::Float(v) => format!("Float({})", v.get()), - ParameterValue::FloatVec2(v) => format!( - "FloatVec2([{}, {}])", - v[0].get(), - v[1].get(), - ), + ParameterValue::FloatVec2(v) => format!("FloatVec2([{}, {}])", v[0].get(), v[1].get(),), ParameterValue::FloatVec3(v) => format!( "FloatVec3([{}, {}, {}])", v[0].get(), @@ -315,11 +286,10 @@ pub enum ReadUtf16StringError { Bytemuck, } -fn read_utf16_string( - input: &[u8] -) -> Result, ReadUtf16StringError> { +fn read_utf16_string(input: &[u8]) -> Result, ReadUtf16StringError> { // Find the end of the input string - let length = input.chunks_exact(2) + let length = input + .chunks_exact(2) .position(|bytes| bytes[0] == 0x0 && bytes[1] == 0x0) .ok_or(ReadUtf16StringError::NoEndFound)?; @@ -337,7 +307,8 @@ fn read_utf16_string( return Err(ReadUtf16StringError::Bytemuck); } - let aligned_copy = string_bytes.chunks(2) + let aligned_copy = string_bytes + .chunks(2) .map(|a| u16::from_le_bytes([a[0], a[1]])) .collect::>();