diff --git a/Cargo.lock b/Cargo.lock index bed3e9f..412ad5d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2159,6 +2159,7 @@ dependencies = [ "rsa", "rug", "thiserror", + "widestring", "zerocopy", ] diff --git a/crates/cli/src/bin/matbin-test.rs b/crates/cli/src/bin/matbin-test.rs new file mode 100644 index 0000000..eacd12d --- /dev/null +++ b/crates/cli/src/bin/matbin-test.rs @@ -0,0 +1,83 @@ +use std::{error::Error, io::Read, path::PathBuf}; + +use clap::Parser; +use fstools_formats::{bnd4::BND4, dcx::DcxHeader, matbin::Matbin}; +use fstools_vfs::{FileKeyProvider, Vfs}; + +#[derive(Parser, Debug)] +#[command(version, about, long_about = None)] +struct Args { + #[arg(long)] + erpath: PathBuf, +} + +fn main() -> Result<(), Box> { + let args = Args::parse(); + + let er_path = args.erpath; + + let keys = FileKeyProvider::new("keys"); + let archives = [ + er_path.join("Data0"), + er_path.join("Data1"), + er_path.join("Data2"), + er_path.join("Data3"), + er_path.join("sd/sd"), + ]; + + let vfs = Vfs::create(archives.clone(), &keys).expect("unable to create vfs"); + let matbinbnd = vfs.open("/material/allmaterial.matbinbnd.dcx").unwrap(); + + let (_, mut decoder) = DcxHeader::read(matbinbnd)?; + + let mut decompressed = Vec::with_capacity(decoder.hint_size()); + decoder.read_to_end(&mut decompressed)?; + + let mut cursor = std::io::Cursor::new(decompressed); + let bnd4 = BND4::from_reader(&mut cursor)?; + + for file in bnd4.files.iter() { + println!(" + Walking file {}", file.path); + + let start = file.data_offset as usize; + let end = start + file.compressed_size as usize; + + let bytes = &bnd4.data[start..end]; + let matbin = Matbin::parse(bytes).unwrap(); + + println!( + " - Source path: {}", + matbin.source_path().unwrap().to_string().unwrap() + ); + println!( + " - Shader path: {}", + matbin.shader_path().unwrap().to_string().unwrap() + ); + + let parameters = matbin + .parameters() + .collect::, _>>() + .expect("Could not collect samplers"); + for parameter in parameters.iter() { + println!( + " - Parameter: {} = {:?}", + parameter.name.to_string().unwrap(), + parameter.value, + ); + } + + let samplers = matbin + .samplers() + .collect::, _>>() + .expect("Could not collect samplers"); + for sampler in samplers.iter() { + println!( + " - Sampler: {} = {}", + sampler.name.to_string().unwrap(), + sampler.path.to_string().unwrap(), + ); + } + } + + Ok(()) +} diff --git a/crates/formats/Cargo.toml b/crates/formats/Cargo.toml index 64e93a5..59b4bb7 100644 --- a/crates/formats/Cargo.toml +++ b/crates/formats/Cargo.toml @@ -20,4 +20,4 @@ rsa = "0.9" rug = "1.24" zerocopy = { version = "0.7.32", features = ["derive"] } thiserror.workspace = true - +widestring = "1" diff --git a/crates/formats/src/matbin.rs b/crates/formats/src/matbin.rs deleted file mode 100644 index 51c9475..0000000 --- a/crates/formats/src/matbin.rs +++ /dev/null @@ -1,141 +0,0 @@ -use std::io::{self, SeekFrom}; - -use byteorder::{ReadBytesExt, LE}; - -use crate::io_ext::ReadFormatsExt; - -#[derive(Debug)] -pub enum MatbinError { - IO(io::Error), -} - -#[derive(Debug)] -pub struct Matbin { - pub unk04: u32, - pub shader_path: String, - pub source_path: String, - pub key: u32, - pub param_count: u32, - pub sampler_count: u32, - pub params: Vec, - pub samplers: Vec, -} - -impl Matbin { - pub fn from_reader(r: &mut (impl io::Read + io::Seek)) -> Result { - let _magic = r.read_u32::()?; - // assert!(magic == 0x42414d, "Matbin was not of expected format"); - - let unk04 = r.read_u32::()?; - let shader_path_offset = r.read_u64::()?; - let source_path_offset = r.read_u64::()?; - let key = r.read_u32::()?; - let param_count = r.read_u32::()?; - let sampler_count = r.read_u32::()?; - - let current_pos = r.stream_position()?; - r.seek(SeekFrom::Start(shader_path_offset))?; - let shader_path = r.read_utf16::()?; - r.seek(SeekFrom::Start(source_path_offset))?; - let source_path = r.read_utf16::()?; - r.seek(SeekFrom::Start(current_pos))?; - - assert!(r.read_u64::()? == 0x0); - assert!(r.read_u64::()? == 0x0); - assert!(r.read_u32::()? == 0x0); - - let mut params = vec![]; - for _ in 0..param_count { - params.push(MatbinParam::from_reader(r)?); - } - - let mut samplers = vec![]; - for _ in 0..sampler_count { - samplers.push(MatbinSampler::from_reader(r)?); - } - - Ok(Self { - unk04, - shader_path, - source_path, - key, - param_count, - sampler_count, - params, - samplers, - }) - } -} - -#[derive(Debug)] -pub struct MatbinParam { - pub name: String, - pub value: u32, - pub key: u32, - pub value_type: u32, -} - -impl MatbinParam { - pub fn from_reader(r: &mut (impl io::Read + io::Seek)) -> Result { - let name_offset = r.read_u64::()?; - - // TODO: read values - let _value_offset = r.read_u64::()?; - let key = r.read_u32::()?; - let value_type = r.read_u32::()?; - - assert!(r.read_u64::()? == 0x0); - assert!(r.read_u64::()? == 0x0); - - let current_pos = r.stream_position()?; - r.seek(SeekFrom::Start(name_offset))?; - let name = r.read_utf16::()?; - r.seek(SeekFrom::Start(current_pos))?; - - Ok(Self { - name, - value: 0x0, - key, - value_type, - }) - } -} - -#[derive(Debug)] -pub struct MatbinSampler { - pub sampler_type: String, - pub path: String, - pub key: u32, - pub unkx: f32, - pub unky: f32, -} - -impl MatbinSampler { - pub fn from_reader(r: &mut (impl io::Read + io::Seek)) -> Result { - let type_offset = r.read_u64::()?; - let path_offset = r.read_u64::()?; - let key = r.read_u32::()?; - - let unkx = r.read_f32::()?; - let unky = r.read_f32::()?; - - assert!(r.read_u64::()? == 0x0); - assert!(r.read_u64::()? == 0x0); - assert!(r.read_u32::()? == 0x0); - - let current_pos = r.stream_position()?; - r.seek(SeekFrom::Start(type_offset))?; - let sampler_type = r.read_utf16::()?; - r.seek(SeekFrom::Start(path_offset))?; - let path = r.read_utf16::()?; - r.seek(SeekFrom::Start(current_pos))?; - - Ok(Self { - sampler_type, - path, - key, - unkx, - unky, - }) - } -} diff --git a/crates/formats/src/matbin/mod.rs b/crates/formats/src/matbin/mod.rs new file mode 100644 index 0000000..8e31b77 --- /dev/null +++ b/crates/formats/src/matbin/mod.rs @@ -0,0 +1,318 @@ +use std::{borrow::Cow, io}; + +use bytemuck::PodCastError; +use byteorder::LE; +use thiserror::Error; +use widestring::{U16Str, U16String}; +use zerocopy::{FromBytes, FromZeroes, Ref, F32, U32, U64}; + +use crate::io_ext::zerocopy::Padding; + +#[derive(Debug, Error)] +pub enum MatbinError { + #[error("Could not copy bytes {0}")] + Io(#[from] io::Error), + + #[error("Could not read string")] + String(#[from] ReadUtf16StringError), + + #[error("Got unknown parameter type {0}")] + UnknownParameterType(u32), + + #[error("Could not create reference to value")] + UnalignedValue, +} + +// Defines a material for instancing in FLVERs and such. +// It does so by pointing at a shader and specifying the parameter/sampler +// setup. +#[allow(unused)] +pub struct Matbin<'a> { + bytes: &'a [u8], + + header: &'a Header, + + parameters: &'a [Parameter], + + samplers: &'a [Sampler], +} + +impl<'a> Matbin<'a> { + pub fn parse(bytes: &'a [u8]) -> Option { + let (header, next) = Ref::<_, Header>::new_from_prefix(bytes)?; + let (parameters, next) = + Parameter::slice_from_prefix(next, header.parameter_count.get() as usize)?; + + let (samplers, _) = Sampler::slice_from_prefix(next, header.sampler_count.get() as usize)?; + + Some(Self { + bytes, + header: header.into_ref(), + parameters, + samplers, + }) + } + + pub fn shader_path(&self) -> Result, MatbinError> { + let offset = self.header.shader_path_offset.get() as usize; + let bytes = &self.bytes[offset..]; + + Ok(read_utf16_string(bytes)?) + } + + pub fn source_path(&self) -> Result, MatbinError> { + let offset = self.header.source_path_offset.get() as usize; + let bytes = &self.bytes[offset..]; + + Ok(read_utf16_string(bytes)?) + } + + pub fn samplers(&self) -> impl Iterator> { + self.samplers.iter().map(|e| { + let name = { + let offset = e.name_offset.get() as usize; + let bytes = &self.bytes[offset..]; + read_utf16_string(bytes) + }?; + + let path = { + let offset = e.path_offset.get() as usize; + let bytes = &self.bytes[offset..]; + read_utf16_string(bytes) + }?; + + Ok(SamplerIterElement { name, path }) + }) + } + + pub fn parameters(&self) -> impl Iterator> { + self.parameters.iter().map(|e| { + let name = { + let offset = e.name_offset.get() as usize; + let bytes = &self.bytes[offset..]; + read_utf16_string(bytes) + }?; + + let value_slice = &self.bytes[e.value_offset.get() as usize..]; + let value = ParameterValue::from_type_and_slice(e.value_type.get(), value_slice)?; + + Ok(ParameterIterElement { name, value }) + }) + } +} + +impl<'a> std::fmt::Debug for Matbin<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Matbin") + .field("shader_path", &self.shader_path()) + .field("source_path", &self.source_path()) + .field("header", self.header) + .field("parameters", &self.parameters) + .field("samplers", &self.samplers) + .finish() + } +} + +pub struct ParameterIterElement<'a> { + pub name: Cow<'a, U16Str>, + pub value: ParameterValue<'a>, +} + +pub struct SamplerIterElement<'a> { + pub name: Cow<'a, U16Str>, + pub path: Cow<'a, U16Str>, +} + +pub enum ParameterValue<'a> { + Bool(bool), + Int(&'a U32), + IntVec2(&'a [U32]), + Float(&'a F32), + FloatVec2(&'a [F32]), + FloatVec3(&'a [F32]), + FloatVec4(&'a [F32]), + FloatVec5(&'a [F32]), +} + +impl<'a> ParameterValue<'a> { + pub fn from_type_and_slice( + value_type: u32, + value_slice: &'a [u8], + ) -> Result { + Ok(match value_type { + 0x0 => ParameterValue::Bool(value_slice[0] != 0x0), + 0x4 => ParameterValue::Int( + U32::::ref_from_prefix(value_slice).ok_or(MatbinError::UnalignedValue)?, + ), + 0x5 => ParameterValue::IntVec2( + U32::::slice_from_prefix(value_slice, 2) + .ok_or(MatbinError::UnalignedValue)? + .0, + ), + 0x8 => ParameterValue::Float( + F32::::ref_from_prefix(value_slice).ok_or(MatbinError::UnalignedValue)?, + ), + 0x9 => ParameterValue::FloatVec2( + F32::::slice_from_prefix(value_slice, 2) + .ok_or(MatbinError::UnalignedValue)? + .0, + ), + 0xA => ParameterValue::FloatVec3( + F32::::slice_from_prefix(value_slice, 3) + .ok_or(MatbinError::UnalignedValue)? + .0, + ), + 0xB => ParameterValue::FloatVec4( + F32::::slice_from_prefix(value_slice, 4) + .ok_or(MatbinError::UnalignedValue)? + .0, + ), + 0xC => ParameterValue::FloatVec5( + F32::::slice_from_prefix(value_slice, 5) + .ok_or(MatbinError::UnalignedValue)? + .0, + ), + _ => return Err(MatbinError::UnknownParameterType(value_type)), + }) + } +} + +impl<'a> std::fmt::Debug for ParameterValue<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&match self { + ParameterValue::Bool(v) => format!("Bool({})", v), + ParameterValue::Int(v) => format!("Int({})", v.get()), + ParameterValue::IntVec2(v) => format!("IntVec2([{}, {}])", v[0].get(), v[1].get(),), + ParameterValue::Float(v) => format!("Float({})", v.get()), + ParameterValue::FloatVec2(v) => format!("FloatVec2([{}, {}])", v[0].get(), v[1].get(),), + ParameterValue::FloatVec3(v) => format!( + "FloatVec3([{}, {}, {}])", + v[0].get(), + v[1].get(), + v[2].get(), + ), + ParameterValue::FloatVec4(v) => format!( + "FloatVec4([{}, {}, {}, {}])", + v[0].get(), + v[1].get(), + v[2].get(), + v[3].get(), + ), + ParameterValue::FloatVec5(v) => format!( + "FloatVec5([{}, {}, {}, {}, {}])", + v[0].get(), + v[1].get(), + v[2].get(), + v[3].get(), + v[4].get(), + ), + }) + } +} + +#[derive(FromZeroes, FromBytes, Debug)] +#[repr(packed)] +#[allow(unused)] +pub struct Header { + chunk_magic: [u8; 4], + + // Seems to be 2? Might be some version number. Couldn't easily find the + // parser with Ghidra so :shrug:. + unk04: U32, + + /// Offset to the shader path + shader_path_offset: U64, + + /// Offset to the source path as a wstring. Seems to reference the source + /// for the current matbin file. + source_path_offset: U64, + + /// Adler32 hash of the source path string without the string terminator + source_path_hash: U32, + + /// Amount of parameters for this material + parameter_count: U32, + + /// Amount of samples for this material + sampler_count: U32, + + _padding24: Padding<20>, +} + +#[derive(FromZeroes, FromBytes, Debug)] +#[repr(packed)] +#[allow(unused)] +pub struct Parameter { + /// Offset to name of the parameter + name_offset: U64, + + /// Offset to value of the parameter + value_offset: U64, + + /// Adler32 hash of the name string without the string terminator + name_hash: U32, + + /// Type of the value pointed at by value_offset + value_type: U32, + + _padding18: Padding<16>, +} + +#[derive(FromZeroes, FromBytes, Debug)] +#[repr(packed)] +#[allow(unused)] +pub struct Sampler { + /// Offset to the samplers name + name_offset: U64, + + /// Offset to the samplers path + path_offset: U64, + + /// Adler32 hash of the name string without the string terminator + name_hash: U32, + + /// ??? + unkxy: [F32; 2], + + _padding1c: Padding<20>, +} + +#[derive(Debug, Error)] +pub enum ReadUtf16StringError { + #[error("Could not find end of string")] + NoEndFound, + + #[error("Bytemuck could not cast pod")] + Bytemuck, +} + +fn read_utf16_string(input: &[u8]) -> Result, ReadUtf16StringError> { + // Find the end of the input string + let length = input + .chunks_exact(2) + .position(|bytes| bytes[0] == 0x0 && bytes[1] == 0x0) + .ok_or(ReadUtf16StringError::NoEndFound)?; + + // Create a view that has a proper end so we don't copy + // the entire input slice if required and so we don't have to deal with + // bytemuck freaking out over the end not being aligned + let string_bytes = &input[..length * 2]; + + Ok(match bytemuck::try_cast_slice::(string_bytes) { + Ok(s) => Cow::Borrowed(U16Str::from_slice(s)), + Err(e) => { + // We should probably return the error if it isn't strictly + // about the alignment of the input. + if e != PodCastError::TargetAlignmentGreaterAndInputNotAligned { + return Err(ReadUtf16StringError::Bytemuck); + } + + let aligned_copy = string_bytes + .chunks(2) + .map(|a| u16::from_le_bytes([a[0], a[1]])) + .collect::>(); + + Cow::Owned(U16String::from_vec(aligned_copy)) + } + }) +}