Skip to content

Commit

Permalink
fix(xtask): 拷贝 vocabs.txt
Browse files Browse the repository at this point in the history
Signed-off-by: YdrMaster <[email protected]>
  • Loading branch information
YdrMaster committed Feb 29, 2024
1 parent b959567 commit fa62f1a
Showing 1 changed file with 12 additions and 7 deletions.
19 changes: 12 additions & 7 deletions xtask/src/cast.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use std::{fs, path::PathBuf, time::Instant};
use tensor::DataType;
use transformer_cpu::model_parameters::{save, Llama2, Memory};
use transformer_cpu::model_parameters::{save, Memory};

#[derive(Args, Default)]
pub(crate) struct CastArgs {
Expand Down Expand Up @@ -45,11 +45,16 @@ impl CastArgs {
save(&model, &target).unwrap();
println!("save model ... {:?}", time.elapsed());

let tokenizer = model_dir.join("tokenizer.model");
if tokenizer.is_file() {
let time = Instant::now();
fs::copy(&tokenizer, target.join("tokenizer.model")).unwrap();
println!("copy tokenizer ... {:?}", time.elapsed());
}
let copy_file = |name: &str| {
let src = model_dir.join(name);
if src.is_file() {
let time = Instant::now();
fs::copy(&src, target.join(name)).unwrap();
println!("copy {name} ... {:?}", time.elapsed());
}
};

copy_file("tokenizer.model");
copy_file("vocabs.txt");
}
}

0 comments on commit fa62f1a

Please sign in to comment.