Skip to content

Commit

Permalink
feat(show): 添加完整显示指定元数据功能,并支持不显示张量信息
Browse files Browse the repository at this point in the history
Signed-off-by: YdrMaster <ydrml@hotmail.com>
  • Loading branch information
YdrMaster committed Jul 31, 2024
1 parent c9ad1a4 commit a5674ef
Show file tree
Hide file tree
Showing 4 changed files with 86 additions and 31 deletions.
1 change: 1 addition & 0 deletions .cargo/config.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,6 @@
xtask = "run --package xtask --release --"
debug = "run --package xtask --"
show = "xtask show"
show-meta = "xtask show-meta"
split = "xtask split"
merge = "xtask merge"
48 changes: 24 additions & 24 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions xtask/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ fn main() {
use Commands::*;
match Cli::parse().command {
Show(args) => args.show(),
ShowMeta(args) => args.show(),
Split(args) => args.split(),
Merge(args) => args.merge(),
}
Expand All @@ -29,6 +30,7 @@ struct Cli {
#[derive(Subcommand)]
enum Commands {
Show(show::ShowArgs),
ShowMeta(show::ShowMetaArgs),
Split(split::SplitArgs),
Merge(merge::MergeArgs),
}
Expand Down
66 changes: 59 additions & 7 deletions xtask/src/show.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ pub struct ShowArgs {
/// If set, show all shards in the directory
#[clap(long)]
shards: bool,
/// If set, do not show tensor info
#[clap(long)]
no_tensor: bool,
}

struct Failed;
Expand Down Expand Up @@ -64,7 +67,7 @@ impl ShowArgs {
continue;
}
};
if let Err(Failed) = show_meta_kvs(&kvs) {
if show_meta_kvs(&kvs).is_err() || self.no_tensor {
println!();
continue;
}
Expand Down Expand Up @@ -146,7 +149,7 @@ fn show_meta_kv(kv: GGufMetaKV, width: usize) -> Result<(), Failed> {
let ty = kv.ty();
let mut reader = kv.value_reader();
let mut buf = String::new();
match fmt_meta_val(&mut reader, ty, 1, &mut buf) {
match fmt_meta_val(&mut reader, ty, 1, 8, &mut buf) {
Ok(()) => {
println!("{YES}{key:·<width$} {buf}");
Ok(())
Expand All @@ -162,6 +165,7 @@ fn fmt_meta_val<'a>(
reader: &mut GGufReader<'a>,
ty: GGufMetaDataValueType,
len: usize,
ele_arr_to_show: usize,
buf: &mut String,
) -> Result<(), GGufReadError<'a>> {
struct MultiLines<'a>(&'a str);
Expand Down Expand Up @@ -212,24 +216,24 @@ fn fmt_meta_val<'a>(
}
T::Array => {
let (ty, len) = reader.read_arr_header()?;
fmt_meta_val(reader, ty, len, buf)?;
fmt_meta_val(reader, ty, len, ele_arr_to_show, buf)?;
}
}
}
_ if len <= 8 => {
_ if len <= ele_arr_to_show => {
buf.push('[');
for i in 0..len {
if i > 0 {
buf.push_str(", ");
}
fmt_meta_val(reader, ty, 1, buf)?;
fmt_meta_val(reader, ty, 1, ele_arr_to_show, buf)?;
}
buf.push(']');
}
_ => {
buf.push('[');
for _ in 0..8 {
fmt_meta_val(reader, ty, 1, buf)?;
for _ in 0..ele_arr_to_show {
fmt_meta_val(reader, ty, 1, ele_arr_to_show, buf)?;
buf.push_str(", ");
}
buf.push_str(&format!("...({} more)]", len - 8));
Expand All @@ -256,3 +260,51 @@ fn show_tensors(tensors: &GGufTensors) -> Result<(), Failed> {

Ok(())
}

#[derive(Args, Default)]
pub struct ShowMetaArgs {
/// The file to show
file: PathBuf,
/// Meta data key to show
key: String,
}

impl ShowMetaArgs {
pub fn show(self) {
let Self { file, key } = self;

let file = match File::open(&file) {
Ok(file) => file,
Err(e) => {
eprintln!("Failed to open");
eprintln!(" file: {}", file.display());
eprintln!(" cause: {e}");
return;
}
};
let file = unsafe { memmap2::Mmap::map(&file) }.unwrap();
let header = unsafe { file.as_ptr().cast::<GGufFileHeader>().read() };
if let Err(Failed) = show_header(&header) {
return;
}

let cursor = header.nbytes();
let kvs = match GGufMetaKVPairs::scan(header.metadata_kv_count, &file[cursor..]) {
Ok(kvs) => kvs,
Err(e) => {
eprintln!("{ERR}{e:?}");
return;
}
};

let Some(kv) = kvs.get(&key) else {
eprintln!("{ERR}Key \"{key}\" not found");
return;
};
let mut buf = String::new();
match fmt_meta_val(&mut kv.value_reader(), kv.ty(), 1, usize::MAX, &mut buf) {
Ok(()) => println!("{YES}{key}: {buf}"),
Err(e) => eprintln!("{ERR}{key} {e:?}"),
}
}
}

0 comments on commit a5674ef

Please sign in to comment.