From ab1217d4334752200812eab634006095f1f97040 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B3=B0=E5=8F=8B?= Date: Tue, 7 Jun 2022 18:59:33 +0800 Subject: [PATCH 1/7] feat: decompress ut MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 泰友 --- src/bin/nydus-image/decompress.rs | 895 +++++++++++++++++++++++++ src/bin/nydus-image/decompress/test.rs | 369 ++++++++++ src/bin/nydus-image/main.rs | 50 ++ 3 files changed, 1314 insertions(+) create mode 100644 src/bin/nydus-image/decompress.rs create mode 100644 src/bin/nydus-image/decompress/test.rs diff --git a/src/bin/nydus-image/decompress.rs b/src/bin/nydus-image/decompress.rs new file mode 100644 index 00000000000..6b75c1ca445 --- /dev/null +++ b/src/bin/nydus-image/decompress.rs @@ -0,0 +1,895 @@ +extern crate tar; + +use std::{ + borrow::Borrow, + collections::HashMap, + ffi::OsStr, + fs::{File, OpenOptions}, + io::{self, Cursor, Error, ErrorKind, Read}, + iter::from_fn, + os::unix::prelude::{OsStrExt, OsStringExt}, + path::{Path, PathBuf}, + str, + sync::Arc, + vec::IntoIter, +}; + +use anyhow::{Context, Result}; +use nydus_rafs::{ + metadata::{layout::RAFS_ROOT_INODE, RafsInode, RafsMode, RafsSuper}, + RafsIoReader, +}; +use nydus_utils::compress::{self, Algorithm}; +use storage::{ + backend::{localfs::LocalFs, BlobBackend, BlobReader, LocalFsConfig}, + device::{BlobChunkInfo, BlobInfo}, + utils::alloc_buf, +}; +use tar::{Builder, EntryType, Header}; + +use crate::core::node::{InodeWrapper, OCISPEC_WHITEOUT_PREFIX, ROOT_PATH_NAME}; + +static PAX_SEP1: &'static [u8; 1] = b" "; +static PAX_SEP2: &'static [u8; 1] = b"="; +static PAX_PREFIX: &'static [u8; 13] = b"SCHILY.xattr."; +static PAX_DELIMITER: &'static [u8; 1] = b"\n"; + +pub trait Decompressor { + fn decompress(&self) -> Result<()>; +} + +/// A decompressor with the ability to convert bootstrap file and blob file to tar +pub struct DefaultDecompressor { + bootstrap_path: Box, + blob_path: Box, + output_path: Box, +} + +impl DefaultDecompressor { + pub fn new(bootstrap: &str, blob: &str, output: &str) -> Result { + let bootstrap_path = PathBuf::from(bootstrap) + .canonicalize() + .with_context(|| format!("fail to parse bootstrap {}", bootstrap))? + .into_boxed_path(); + + let blob_path = PathBuf::from(blob) + .canonicalize() + .with_context(|| format!("fail to parse blob {}", blob))? + .into_boxed_path(); + + let output_path = PathBuf::from(output) + .canonicalize() + .with_context(|| format!("fail to parse output {}", output))? + .into_boxed_path(); + + Ok(DefaultDecompressor { + bootstrap_path, + blob_path, + output_path, + }) + } + + fn load_meta(&self) -> Result { + let bootstrap = OpenOptions::new() + .read(true) + .write(false) + .open(self.bootstrap_path.as_ref()) + .map_err(|err| { + error!( + "fail to load bootstrap {:?}, error: {}", + self.bootstrap_path, err + ); + anyhow!( + "fail to load bootstrap {:?}, error: {}", + self.bootstrap_path, + err + ) + })?; + + let mut rs = RafsSuper { + mode: RafsMode::Direct, + validate_digest: true, + ..Default::default() + }; + rs.load(&mut (Box::new(bootstrap) as RafsIoReader)) + .map_err(|err| { + error!( + "fail to load bootstrap {:?}, error: {}", + self.bootstrap_path, err + ); + anyhow!( + "fail to load bootstrap {:?}, error: {}", + self.bootstrap_path, + err + ) + })?; + + Ok(rs) + } + + /// A lazy iterator of RafsInode in DFS. + fn iterator<'a>( + &'a self, + rs: &'a RafsSuper, + ) -> Box, Box)> + 'a> { + let mut cursor_stack = Vec::with_capacity(32); + cursor_stack.push(self.cursor_of_root(rs)); + + let dfs_cursor = move || { + while !cursor_stack.is_empty() { + let mut cursor = cursor_stack.pop().unwrap(); + let (node, path) = if let Some(iterm) = cursor.next() { + cursor_stack.push(cursor); + iterm + } else { + continue; + }; + + if node.is_dir() { + cursor_stack.push(self.cursor_of_children(node.clone(), path.as_ref())) + } + + return Some((node, path)); + } + + None + }; + + Box::new(from_fn(dfs_cursor)) + } + + fn cursor_of_children( + &self, + node: Arc, + path: &Path, + ) -> Box, Box)>> { + let path = path.to_path_buf(); + let mut cursor = 0..node.get_child_count(); + + let visitor = from_fn(move || { + if cursor.is_empty() { + return None; + } + + let child = node.get_child_by_index(cursor.next().unwrap()).unwrap(); + let child_path = path.join(child.name()).into_boxed_path(); + + Some((child, child_path)) + }); + + Box::new(visitor) + } + + fn cursor_of_root<'a>( + &self, + rs: &'a RafsSuper, + ) -> Box, Box)> + 'a> { + let mut has_more = true; + let visitor = from_fn(move || { + if !has_more { + return None; + } + has_more = false; + + let node = rs.get_inode(RAFS_ROOT_INODE, false).unwrap(); + let path = PathBuf::from("/").join(node.name()).into_boxed_path(); + + Some((node, path)) + }); + + Box::new(visitor) + } +} + +impl Decompressor for DefaultDecompressor { + fn decompress(&self) -> Result<()> { + info!( + "default decompressor, bootstrap file: {:?}, blob file: {:?}, output file: {:?}", + self.bootstrap_path, self.blob_path, self.output_path + ); + + let meta = self.load_meta().with_context(|| "fail to load meta")?; + + let mut builder = Box::new( + DefaultTarBuilder::new(&meta, self.blob_path.as_ref(), self.output_path.as_ref()) + .with_context(|| "fail to create tar builder")?, + ) as Box; + + for (node, path) in self.iterator(&meta) { + // Not write root node to tar. + if node.name() == OsStr::from_bytes(ROOT_PATH_NAME) { + continue; + } + + builder + .append(node.as_ref(), path.as_ref()) + .with_context(|| "fail to append inode to tar")?; + } + + Ok(()) + } +} + +trait TarBuilder { + fn append(&mut self, node: &dyn RafsInode, path: &Path) -> Result<()>; +} + +struct TarSection { + header: Header, + data: Box, + presections: Vec, +} + +struct DefaultTarBuilder { + blob: Option>, + reader: Option>, + writer: Builder, + links: HashMap>, +} + +impl DefaultTarBuilder { + fn new(meta: &RafsSuper, blob_path: &Path, output_path: &Path) -> Result { + let writer = Builder::new( + OpenOptions::new() + .create(true) + .truncate(true) + .read(false) + .open(output_path) + .map_err(|err| { + anyhow!( + "fail to open output file {:?}, error: {:?}", + output_path, + err + ) + })?, + ); + + let blob = meta.superblock.get_blob_infos().pop(); + let reader = if blob.is_some() { + Some(Self::chunk_reader(blob_path)?) + } else { + None + }; + + Ok(Self { + blob, + reader, + writer, + links: HashMap::new(), + }) + } + + fn chunk_reader(blob_path: &Path) -> Result> { + let config = LocalFsConfig { + blob_file: blob_path.to_str().unwrap().to_owned(), + readahead: false, + readahead_sec: Default::default(), + dir: Default::default(), + alt_dirs: Default::default(), + }; + let config = serde_json::to_value(config).map_err(|err| { + anyhow!( + "fail to create local backend config for {:?}, error: {:?}", + blob_path, + err + ) + })?; + + let backend = LocalFs::new(config, Some("decompressor")).map_err(|err| { + anyhow!( + "fail to create local backend for {:?}, error: {:?}", + blob_path, + err + ) + })?; + + let reader = backend.get_reader("").map_err(|err| { + anyhow!( + "fail to get chunk reader for {:?}, error: {:?}", + blob_path, + err + ) + })?; + + Ok(reader) + } + + fn build_whiteout_section(&self, inode: &dyn RafsInode, path: &Path) -> Result { + // In order to save access time and change time, gnu layout is required. + let mut header = Header::new_gnu(); + + header.set_entry_type(EntryType::file()); + header.set_device_major(0).unwrap(); + header.set_device_minor(0).unwrap(); + header.set_size(0); + + let node = InodeWrapper::from_inode_info(inode); + header.set_mtime(node.mtime()); + header.set_uid(node.uid() as u64); + header.set_gid(node.gid() as u64); + header.set_mode(node.mode()); + + // Rafs loses the access time and change time. + // It's required by certain tools, such 7-zip. + // Fill modify time as access time and change time. + header.as_gnu_mut().unwrap().set_atime(node.mtime()); + header.as_gnu_mut().unwrap().set_ctime(node.mtime()); + + let mut requirements = Vec::with_capacity(1); + if let Some(requirement) = self.set_path(&mut header, path)? { + requirements.push(requirement); + } + + header.set_cksum(); + + Ok(TarSection { + header, + data: Box::new(io::empty()), + presections: Vec::new(), + }) + } + + fn build_dir_section(&self, inode: &dyn RafsInode, path: &Path) -> Result { + let mut header = Header::new_ustar(); + + header.set_entry_type(EntryType::dir()); + header.set_size(0); + header.set_device_major(0).unwrap(); + header.set_device_minor(0).unwrap(); + + let node = InodeWrapper::from_inode_info(inode); + header.set_mtime(node.mtime()); + header.set_uid(node.uid() as u64); + header.set_gid(node.gid() as u64); + header.set_mode(node.mode()); + + self.set_path(&mut header, path)?; + + header.set_cksum(); + + let mut requirements = Vec::with_capacity(1); + if let Some(xattr) = self.build_xattr_section(inode, header.path().unwrap().borrow())? { + requirements.push(xattr); + } + + Ok(TarSection { + header, + data: Box::new(io::empty()), + presections: requirements, + }) + } + + fn build_reg_section(&self, inode: &dyn RafsInode, path: &Path) -> Result { + if self.blob.is_none() || self.reader.is_none() { + bail!("miss blob meta or chunk reader for building regular header") + } + + let chunks: Vec> = (0..inode.get_chunk_count()) + .map(|i| inode.get_chunk_info(i).unwrap()) + .collect(); + let data = ChunkReader::new( + self.blob.as_ref().unwrap().compressor(), + self.reader.as_ref().unwrap().clone(), + chunks, + ); + + let mut header = Header::new_ustar(); + + header.set_entry_type(EntryType::file()); + header.set_device_major(0).unwrap(); + header.set_device_minor(0).unwrap(); + + let node = InodeWrapper::from_inode_info(inode); + header.set_mtime(node.mtime()); + header.set_uid(node.uid() as u64); + header.set_gid(node.gid() as u64); + header.set_mode(node.mode()); + header.set_size(node.size()); + + self.set_path(&mut header, path)?; + + header.set_cksum(); + + let mut requirements = Vec::with_capacity(1); + if let Some(xattr) = self.build_xattr_section(inode, header.path().unwrap().borrow())? { + requirements.push(xattr); + } + + Ok(TarSection { + header, + data: Box::new(data), + presections: requirements, + }) + } + + fn build_symlink_section(&self, node: &dyn RafsInode, path: &Path) -> Result { + let link = node.get_symlink().unwrap(); + + self.build_link_section(EntryType::symlink(), node, path, &PathBuf::from(link)) + } + + fn build_hard_link_section(&self, node: &dyn RafsInode, path: &Path) -> Result { + let link = self.links.get(&node.ino()).unwrap(); + + self.build_link_section(EntryType::hard_link(), node, path, link) + } + + fn build_fifo_section(&self, node: &dyn RafsInode, path: &Path) -> Result { + self.build_special_file_section(EntryType::Fifo, node, path) + } + + fn build_char_section(&self, node: &dyn RafsInode, path: &Path) -> Result { + self.build_special_file_section(EntryType::character_special(), node, path) + } + + fn build_block_section(&self, node: &dyn RafsInode, path: &Path) -> Result { + self.build_special_file_section(EntryType::block_special(), node, path) + } + + fn build_link_section( + &self, + entry_type: EntryType, + inode: &dyn RafsInode, + path: &Path, + link: &Path, + ) -> Result { + let mut header = Header::new_ustar(); + + header.set_entry_type(entry_type); + header.set_size(0); + header.set_device_major(0).unwrap(); + header.set_device_minor(0).unwrap(); + + let node = InodeWrapper::from_inode_info(inode); + header.set_mtime(node.mtime()); + header.set_uid(node.uid() as u64); + header.set_gid(node.gid() as u64); + header.set_mode(node.mode()); + + self.set_path(&mut header, path)?; + self.set_link(&mut header, link)?; + + header.set_cksum(); + + let mut requirements = Vec::with_capacity(1); + if let Some(xattr) = self.build_xattr_section(inode, header.path().unwrap().borrow())? { + requirements.push(xattr); + } + + Ok(TarSection { + header, + data: Box::new(io::empty()), + presections: requirements, + }) + } + + fn build_special_file_section( + &self, + entry_type: EntryType, + inode: &dyn RafsInode, + path: &Path, + ) -> Result { + let mut header = Header::new_ustar(); + + header.set_entry_type(entry_type); + + let node = InodeWrapper::from_inode_info(inode); + header.set_mtime(node.mtime()); + header.set_uid(node.uid() as u64); + header.set_gid(node.gid() as u64); + header.set_mode(node.mode()); + header.set_size(node.size()); + + let dev_id = inode.rdev() as u64; + let dev_major = ((dev_id >> 32) & 0xffff_f000) | ((dev_id >> 8) & 0x0000_0fff); + header.set_device_major(dev_major as u32)?; + + let dev_minor = ((dev_id >> 12) & 0xffff_ff00) | ((dev_id) & 0x0000_00ff); + header.set_device_minor(dev_minor as u32)?; + + self.set_path(&mut header, path)?; + + header.set_cksum(); + + let mut requirements = Vec::with_capacity(1); + if let Some(xattr) = self.build_xattr_section(inode, header.path().unwrap().borrow())? { + requirements.push(xattr); + } + + Ok(TarSection { + header, + data: Box::new(io::empty()), + presections: requirements, + }) + } + + fn build_xattr_section( + &self, + inode: &dyn RafsInode, + header_name: &Path, + ) -> Result> { + fn normalize_path(path: &Path) -> PathBuf { + let mut name = PathBuf::new(); + name.push(path.parent().unwrap()); + name.push(AsRef::::as_ref("PaxHeaders.0")); + name.push(path.file_name().unwrap()); + name + } + + let (data, size) = if let Some(rs) = self.build_pax_data(inode) { + rs + } else { + return Ok(None); + }; + + let mut header = Header::new_ustar(); + + header.set_entry_type(EntryType::XHeader); + header.set_mode(0o644); + header.set_uid(0); + header.set_gid(0); + header.set_mtime(0); + header.set_size(size as u64); + + self.set_path(&mut header, &normalize_path(header_name))?; + + header.set_cksum(); + + Ok(Some(TarSection { + header, + data: Box::new(data), + presections: Vec::new(), + })) + } + + fn build_long_section(&self, entry_type: EntryType, mut data: Vec) -> TarSection { + // gnu requires, especially for long-link section and long-name section + let mut header = Header::new_gnu(); + + data.push(0x00); + header.set_size(data.len() as u64); + let data = Cursor::new(data); + + let title = b"././@LongLink"; + header.as_gnu_mut().unwrap().name[..title.len()].clone_from_slice(&title[..]); + + header.set_entry_type(entry_type); + header.set_mode(0o644); + header.set_uid(0); + header.set_gid(0); + header.set_mtime(0); + + header.set_cksum(); + + TarSection { + header, + data: Box::new(data), + presections: Vec::new(), + } + } + + fn build_pax_data(&self, inode: &dyn RafsInode) -> Option<(impl Read, usize)> { + if !inode.has_xattr() { + return None; + } + + // Seek for a stable order + let mut keys = inode.get_xattrs().unwrap(); + keys.sort_by(|k1, k2| str::from_utf8(k1).unwrap().cmp(str::from_utf8(k2).unwrap())); + + let mut data = Vec::new(); + for key in keys { + let value = inode + .get_xattr(OsStr::from_bytes(&key)) + .unwrap() + .unwrap_or_default(); + + data.append(&mut self.build_pax_record(&key, &value)); + } + + let len = data.len(); + Some((Cursor::new(data), len)) + } + + fn build_pax_record(&self, k: &[u8], v: &[u8]) -> Vec { + fn pax(buf: &mut Vec, size: usize, k: &[u8], v: &[u8]) { + buf.extend_from_slice(size.to_string().as_bytes()); + buf.extend_from_slice(PAX_SEP1); + buf.extend_from_slice(PAX_PREFIX); + buf.extend_from_slice(k); + buf.extend_from_slice(PAX_SEP2); + buf.extend_from_slice(v); + buf.extend_from_slice(PAX_DELIMITER); + } + + let mut size = k.len() + + v.len() + + PAX_SEP1.len() + + PAX_SEP2.len() + + PAX_DELIMITER.len() + + PAX_PREFIX.len(); + size += size.to_string().as_bytes().len(); + + let mut record = Vec::with_capacity(size); + pax(&mut record, size, k, v); + + if record.len() != size { + size = record.len(); + record.clear(); + pax(&mut record, size, k, v); + } + + record + } + + /// If path size is too long, a long-name-header is returned as precondition of input header. + fn set_path(&self, header: &mut Header, path: &Path) -> Result> { + fn try_best_to_str(bs: &[u8]) -> Result<&str> { + match str::from_utf8(bs) { + Ok(s) => Ok(s), + Err(err) => str::from_utf8(&bs[..err.valid_up_to()]) + .map_err(|err| anyhow!("fail to convert bytes to utf8 str, error: {}", err)), + } + } + + let normalized = self + .normalize_path(path, header.entry_type().is_dir()) + .map_err(|err| anyhow!("fail to normalize path, error {}", err))?; + + let rs = header + .set_path(&normalized) + .map(|_| None) + .map_err(|err| anyhow!("fail to set path for header, error {}", err)); + if rs.is_ok() || header.as_ustar().is_some() { + return rs; + } + + let data = normalized.into_os_string().into_vec(); + let max = header.as_old().name.len(); + if data.len() < max { + return rs; + } + + // try to set truncated path to header + header + .set_path( + try_best_to_str(&data.as_slice()[..max]) + .with_context(|| "fail to truncate path")?, + ) + .map_err(|err| { + anyhow!( + "fail to set header path again for {:?}, error {}", + path, + err + ) + })?; + + Ok(Some(self.build_long_section(EntryType::GNULongName, data))) + } + + /// If link size is too long, a long-link-header is returned as precondition of input + /// header. + fn set_link(&self, header: &mut Header, path: &Path) -> Result> { + let normalized = self + .normalize_path(path, header.entry_type().is_dir()) + .map_err(|err| anyhow!("fail to normalize path, error {}", err))?; + + let rs = header + .set_link_name(normalized) + .map(|_| None) + .map_err(|err| anyhow!("fail to set linkname for header, error {}", err)); + if rs.is_ok() || header.as_ustar().is_some() { + return rs; + } + + let data = path.to_path_buf().into_os_string().into_vec(); + let max = header.as_old().linkname.len(); + if data.len() < max { + return rs; + } + + Ok(Some(self.build_long_section(EntryType::GNULongLink, data))) + } + + fn normalize_path(&self, path: &Path, is_dir: bool) -> Result { + fn has_trailing_slash(p: &Path) -> bool { + p.as_os_str().as_bytes().last() == Some(&b'/') + } + + // remove root + let mut normalized = if path.has_root() { + path.strip_prefix("/") + .map_err(|err| anyhow!("fail to strip prefix /, error {}", err))? + .to_path_buf() + } else { + path.to_path_buf() + }; + + // handle trailing slash + if is_dir { + normalized.push(""); + } else if has_trailing_slash(path) { + normalized.set_file_name(normalized.file_name().unwrap().to_owned()) + } + + Ok(normalized) + } + + fn is_hard_link_section(&mut self, node: &dyn RafsInode, path: &Path) -> bool { + let new_face = node.is_hardlink() && !self.links.contains_key(&node.ino()); + + if new_face { + self.links + .insert(node.ino(), path.to_path_buf().into_boxed_path()); + } + + !new_face + } + + fn is_symlink_section(&self, node: &dyn RafsInode) -> bool { + node.is_symlink() + } + + fn is_socket_section(&self, node: &dyn RafsInode) -> bool { + InodeWrapper::from_inode_info(node).is_sock() + } + + fn is_dir_section(&self, node: &dyn RafsInode) -> bool { + node.is_dir() + } + + fn is_reg_section(&self, node: &dyn RafsInode) -> bool { + node.is_reg() + } + + fn is_fifo_section(&self, node: &dyn RafsInode) -> bool { + InodeWrapper::from_inode_info(node).is_fifo() + } + + fn is_char_section(&self, node: &dyn RafsInode) -> bool { + InodeWrapper::from_inode_info(node).is_chrdev() + } + + fn is_block_section(&self, node: &dyn RafsInode) -> bool { + InodeWrapper::from_inode_info(node).is_blkdev() + } + + fn is_white_out_section(&self, path: &Path) -> bool { + path.file_name() + .unwrap() + .to_str() + .unwrap() + .starts_with(OCISPEC_WHITEOUT_PREFIX) + } +} + +impl TarBuilder for DefaultTarBuilder { + fn append(&mut self, inode: &dyn RafsInode, path: &Path) -> Result<()> { + let tar_sect = match inode { + // Ignore socket file + node if self.is_socket_section(node) => return Ok(()), + + node if self.is_hard_link_section(node, path) => { + self.build_hard_link_section(node, path)? + } + + node if self.is_white_out_section(path) => self.build_whiteout_section(node, path)?, + + node if self.is_dir_section(node) => self.build_dir_section(node, path)?, + + node if self.is_reg_section(node) => self.build_reg_section(node, path)?, + + node if self.is_symlink_section(node) => self.build_symlink_section(node, path)?, + + node if self.is_fifo_section(node) => self.build_fifo_section(node, path)?, + + node if self.is_char_section(node) => self.build_char_section(node, path)?, + + node if self.is_block_section(node) => self.build_block_section(node, path)?, + + _ => bail!("unknow inde type"), + }; + + for sect in tar_sect.presections { + self.writer + .append(§.header, sect.data) + .map_err(|err| anyhow!("fail to append inode {:?}, error: {}", path, err))?; + } + + self.writer + .append(&tar_sect.header, tar_sect.data) + .map_err(|err| anyhow!("fail to append inode {:?}, error: {}", path, err))?; + + Ok(()) + } +} + +struct ChunkReader { + compressor: Algorithm, + reader: Arc, + + chunks: IntoIter>, + chunk: Cursor>, +} + +impl ChunkReader { + fn new( + compressor: Algorithm, + reader: Arc, + chunks: Vec>, + ) -> Self { + Self { + compressor, + reader, + chunks: chunks.into_iter(), + chunk: Cursor::new(Vec::new()), + } + } + + fn load_chunk(&mut self, chunk: &dyn BlobChunkInfo) -> Result<()> { + let mut buf = alloc_buf(chunk.compress_size() as usize); + self.reader + .read(buf.as_mut_slice(), chunk.compress_offset()) + .map_err(|err| { + error!("fail to read chunk, error: {:?}", err); + anyhow!("fail to read chunk, error: {:?}", err) + })?; + + if !chunk.is_compressed() { + self.chunk = Cursor::new(buf); + return Ok(()); + } + + let mut data = vec![0u8; chunk.uncompress_size() as usize]; + compress::decompress( + buf.as_mut_slice(), + None, + data.as_mut_slice(), + self.compressor, + ) + .map_err(|err| { + error!("fail to decompress, error: {:?}", err); + anyhow!("fail to decompress, error: {:?}", err) + })?; + + self.chunk = Cursor::new(data); + + Ok(()) + } + + fn is_chunk_empty(&self) -> bool { + self.chunk.position() >= self.chunk.get_ref().len() as u64 + } +} + +impl Read for ChunkReader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let mut size = 0; + + loop { + if self.is_chunk_empty() { + let chunk = self.chunks.next(); + if chunk.is_none() { + break; + } + + self.load_chunk(chunk.unwrap().as_ref()).map_err(|err| { + Error::new( + ErrorKind::InvalidData, + format!("fail to load chunk, error: {}", err), + ) + })?; + } + + size += Read::read(&mut self.chunk, &mut buf[size..])?; + if size == buf.len() { + break; + } + } + + Ok(size) + } +} + +#[cfg(test)] +mod test; diff --git a/src/bin/nydus-image/decompress/test.rs b/src/bin/nydus-image/decompress/test.rs new file mode 100644 index 00000000000..fc5d763fffa --- /dev/null +++ b/src/bin/nydus-image/decompress/test.rs @@ -0,0 +1,369 @@ +use std::{ + collections::HashMap, + ffi::OsString, + fs::OpenOptions, + io::Read, + path::{Path, PathBuf}, + sync::Arc, +}; + +use nydus_utils::{ + compress::{self, Algorithm}, + metrics::BackendMetrics, +}; +use storage::{backend::BlobReader, device::BlobChunkInfo}; +use tar::{Builder, EntryType, Header}; +use vmm_sys_util::tempfile::TempFile; + +use super::{ChunkReader, DefaultTarBuilder}; + +struct MockBlobReader { + data: Vec, + metrics: Arc, +} + +impl MockBlobReader { + fn new(data: Vec) -> Self { + Self { + data, + metrics: Default::default(), + } + } +} + +impl BlobReader for MockBlobReader { + fn try_read(&self, buf: &mut [u8], offset: u64) -> storage::backend::BackendResult { + let offset = offset as usize; + if offset >= self.data.len() { + return Ok(0 as usize); + } + + let end = self.data.len().min(offset as usize + buf.len()); + buf.clone_from_slice(&self.data[offset..end]); + + Ok(end - offset) + } + + fn metrics(&self) -> &BackendMetrics { + self.metrics.as_ref() + } + + fn blob_size(&self) -> storage::backend::BackendResult { + todo!(); + } + + fn prefetch_blob_data_range(&self, _: u64, _: u64) -> storage::backend::BackendResult<()> { + todo!(); + } + + fn stop_data_prefetch(&self) -> storage::backend::BackendResult<()> { + todo!(); + } +} + +struct MockChunkInfo { + compress_offset: u64, + compress_size: u32, + uncompress_offset: u64, + uncompress_size: u32, + is_compressed: bool, +} + +impl MockChunkInfo { + fn new( + compress_offset: u64, + compress_size: u32, + uncompress_offset: u64, + uncompress_size: u32, + is_compressed: bool, + ) -> Self { + Self { + compress_offset, + compress_size, + uncompress_offset, + uncompress_size, + is_compressed, + } + } +} + +impl BlobChunkInfo for MockChunkInfo { + fn is_compressed(&self) -> bool { + self.is_compressed + } + + fn uncompress_size(&self) -> u32 { + self.uncompress_size + } + + fn uncompress_offset(&self) -> u64 { + self.uncompress_offset + } + + fn compress_size(&self) -> u32 { + self.compress_size + } + + fn compress_offset(&self) -> u64 { + self.compress_offset + } + + fn id(&self) -> u32 { + todo!(); + } + + fn as_any(&self) -> &dyn std::any::Any { + todo!(); + } + + fn is_hole(&self) -> bool { + todo!(); + } + + fn blob_index(&self) -> u32 { + todo!(); + } + + fn chunk_id(&self) -> &nydus_utils::digest::RafsDigest { + todo!(); + } +} + +#[test] +fn test_read_chunk() { + let mut reader = create_default_chunk_reader(); + let mut buf = [0u8; 256]; + + assert_eq!(256, reader.read(&mut buf).unwrap()); + assert_eq!(buf, [1u8; 256]); + + assert_eq!(256, reader.read(&mut buf).unwrap()); + assert_eq!(buf, [2u8; 256]); + + assert_eq!(0, reader.read(&mut buf).unwrap()); + assert_eq!(buf, [2u8; 256]); +} + +#[test] +fn test_read_chunk_smaller_buffer() { + let mut reader = create_default_chunk_reader(); + let mut buf = [0u8; 255]; + + assert_eq!(255, reader.read(&mut buf).unwrap()); + assert_eq!(buf, [1u8; 255]); + + assert_eq!(255, reader.read(&mut buf).unwrap()); + assert_eq!(buf[0], 1u8); + assert_eq!(buf[1..255], [2u8; 254]); + + assert_eq!(2, reader.read(&mut buf).unwrap()); + assert_eq!(buf[0..2], [2u8; 2]); + + assert_eq!(0, reader.read(&mut buf).unwrap()); +} + +#[test] +fn test_read_chunk_larger_buffer() { + let mut reader = create_default_chunk_reader(); + let mut buf = [0u8; 257]; + + assert_eq!(257, reader.read(&mut buf).unwrap()); + assert_eq!(buf[..256], [1u8; 256]); + assert_eq!(buf[256], 2u8); + + assert_eq!(255, reader.read(&mut buf).unwrap()); + assert_eq!(buf[..255], [2u8; 255]); + + assert_eq!(0, reader.read(&mut buf).unwrap()); +} + +#[test] +fn test_read_chunk_zero_buffer() { + let mut reader = create_default_chunk_reader(); + let mut buf = [0u8; 0]; + + assert_eq!(0, reader.read(&mut buf).unwrap()); + assert_eq!(0, reader.read(&mut buf).unwrap()); + assert_eq!(0, reader.read(&mut buf).unwrap()); +} + +#[test] +fn test_read_chunk_compress() { + let mut reader = creater_compress_chunk_reader(); + let mut buf = [0u8; 256]; + + assert_eq!(256, reader.read(&mut buf).unwrap()); + assert_eq!(buf, [1u8; 256]); + + assert_eq!(256, reader.read(&mut buf).unwrap()); + assert_eq!(buf, [2u8; 256]); + + assert_eq!(256, reader.read(&mut buf).unwrap()); + assert_eq!(buf, [3u8; 256]); + + assert_eq!(256, reader.read(&mut buf).unwrap()); + assert_eq!(buf, [4u8; 256]); + + assert_eq!(0, reader.read(&mut buf).unwrap()); + assert_eq!(buf, [4u8; 256]); +} + +#[test] +fn test_set_path_for_reg_header() { + let builder = create_default_tar_build(TempFile::new().unwrap().as_path()); + let mut header = create_default_reg_header(); + + let rs = builder.set_path(&mut header, &PathBuf::from("/dir/short_file_name")); + + assert!(rs.unwrap().is_none(), "expect no long section returned"); + assert_eq!(header.path_bytes().into_owned(), b"dir/short_file_name"); +} + +#[test] +fn test_set_path_for_long_reg_header() { + let builder = create_default_tar_build(TempFile::new().unwrap().as_path()); + let mut header = create_default_reg_header(); + + let path = PathBuf::from("/").join("path".repeat(1000)); + let rs = builder.set_path(&mut header, &path); + + assert!( + rs.is_err(), + "should fail to set long path for regular file header" + ); +} + +#[test] +fn test_set_path_for_whiteout_header() { + let builder = create_default_tar_build(TempFile::new().unwrap().as_path()); + let mut header = create_default_whiteout_header(); + + let path = PathBuf::from("/dir/.wh.file_name"); + let rs = builder.set_path(&mut header, &path); + + assert!( + builder.is_white_out_section(&path), + "expect whiteout type assertion" + ); + assert!(rs.unwrap().is_none(), "expect no long section returned"); + assert_eq!(header.path_bytes().into_owned(), b"dir/.wh.file_name"); +} + +#[test] +fn test_set_path_for_long_whiteout_header() { + let builder = create_default_tar_build(TempFile::new().unwrap().as_path()); + let mut header = create_default_whiteout_header(); + + let mut path = OsString::from("/.wh."); + path.push("path".repeat(1000)); + let path = PathBuf::from(path); + + let rs = builder.set_path(&mut header, &path); + + let mut sect = rs.unwrap().unwrap(); + + let header = sect.header; + assert!(header.entry_type().is_gnu_longname()); + assert_eq!(header.path_bytes().into_owned(), b"././@LongLink"); + + let mut data = Vec::new(); + let size = sect.data.read_to_end(&mut data).unwrap(); + assert_eq!(header.size().unwrap() as usize, size); + + let mut path = Vec::new(); + path.extend_from_slice(b".wh."); + path.extend_from_slice(&b"path".repeat(1000)); + path.push(0x00); + assert_eq!(data, path); +} + +#[test] +fn test_set_path_for_dir_header() { + let builder = create_default_tar_build(TempFile::new().unwrap().as_path()); + let mut header = create_default_dir_header(); + + let rs = builder.set_path(&mut header, &PathBuf::from("/dir")); + + assert!(rs.unwrap().is_none(), "expect no long section"); + assert_eq!(header.path_bytes().into_owned(), b"dir/"); +} + +fn create_default_dir_header() -> Header { + let mut header = Header::new_ustar(); + header.set_entry_type(EntryType::dir()); + + header +} + +fn create_default_whiteout_header() -> Header { + let mut header = Header::new_gnu(); + header.set_entry_type(EntryType::file()); + + header +} + +fn create_default_reg_header() -> Header { + let mut header = Header::new_ustar(); + header.set_entry_type(EntryType::file()); + + header +} + +fn create_default_tar_build(output: &Path) -> DefaultTarBuilder { + let output = OpenOptions::new() + .write(true) + .truncate(true) + .read(false) + .open(output) + .unwrap(); + + DefaultTarBuilder { + blob: None, + reader: None, + writer: Builder::new(output), + links: HashMap::new(), + } +} + +fn creater_compress_chunk_reader() -> ChunkReader { + let chunk = [[1u8; 256], [2u8; 256], [3u8; 256], [4u8; 256]].concat(); + + let (compressed_chunk, is_compressed) = compress::compress(&chunk, Algorithm::GZip).unwrap(); + assert!(is_compressed, "expect compressed chunk"); + + let meta = Arc::new(MockChunkInfo::new( + 0, + compressed_chunk.len() as u32, + 0, + chunk.len() as u32, + true, + )); + + let blob_reader = Arc::new(MockBlobReader::new(compressed_chunk.into_owned())); + + ChunkReader::new(Algorithm::GZip, blob_reader, vec![meta]) +} + +fn create_default_chunk_reader() -> ChunkReader { + let chunk1 = [1u8; 256]; + let chunk2 = [2u8; 256]; + + let chunk_meta1 = Arc::new(MockChunkInfo::new( + 0, + chunk1.len() as u32, + 0, + chunk1.len() as u32, + false, + )); + let chunk_meta2 = Arc::new(MockChunkInfo::new( + chunk1.len() as u64, + chunk2.len() as u32, + chunk1.len() as u64, + chunk2.len() as u32, + false, + )); + + let blob_reader = Arc::new(MockBlobReader::new([chunk1, chunk2].concat())); + + ChunkReader::new(Algorithm::None, blob_reader, vec![chunk_meta1, chunk_meta2]) +} diff --git a/src/bin/nydus-image/main.rs b/src/bin/nydus-image/main.rs index fddb9c8ff45..259b6a391fe 100644 --- a/src/bin/nydus-image/main.rs +++ b/src/bin/nydus-image/main.rs @@ -22,6 +22,7 @@ use std::path::{Path, PathBuf}; use anyhow::{bail, Context, Result}; use clap::{App, Arg, ArgMatches, SubCommand}; +use decompress::Decompressor; use nix::unistd::{getegid, geteuid}; use serde::{Deserialize, Serialize}; @@ -42,6 +43,7 @@ use crate::core::context::{ use crate::core::node::{self, WhiteoutSpec}; use crate::core::prefetch::Prefetch; use crate::core::tree; +use crate::decompress::DefaultDecompressor; use crate::merge::Merger; use crate::trace::{EventTracerClass, TimingTracerClass, TraceClass}; use crate::validator::Validator; @@ -50,6 +52,7 @@ use crate::validator::Validator; mod trace; mod builder; mod core; +mod decompress; mod inspect; mod merge; mod stat; @@ -509,6 +512,33 @@ fn prepare_cmd_args(bti_string: String) -> ArgMatches<'static> { .help("path to JSON output file") .takes_value(true)) ) + .subcommand( + SubCommand::with_name("decompress") + .about("Decompress nydus bootstrap and blob files to tar") + .arg( + Arg::with_name("bootstrap") + .long("bootstrap") + .short("B") + .help("path to bootstrap file") + .required(true) + .takes_value(true)) + .arg( + Arg::with_name("blob") + .long("blob") + .short("b") + .help("path to blob file") + .required(true) + .takes_value(true) + ) + .arg( + Arg::with_name("output") + .long("output") + .short("o") + .help("path to output") + .required(true) + .takes_value(true) + ) + ) .arg( Arg::with_name("log-file") .long("log-file") @@ -567,6 +597,8 @@ fn main() -> Result<()> { Command::stat(matches) } else if let Some(matches) = cmd.subcommand_matches("compact") { Command::compact(matches, &build_info) + } else if let Some(matches) = cmd.subcommand_matches("decompress") { + Command::decompress(matches) } else { println!("{}", cmd.usage()); Ok(()) @@ -765,6 +797,24 @@ impl Command { Ok(()) } + fn decompress(args: &clap::ArgMatches) -> Result<()> { + let bootstrap = args.value_of("bootstrap").expect("pass in bootstrap"); + let blob = args.value_of("blob").expect("pass in blob"); + let output = args.value_of("output").expect("pass in output"); + + let decompressor = Box::new(DefaultDecompressor::new(bootstrap, blob, output).map_err( + |err| { + error!("fail to create decompressor, error: {}", err); + anyhow!("fail to create decompressor, error: {}", err) + }, + )?) as Box; + + decompressor.decompress().map_err(|err| { + error!("fail to decompress, error: {}", err); + anyhow!("fail to decompress, error: {}", err) + }) + } + fn check(matches: &clap::ArgMatches, build_info: &BuildTimeInfo) -> Result<()> { let bootstrap_path = Self::get_bootstrap(matches)?; let verbose = matches.is_present("verbose"); From 4960f727c697f58b579600a0f4e9e5ebf89bb8c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B3=B0=E5=8F=8B?= Date: Wed, 22 Jun 2022 16:06:34 +0800 Subject: [PATCH 2/7] feat: decompress smoke test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 泰友 --- src/bin/nydus-image/decompress.rs | 355 ++++++++++++++++--------- src/bin/nydus-image/decompress/test.rs | 246 ++++++++--------- tests/builder.rs | 11 + tests/smoke.rs | 49 +++- 4 files changed, 401 insertions(+), 260 deletions(-) diff --git a/src/bin/nydus-image/decompress.rs b/src/bin/nydus-image/decompress.rs index 6b75c1ca445..967a91939f2 100644 --- a/src/bin/nydus-image/decompress.rs +++ b/src/bin/nydus-image/decompress.rs @@ -42,25 +42,15 @@ pub trait Decompressor { pub struct DefaultDecompressor { bootstrap_path: Box, blob_path: Box, - output_path: Box, + output_path: String, } impl DefaultDecompressor { pub fn new(bootstrap: &str, blob: &str, output: &str) -> Result { - let bootstrap_path = PathBuf::from(bootstrap) - .canonicalize() - .with_context(|| format!("fail to parse bootstrap {}", bootstrap))? - .into_boxed_path(); + let bootstrap_path = PathBuf::from(bootstrap).canonicalize()?.into_boxed_path(); + let blob_path = PathBuf::from(blob).canonicalize()?.into_boxed_path(); - let blob_path = PathBuf::from(blob) - .canonicalize() - .with_context(|| format!("fail to parse blob {}", blob))? - .into_boxed_path(); - - let output_path = PathBuf::from(output) - .canonicalize() - .with_context(|| format!("fail to parse output {}", output))? - .into_boxed_path(); + let output_path = output.to_owned(); Ok(DefaultDecompressor { bootstrap_path, @@ -184,16 +174,17 @@ impl DefaultDecompressor { impl Decompressor for DefaultDecompressor { fn decompress(&self) -> Result<()> { info!( - "default decompressor, bootstrap file: {:?}, blob file: {:?}, output file: {:?}", + "default decompressor, bootstrap file: {:?}, blob file: {:?}, output file: {}", self.bootstrap_path, self.blob_path, self.output_path ); - let meta = self.load_meta().with_context(|| "fail to load meta")?; + let meta = self.load_meta()?; - let mut builder = Box::new( - DefaultTarBuilder::new(&meta, self.blob_path.as_ref(), self.output_path.as_ref()) - .with_context(|| "fail to create tar builder")?, - ) as Box; + let mut builder = Box::new(DefaultTarBuilder::new( + &meta, + self.blob_path.as_ref(), + self.output_path.as_ref(), + )?) as Box; for (node, path) in self.iterator(&meta) { // Not write root node to tar. @@ -231,6 +222,7 @@ impl DefaultTarBuilder { fn new(meta: &RafsSuper, blob_path: &Path, output_path: &Path) -> Result { let writer = Builder::new( OpenOptions::new() + .write(true) .create(true) .truncate(true) .read(false) @@ -246,7 +238,9 @@ impl DefaultTarBuilder { let blob = meta.superblock.get_blob_infos().pop(); let reader = if blob.is_some() { - Some(Self::chunk_reader(blob_path)?) + Some(Self::chunk_reader( + &blob_path.join(blob.as_ref().unwrap().blob_id()), + )?) } else { None }; @@ -297,7 +291,6 @@ impl DefaultTarBuilder { fn build_whiteout_section(&self, inode: &dyn RafsInode, path: &Path) -> Result { // In order to save access time and change time, gnu layout is required. let mut header = Header::new_gnu(); - header.set_entry_type(EntryType::file()); header.set_device_major(0).unwrap(); header.set_device_minor(0).unwrap(); @@ -316,7 +309,7 @@ impl DefaultTarBuilder { header.as_gnu_mut().unwrap().set_ctime(node.mtime()); let mut requirements = Vec::with_capacity(1); - if let Some(requirement) = self.set_path(&mut header, path)? { + if let Some(requirement) = self.set_path_for_gnu_header(&mut header, path)? { requirements.push(requirement); } @@ -331,7 +324,6 @@ impl DefaultTarBuilder { fn build_dir_section(&self, inode: &dyn RafsInode, path: &Path) -> Result { let mut header = Header::new_ustar(); - header.set_entry_type(EntryType::dir()); header.set_size(0); header.set_device_major(0).unwrap(); @@ -343,12 +335,21 @@ impl DefaultTarBuilder { header.set_gid(node.gid() as u64); header.set_mode(node.mode()); - self.set_path(&mut header, path)?; + let mut extensions = Vec::with_capacity(3); + if let Some(extension) = self.set_path_for_pax_header(&mut header, path)? { + extensions.extend(extension); + } header.set_cksum(); + if let Some(extension) = self.get_xattr_as_extensions(inode) { + extensions.extend(extension); + } + let mut requirements = Vec::with_capacity(1); - if let Some(xattr) = self.build_xattr_section(inode, header.path().unwrap().borrow())? { + if let Some(xattr) = + self.build_extension_section(header.path().unwrap().borrow(), extensions)? + { requirements.push(xattr); } @@ -374,7 +375,6 @@ impl DefaultTarBuilder { ); let mut header = Header::new_ustar(); - header.set_entry_type(EntryType::file()); header.set_device_major(0).unwrap(); header.set_device_minor(0).unwrap(); @@ -386,12 +386,21 @@ impl DefaultTarBuilder { header.set_mode(node.mode()); header.set_size(node.size()); - self.set_path(&mut header, path)?; + let mut extensions = Vec::with_capacity(3); + if let Some(extension) = self.set_path_for_pax_header(&mut header, path)? { + extensions.extend(extension); + } header.set_cksum(); + if let Some(extension) = self.get_xattr_as_extensions(inode) { + extensions.extend(extension); + } + let mut requirements = Vec::with_capacity(1); - if let Some(xattr) = self.build_xattr_section(inode, header.path().unwrap().borrow())? { + if let Some(xattr) = + self.build_extension_section(header.path().unwrap().borrow(), extensions)? + { requirements.push(xattr); } @@ -405,13 +414,13 @@ impl DefaultTarBuilder { fn build_symlink_section(&self, node: &dyn RafsInode, path: &Path) -> Result { let link = node.get_symlink().unwrap(); - self.build_link_section(EntryType::symlink(), node, path, &PathBuf::from(link)) + self._build_link_section(EntryType::symlink(), node, path, &PathBuf::from(link)) } - fn build_hard_link_section(&self, node: &dyn RafsInode, path: &Path) -> Result { + fn build_link_section(&self, node: &dyn RafsInode, path: &Path) -> Result { let link = self.links.get(&node.ino()).unwrap(); - self.build_link_section(EntryType::hard_link(), node, path, link) + self._build_link_section(EntryType::hard_link(), node, path, link) } fn build_fifo_section(&self, node: &dyn RafsInode, path: &Path) -> Result { @@ -426,7 +435,7 @@ impl DefaultTarBuilder { self.build_special_file_section(EntryType::block_special(), node, path) } - fn build_link_section( + fn _build_link_section( &self, entry_type: EntryType, inode: &dyn RafsInode, @@ -434,7 +443,6 @@ impl DefaultTarBuilder { link: &Path, ) -> Result { let mut header = Header::new_ustar(); - header.set_entry_type(entry_type); header.set_size(0); header.set_device_major(0).unwrap(); @@ -446,13 +454,24 @@ impl DefaultTarBuilder { header.set_gid(node.gid() as u64); header.set_mode(node.mode()); - self.set_path(&mut header, path)?; - self.set_link(&mut header, link)?; + let mut extensions = Vec::with_capacity(3); + if let Some(extension) = self.set_path_for_pax_header(&mut header, path)? { + extensions.extend(extension); + } + if let Some(extension) = self.set_link_for_pax_header(&mut header, link)? { + extensions.extend(extension); + } header.set_cksum(); + if let Some(extension) = self.get_xattr_as_extensions(inode) { + extensions.extend(extension); + } + let mut requirements = Vec::with_capacity(1); - if let Some(xattr) = self.build_xattr_section(inode, header.path().unwrap().borrow())? { + if let Some(xattr) = + self.build_extension_section(header.path().unwrap().borrow(), extensions)? + { requirements.push(xattr); } @@ -470,7 +489,6 @@ impl DefaultTarBuilder { path: &Path, ) -> Result { let mut header = Header::new_ustar(); - header.set_entry_type(entry_type); let node = InodeWrapper::from_inode_info(inode); @@ -487,12 +505,21 @@ impl DefaultTarBuilder { let dev_minor = ((dev_id >> 12) & 0xffff_ff00) | ((dev_id) & 0x0000_00ff); header.set_device_minor(dev_minor as u32)?; - self.set_path(&mut header, path)?; + let mut extensions = Vec::with_capacity(3); + if let Some(extension) = self.set_path_for_pax_header(&mut header, path)? { + extensions.extend(extension); + } header.set_cksum(); + if let Some(extension) = self.get_xattr_as_extensions(inode) { + extensions.extend(extension); + } + let mut requirements = Vec::with_capacity(1); - if let Some(xattr) = self.build_xattr_section(inode, header.path().unwrap().borrow())? { + if let Some(xattr) = + self.build_extension_section(header.path().unwrap().borrow(), extensions)? + { requirements.push(xattr); } @@ -503,41 +530,49 @@ impl DefaultTarBuilder { }) } - fn build_xattr_section( + fn build_extension_section( &self, - inode: &dyn RafsInode, - header_name: &Path, + path: &Path, + mut extensions: Vec<(Vec, Vec)>, ) -> Result> { - fn normalize_path(path: &Path) -> PathBuf { - let mut name = PathBuf::new(); - name.push(path.parent().unwrap()); - name.push(AsRef::::as_ref("PaxHeaders.0")); - name.push(path.file_name().unwrap()); - name - } - - let (data, size) = if let Some(rs) = self.build_pax_data(inode) { - rs - } else { + if extensions.len() == 0 { return Ok(None); - }; + } let mut header = Header::new_ustar(); - header.set_entry_type(EntryType::XHeader); header.set_mode(0o644); header.set_uid(0); header.set_gid(0); header.set_mtime(0); - header.set_size(size as u64); - self.set_path(&mut header, &normalize_path(header_name))?; + extensions.sort_by(|(k1, _), (k2, _)| { + let k1 = str::from_utf8(k1).unwrap(); + let k2 = str::from_utf8(k2).unwrap(); + + k1.cmp(k2) + }); + let data: Vec = extensions + .into_iter() + .map(|(k, v)| self.build_pax_record(&k, &v)) + .flatten() + .collect(); + header.set_size(data.len() as u64); + + let mut path = self.build_pax_name(path); + let max_len = header.as_old().name.len(); + if path.as_os_str().len() > max_len { + path = self.truncate_path(&path, max_len)?; + } + header + .set_path(&path) + .map_err(|err| anyhow!("fail to set path for pax section, error {}", err))?; header.set_cksum(); Ok(Some(TarSection { header, - data: Box::new(data), + data: Box::new(Cursor::new(data)), presections: Vec::new(), })) } @@ -568,46 +603,48 @@ impl DefaultTarBuilder { } } - fn build_pax_data(&self, inode: &dyn RafsInode) -> Option<(impl Read, usize)> { + fn get_xattr_as_extensions(&self, inode: &dyn RafsInode) -> Option, Vec)>> { if !inode.has_xattr() { return None; } - // Seek for a stable order - let mut keys = inode.get_xattrs().unwrap(); - keys.sort_by(|k1, k2| str::from_utf8(k1).unwrap().cmp(str::from_utf8(k2).unwrap())); + let keys = inode.get_xattrs().unwrap(); + let mut extensions = Vec::with_capacity(keys.len()); - let mut data = Vec::new(); for key in keys { let value = inode .get_xattr(OsStr::from_bytes(&key)) .unwrap() .unwrap_or_default(); - data.append(&mut self.build_pax_record(&key, &value)); + let key = Vec::from(PAX_PREFIX.to_owned()) + .into_iter() + .chain(key.into_iter()) + .collect(); + extensions.push((key, value)); } - let len = data.len(); - Some((Cursor::new(data), len)) + Some(extensions) + } + + fn build_pax_name(&self, path: &Path) -> PathBuf { + let mut path = path.to_path_buf(); + let filename = path.file_name().unwrap().to_owned(); + path.set_file_name("PaxHeaders.0"); + path.join(filename) } fn build_pax_record(&self, k: &[u8], v: &[u8]) -> Vec { fn pax(buf: &mut Vec, size: usize, k: &[u8], v: &[u8]) { buf.extend_from_slice(size.to_string().as_bytes()); buf.extend_from_slice(PAX_SEP1); - buf.extend_from_slice(PAX_PREFIX); buf.extend_from_slice(k); buf.extend_from_slice(PAX_SEP2); buf.extend_from_slice(v); buf.extend_from_slice(PAX_DELIMITER); } - let mut size = k.len() - + v.len() - + PAX_SEP1.len() - + PAX_SEP2.len() - + PAX_DELIMITER.len() - + PAX_PREFIX.len(); + let mut size = k.len() + v.len() + PAX_SEP1.len() + PAX_SEP2.len() + PAX_DELIMITER.len(); size += size.to_string().as_bytes().len(); let mut record = Vec::with_capacity(size); @@ -622,73 +659,129 @@ impl DefaultTarBuilder { record } - /// If path size is too long, a long-name-header is returned as precondition of input header. - fn set_path(&self, header: &mut Header, path: &Path) -> Result> { - fn try_best_to_str(bs: &[u8]) -> Result<&str> { - match str::from_utf8(bs) { - Ok(s) => Ok(s), - Err(err) => str::from_utf8(&bs[..err.valid_up_to()]) - .map_err(|err| anyhow!("fail to convert bytes to utf8 str, error: {}", err)), - } + fn set_path_for_pax_header( + &self, + header: &mut Header, + path: &Path, + ) -> Result, Vec)>>> { + let path = self + .normalize_path(&path, header.entry_type().is_dir()) + .map_err(|err| anyhow!("fail to normalize path, error {}", err))?; + + let max_len = header.as_old().name.len(); + if path.as_os_str().len() <= max_len { + return header + .set_path(path) + .map_err(|err| anyhow!("fail to set short path for pax header, error {}", err)) + .map(|_| None); } - let normalized = self - .normalize_path(path, header.entry_type().is_dir()) + let extension = vec![( + "path".to_owned().into_bytes(), + path.to_owned().into_os_string().into_vec(), + )]; + + let path = self + .truncate_path(&path, max_len) + .map_err(|err| anyhow!("fail to truncate path for pax header, error {}", err))?; + + header.set_path(&path).map_err(|err| { + anyhow!( + "fail to set header path again for {:?}, error {}", + path, + err + ) + })?; + + Ok(Some(extension)) + } + + fn set_path_for_gnu_header( + &self, + header: &mut Header, + path: &Path, + ) -> Result> { + let path = self + .normalize_path(&path, header.entry_type().is_dir()) .map_err(|err| anyhow!("fail to normalize path, error {}", err))?; - let rs = header - .set_path(&normalized) - .map(|_| None) - .map_err(|err| anyhow!("fail to set path for header, error {}", err)); - if rs.is_ok() || header.as_ustar().is_some() { - return rs; - } + let rs = match header.set_path(&path) { + Ok(_) => return Ok(None), + Err(err) => Err(anyhow!("fail to set path for header, error {}", err)), + }; - let data = normalized.into_os_string().into_vec(); - let max = header.as_old().name.len(); - if data.len() < max { - return rs; - } + let path = match self.truncate_path(&path, header.as_old().name.len()) { + Ok(path) => path, + Err(_) => return rs, + }; - // try to set truncated path to header - header - .set_path( - try_best_to_str(&data.as_slice()[..max]) - .with_context(|| "fail to truncate path")?, + header.set_path(&path).map_err(|err| { + anyhow!( + "fail to set header path again for {:?}, error {}", + path, + err ) - .map_err(|err| { - anyhow!( - "fail to set header path again for {:?}, error {}", - path, - err - ) - })?; + })?; - Ok(Some(self.build_long_section(EntryType::GNULongName, data))) + Ok(Some(self.build_long_section( + EntryType::GNULongName, + path.into_os_string().into_vec(), + ))) + } + + // path is required longer than max_len + fn truncate_path(&self, path: &Path, max_len: usize) -> Result { + let path = path.as_os_str().as_bytes(); + if path.len() < max_len { + bail!("path is shorter than limit") + } + + let path = match str::from_utf8(&path[..max_len]) { + Ok(s) => Ok(s), + Err(err) => str::from_utf8(&path[..err.valid_up_to()]) + .map_err(|err| anyhow!("fail to convert bytes to utf8 str, error: {}", err)), + }?; + + Ok(PathBuf::from(path)) } /// If link size is too long, a long-link-header is returned as precondition of input /// header. - fn set_link(&self, header: &mut Header, path: &Path) -> Result> { - let normalized = self + fn set_link_for_pax_header( + &self, + header: &mut Header, + path: &Path, + ) -> Result, Vec)>>> { + let path = self .normalize_path(path, header.entry_type().is_dir()) .map_err(|err| anyhow!("fail to normalize path, error {}", err))?; - let rs = header - .set_link_name(normalized) - .map(|_| None) - .map_err(|err| anyhow!("fail to set linkname for header, error {}", err)); - if rs.is_ok() || header.as_ustar().is_some() { - return rs; + let max_len = header.as_old().linkname.len(); + if path.as_os_str().len() <= max_len { + return header + .set_link_name(&path) + .map_err(|err| anyhow!("fail to set short path for pax header, error {}", err)) + .map(|_| None); } - let data = path.to_path_buf().into_os_string().into_vec(); - let max = header.as_old().linkname.len(); - if data.len() < max { - return rs; - } + let extension = vec![( + "linkpath".to_owned().into_bytes(), + path.to_owned().into_os_string().into_vec(), + )]; + + let path = self + .truncate_path(&path, max_len) + .map_err(|err| anyhow!("fail to truncate path for pax header, error {}", err))?; + + header.set_link_name(&path).map_err(|err| { + anyhow!( + "fail to set header path again for {:?}, error {}", + path, + err + ) + })?; - Ok(Some(self.build_long_section(EntryType::GNULongLink, data))) + Ok(Some(extension)) } fn normalize_path(&self, path: &Path, is_dir: bool) -> Result { @@ -715,15 +808,18 @@ impl DefaultTarBuilder { Ok(normalized) } - fn is_hard_link_section(&mut self, node: &dyn RafsInode, path: &Path) -> bool { - let new_face = node.is_hardlink() && !self.links.contains_key(&node.ino()); + fn is_link_section(&mut self, node: &dyn RafsInode, path: &Path) -> bool { + if !node.is_hardlink() || node.is_dir() { + return false; + } - if new_face { + let is_appeared = self.links.contains_key(&node.ino()); + if !is_appeared { self.links .insert(node.ino(), path.to_path_buf().into_boxed_path()); } - !new_face + return is_appeared; } fn is_symlink_section(&self, node: &dyn RafsInode) -> bool { @@ -766,12 +862,9 @@ impl DefaultTarBuilder { impl TarBuilder for DefaultTarBuilder { fn append(&mut self, inode: &dyn RafsInode, path: &Path) -> Result<()> { let tar_sect = match inode { - // Ignore socket file node if self.is_socket_section(node) => return Ok(()), - node if self.is_hard_link_section(node, path) => { - self.build_hard_link_section(node, path)? - } + node if self.is_link_section(node, path) => self.build_link_section(node, path)?, node if self.is_white_out_section(path) => self.build_whiteout_section(node, path)?, diff --git a/src/bin/nydus-image/decompress/test.rs b/src/bin/nydus-image/decompress/test.rs index fc5d763fffa..e1384bcd5f5 100644 --- a/src/bin/nydus-image/decompress/test.rs +++ b/src/bin/nydus-image/decompress/test.rs @@ -1,21 +1,11 @@ -use std::{ - collections::HashMap, - ffi::OsString, - fs::OpenOptions, - io::Read, - path::{Path, PathBuf}, - sync::Arc, -}; - use nydus_utils::{ compress::{self, Algorithm}, metrics::BackendMetrics, }; +use std::{io::Read, sync::Arc}; use storage::{backend::BlobReader, device::BlobChunkInfo}; -use tar::{Builder, EntryType, Header}; -use vmm_sys_util::tempfile::TempFile; -use super::{ChunkReader, DefaultTarBuilder}; +use super::ChunkReader; struct MockBlobReader { data: Vec, @@ -208,122 +198,122 @@ fn test_read_chunk_compress() { assert_eq!(buf, [4u8; 256]); } -#[test] -fn test_set_path_for_reg_header() { - let builder = create_default_tar_build(TempFile::new().unwrap().as_path()); - let mut header = create_default_reg_header(); - - let rs = builder.set_path(&mut header, &PathBuf::from("/dir/short_file_name")); - - assert!(rs.unwrap().is_none(), "expect no long section returned"); - assert_eq!(header.path_bytes().into_owned(), b"dir/short_file_name"); -} - -#[test] -fn test_set_path_for_long_reg_header() { - let builder = create_default_tar_build(TempFile::new().unwrap().as_path()); - let mut header = create_default_reg_header(); - - let path = PathBuf::from("/").join("path".repeat(1000)); - let rs = builder.set_path(&mut header, &path); - - assert!( - rs.is_err(), - "should fail to set long path for regular file header" - ); -} - -#[test] -fn test_set_path_for_whiteout_header() { - let builder = create_default_tar_build(TempFile::new().unwrap().as_path()); - let mut header = create_default_whiteout_header(); - - let path = PathBuf::from("/dir/.wh.file_name"); - let rs = builder.set_path(&mut header, &path); - - assert!( - builder.is_white_out_section(&path), - "expect whiteout type assertion" - ); - assert!(rs.unwrap().is_none(), "expect no long section returned"); - assert_eq!(header.path_bytes().into_owned(), b"dir/.wh.file_name"); -} - -#[test] -fn test_set_path_for_long_whiteout_header() { - let builder = create_default_tar_build(TempFile::new().unwrap().as_path()); - let mut header = create_default_whiteout_header(); - - let mut path = OsString::from("/.wh."); - path.push("path".repeat(1000)); - let path = PathBuf::from(path); - - let rs = builder.set_path(&mut header, &path); - - let mut sect = rs.unwrap().unwrap(); - - let header = sect.header; - assert!(header.entry_type().is_gnu_longname()); - assert_eq!(header.path_bytes().into_owned(), b"././@LongLink"); - - let mut data = Vec::new(); - let size = sect.data.read_to_end(&mut data).unwrap(); - assert_eq!(header.size().unwrap() as usize, size); - - let mut path = Vec::new(); - path.extend_from_slice(b".wh."); - path.extend_from_slice(&b"path".repeat(1000)); - path.push(0x00); - assert_eq!(data, path); -} - -#[test] -fn test_set_path_for_dir_header() { - let builder = create_default_tar_build(TempFile::new().unwrap().as_path()); - let mut header = create_default_dir_header(); - - let rs = builder.set_path(&mut header, &PathBuf::from("/dir")); - - assert!(rs.unwrap().is_none(), "expect no long section"); - assert_eq!(header.path_bytes().into_owned(), b"dir/"); -} - -fn create_default_dir_header() -> Header { - let mut header = Header::new_ustar(); - header.set_entry_type(EntryType::dir()); - - header -} - -fn create_default_whiteout_header() -> Header { - let mut header = Header::new_gnu(); - header.set_entry_type(EntryType::file()); - - header -} - -fn create_default_reg_header() -> Header { - let mut header = Header::new_ustar(); - header.set_entry_type(EntryType::file()); - - header -} - -fn create_default_tar_build(output: &Path) -> DefaultTarBuilder { - let output = OpenOptions::new() - .write(true) - .truncate(true) - .read(false) - .open(output) - .unwrap(); - - DefaultTarBuilder { - blob: None, - reader: None, - writer: Builder::new(output), - links: HashMap::new(), - } -} +//#[test] +//fn test_set_path_for_reg_header() { +// let builder = create_default_tar_build(TempFile::new().unwrap().as_path()); +// let mut header = create_default_reg_header(); +// +// let rs = builder.set_path(&mut header, &PathBuf::from("/dir/short_file_name")); +// +// assert!(rs.unwrap().is_none(), "expect no long section returned"); +// assert_eq!(header.path_bytes().into_owned(), b"dir/short_file_name"); +//} +// +//#[test] +//fn test_set_path_for_long_reg_header() { +// let builder = create_default_tar_build(TempFile::new().unwrap().as_path()); +// let mut header = create_default_reg_header(); +// +// let path = PathBuf::from("/").join("path".repeat(1000)); +// let rs = builder.set_path(&mut header, &path); +// +// assert!( +// rs.is_err(), +// "should fail to set long path for regular file header" +// ); +//} +// +//#[test] +//fn test_set_path_for_whiteout_header() { +// let builder = create_default_tar_build(TempFile::new().unwrap().as_path()); +// let mut header = create_default_whiteout_header(); +// +// let path = PathBuf::from("/dir/.wh.file_name"); +// let rs = builder.set_path(&mut header, &path); +// +// assert!( +// builder.is_white_out_section(&path), +// "expect whiteout type assertion" +// ); +// assert!(rs.unwrap().is_none(), "expect no long section returned"); +// assert_eq!(header.path_bytes().into_owned(), b"dir/.wh.file_name"); +//} +// +//#[test] +//fn test_set_path_for_long_whiteout_header() { +// let builder = create_default_tar_build(TempFile::new().unwrap().as_path()); +// let mut header = create_default_whiteout_header(); +// +// let mut path = OsString::from("/.wh."); +// path.push("path".repeat(1000)); +// let path = PathBuf::from(path); +// +// let rs = builder.set_path(&mut header, &path); +// +// let mut sect = rs.unwrap().unwrap(); +// +// let header = sect.header; +// assert!(header.entry_type().is_gnu_longname()); +// assert_eq!(header.path_bytes().into_owned(), b"././@LongLink"); +// +// let mut data = Vec::new(); +// let size = sect.data.read_to_end(&mut data).unwrap(); +// assert_eq!(header.size().unwrap() as usize, size); +// +// let mut path = Vec::new(); +// path.extend_from_slice(b".wh."); +// path.extend_from_slice(&b"path".repeat(1000)); +// path.push(0x00); +// assert_eq!(data, path); +//} +// +//#[test] +//fn test_set_path_for_dir_header() { +// let builder = create_default_tar_build(TempFile::new().unwrap().as_path()); +// let mut header = create_default_dir_header(); +// +// let rs = builder.set_path(&mut header, &PathBuf::from("/dir")); +// +// assert!(rs.unwrap().is_none(), "expect no long section"); +// assert_eq!(header.path_bytes().into_owned(), b"dir/"); +//} + +//fn create_default_dir_header() -> Header { +// let mut header = Header::new_ustar(); +// header.set_entry_type(EntryType::dir()); +// +// header +//} +// +//fn create_default_whiteout_header() -> Header { +// let mut header = Header::new_gnu(); +// header.set_entry_type(EntryType::file()); +// +// header +//} +// +//fn create_default_reg_header() -> Header { +// let mut header = Header::new_ustar(); +// header.set_entry_type(EntryType::file()); +// +// header +//} +// +//fn create_default_tar_build(output: &Path) -> DefaultTarBuilder { +// let output = OpenOptions::new() +// .write(true) +// .truncate(true) +// .read(false) +// .open(output) +// .unwrap(); +// +// DefaultTarBuilder { +// blob: None, +// reader: None, +// writer: Builder::new(output), +// links: HashMap::new(), +// } +//} fn creater_compress_chunk_reader() -> ChunkReader { let chunk = [[1u8; 256], [2u8; 256], [3u8; 256], [4u8; 256]].concat(); diff --git a/tests/builder.rs b/tests/builder.rs index c98c2a8e9f4..56a102f613d 100644 --- a/tests/builder.rs +++ b/tests/builder.rs @@ -450,4 +450,15 @@ impl<'a> Builder<'a> { assert_eq!(&header.path_bytes().as_ref(), b"image.blob"); assert_eq!(cur, header.size().unwrap()); } + + pub fn decompress(&self, bootstrap: &str, blob: &str, output: &str) { + let cmd = format!( + "{:?} decompress --bootstrap {:?} --blob {:?} --output {:?}", + self.builder, + self.work_dir.join(bootstrap), + self.work_dir.join(blob), + self.work_dir.join(output) + ); + exec(&cmd, false).unwrap(); + } } diff --git a/tests/smoke.rs b/tests/smoke.rs index c329d5c1d1b..7fe9c719621 100644 --- a/tests/smoke.rs +++ b/tests/smoke.rs @@ -4,7 +4,10 @@ #[macro_use] extern crate log; -use std::path::Path; +use std::env::var; +use std::fs::File; +use std::io::Read; +use std::path::{Path, PathBuf}; use nydus_app::setup_logging; use nydus_utils::exec; @@ -337,3 +340,47 @@ fn test_inline(rafs_version: &str) { builder.build_inline_lower(rafs_version); builder.check_inline_layout(); } + +fn integration_test_decompress1() { + let mut prefix = + PathBuf::from(var("TEST_WORKDIR_PREFIX").expect("Please specify TEST_WORKDIR_PREFIX env")); + prefix.push(""); + + let wk_dir = TempDir::new_with_prefix(prefix).unwrap(); + + let mut builder = builder::new(wk_dir.as_path(), "oci"); + builder.make_lower(); + builder.build_lower("lz4_block", "5"); + + let tar_name = wk_dir.as_path().join("oci.tar"); + builder.decompress("bootstrap-lower", "blobs", tar_name.to_str().unwrap()); + + let unpack_dir = wk_dir.as_path().join("output"); + exec(&format!("mkdir {:?}", unpack_dir), false, b"").unwrap(); + exec( + &format!("tar --xattrs -xf {:?} -C {:?}", tar_name, unpack_dir), + false, + b"", + ) + .unwrap(); + + let tree_ret = exec(&format!("tree -a -J -v {:?}", unpack_dir), true, b"").unwrap(); + let md5_ret = exec( + &format!("find {:?} -type f -exec md5sum {{}} + | sort", unpack_dir), + true, + b"", + ) + .unwrap(); + + let ret = format!( + "{}{}", + tree_ret.replace(unpack_dir.to_str().unwrap(), ""), + md5_ret.replace(unpack_dir.to_str().unwrap(), "") + ); + + let mut texture = File::open("./tests/texture/directory/lower.result").unwrap(); + let mut expected = String::new(); + texture.read_to_string(&mut expected).unwrap(); + + assert_eq!(ret.trim(), expected.trim()); +} From 1ec8073d1a7e00794f2415d9d3352c83013b10d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B3=B0=E5=8F=8B?= Date: Wed, 22 Jun 2022 23:55:23 +0800 Subject: [PATCH 3/7] refact: split build to separate builders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 泰友 --- src/bin/nydus-image/decompress.rs | 1038 +++++++++++++++++------------ src/bin/nydus-image/main.rs | 14 +- tests/builder.rs | 2 +- 3 files changed, 625 insertions(+), 429 deletions(-) diff --git a/src/bin/nydus-image/decompress.rs b/src/bin/nydus-image/decompress.rs index 967a91939f2..e9c2603ecb9 100644 --- a/src/bin/nydus-image/decompress.rs +++ b/src/bin/nydus-image/decompress.rs @@ -1,7 +1,6 @@ extern crate tar; use std::{ - borrow::Borrow, collections::HashMap, ffi::OsStr, fs::{File, OpenOptions}, @@ -9,12 +8,13 @@ use std::{ iter::from_fn, os::unix::prelude::{OsStrExt, OsStringExt}, path::{Path, PathBuf}, + rc::Rc, str, sync::Arc, vec::IntoIter, }; -use anyhow::{Context, Result}; +use anyhow::Result; use nydus_rafs::{ metadata::{layout::RAFS_ROOT_INODE, RafsInode, RafsMode, RafsSuper}, RafsIoReader, @@ -22,7 +22,7 @@ use nydus_rafs::{ use nydus_utils::compress::{self, Algorithm}; use storage::{ backend::{localfs::LocalFs, BlobBackend, BlobReader, LocalFsConfig}, - device::{BlobChunkInfo, BlobInfo}, + device::BlobChunkInfo, utils::alloc_buf, }; use tar::{Builder, EntryType, Header}; @@ -39,39 +39,42 @@ pub trait Decompressor { } /// A decompressor with the ability to convert bootstrap file and blob file to tar -pub struct DefaultDecompressor { - bootstrap_path: Box, - blob_path: Box, - output_path: String, +pub struct OCIDecompressor { + bootstrap: Box, + blob_dir: Option>, + output: Box, } -impl DefaultDecompressor { - pub fn new(bootstrap: &str, blob: &str, output: &str) -> Result { - let bootstrap_path = PathBuf::from(bootstrap).canonicalize()?.into_boxed_path(); - let blob_path = PathBuf::from(blob).canonicalize()?.into_boxed_path(); +impl OCIDecompressor { + pub fn new(bootstrap: &str, blob_dir: Option<&str>, output: &str) -> Result { + let bootstrap = PathBuf::from(bootstrap).into_boxed_path(); + let output = PathBuf::from(output).into_boxed_path(); - let output_path = output.to_owned(); + let blob_dir = match blob_dir { + Some(dir) => Some(PathBuf::from(dir).canonicalize()?.into_boxed_path()), + None => None, + }; - Ok(DefaultDecompressor { - bootstrap_path, - blob_path, - output_path, + Ok(OCIDecompressor { + bootstrap, + blob_dir, + output, }) } - fn load_meta(&self) -> Result { + fn load_rafs(&self) -> Result { let bootstrap = OpenOptions::new() .read(true) .write(false) - .open(self.bootstrap_path.as_ref()) + .open(&*self.bootstrap) .map_err(|err| { error!( "fail to load bootstrap {:?}, error: {}", - self.bootstrap_path, err + self.bootstrap, err ); anyhow!( "fail to load bootstrap {:?}, error: {}", - self.bootstrap_path, + self.bootstrap, err ) })?; @@ -85,11 +88,11 @@ impl DefaultDecompressor { .map_err(|err| { error!( "fail to load bootstrap {:?}, error: {}", - self.bootstrap_path, err + self.bootstrap, err ); anyhow!( "fail to load bootstrap {:?}, error: {}", - self.bootstrap_path, + self.bootstrap, err ) })?; @@ -97,26 +100,31 @@ impl DefaultDecompressor { Ok(rs) } - /// A lazy iterator of RafsInode in DFS. + /// A lazy iterator of RafsInode in DFS, which travels in preorder. fn iterator<'a>( &'a self, rs: &'a RafsSuper, ) -> Box, Box)> + 'a> { + // A cursor means the next node to visit of certain height in the tree. It always starts with the first one of level. + // A cursor stack is of cursors from root to leaf. let mut cursor_stack = Vec::with_capacity(32); + cursor_stack.push(self.cursor_of_root(rs)); - let dfs_cursor = move || { + let dfs = move || { while !cursor_stack.is_empty() { let mut cursor = cursor_stack.pop().unwrap(); - let (node, path) = if let Some(iterm) = cursor.next() { - cursor_stack.push(cursor); - iterm - } else { - continue; + + let (node, path) = match cursor.next() { + None => continue, + Some(point) => { + cursor_stack.push(cursor); + point + } }; if node.is_dir() { - cursor_stack.push(self.cursor_of_children(node.clone(), path.as_ref())) + cursor_stack.push(self.cursor_of_children(node.clone(), &*path)) } return Some((node, path)); @@ -125,7 +133,7 @@ impl DefaultDecompressor { None }; - Box::new(from_fn(dfs_cursor)) + Box::new(from_fn(dfs)) } fn cursor_of_children( @@ -133,21 +141,21 @@ impl DefaultDecompressor { node: Arc, path: &Path, ) -> Box, Box)>> { - let path = path.to_path_buf(); - let mut cursor = 0..node.get_child_count(); + let base = path.to_path_buf(); + let mut next_idx = 0..node.get_child_count(); - let visitor = from_fn(move || { - if cursor.is_empty() { + let visitor = move || { + if next_idx.is_empty() { return None; } - let child = node.get_child_by_index(cursor.next().unwrap()).unwrap(); - let child_path = path.join(child.name()).into_boxed_path(); + let child = node.get_child_by_index(next_idx.next().unwrap()).unwrap(); + let child_path = base.join(child.name()).into_boxed_path(); Some((child, child_path)) - }); + }; - Box::new(visitor) + Box::new(from_fn(visitor)) } fn cursor_of_root<'a>( @@ -171,30 +179,26 @@ impl DefaultDecompressor { } } -impl Decompressor for DefaultDecompressor { +impl Decompressor for OCIDecompressor { fn decompress(&self) -> Result<()> { info!( - "default decompressor, bootstrap file: {:?}, blob file: {:?}, output file: {}", - self.bootstrap_path, self.blob_path, self.output_path + "default decompressor, bootstrap file: {:?}, blob file: {:?}, output file: {:?}", + self.bootstrap, self.blob_dir, self.output ); - let meta = self.load_meta()?; + let rafs = self.load_rafs()?; - let mut builder = Box::new(DefaultTarBuilder::new( - &meta, - self.blob_path.as_ref(), - self.output_path.as_ref(), + let mut builder = Box::new(OCITarBuilder::new( + &rafs, + self.blob_dir.as_ref().map(|dir| dir.as_ref()), + &*self.output, )?) as Box; - for (node, path) in self.iterator(&meta) { - // Not write root node to tar. + for (node, path) in self.iterator(&rafs) { if node.name() == OsStr::from_bytes(ROOT_PATH_NAME) { continue; } - - builder - .append(node.as_ref(), path.as_ref()) - .with_context(|| "fail to append inode to tar")?; + builder.append(&*node, &*path)?; } Ok(()) @@ -208,87 +212,86 @@ trait TarBuilder { struct TarSection { header: Header, data: Box, - presections: Vec, } -struct DefaultTarBuilder { - blob: Option>, - reader: Option>, - writer: Builder, - links: HashMap>, +trait SectionBuilder { + fn can_handle(&mut self, inode: &dyn RafsInode, path: &Path) -> bool; + fn build(&self, inode: &dyn RafsInode, path: &Path) -> Result>; } -impl DefaultTarBuilder { - fn new(meta: &RafsSuper, blob_path: &Path, output_path: &Path) -> Result { - let writer = Builder::new( - OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .read(false) - .open(output_path) - .map_err(|err| { - anyhow!( - "fail to open output file {:?}, error: {:?}", - output_path, - err - ) - })?, - ); +struct OCISocketBuilder {} - let blob = meta.superblock.get_blob_infos().pop(); - let reader = if blob.is_some() { - Some(Self::chunk_reader( - &blob_path.join(blob.as_ref().unwrap().blob_id()), - )?) - } else { - None - }; +impl OCISocketBuilder { + fn new() -> Self { + OCISocketBuilder {} + } +} - Ok(Self { - blob, - reader, - writer, +impl SectionBuilder for OCISocketBuilder { + fn can_handle(&mut self, node: &dyn RafsInode, _: &Path) -> bool { + InodeWrapper::from_inode_info(node).is_sock() + } + + fn build(&self, _: &dyn RafsInode, _: &Path) -> Result> { + Ok(Vec::new()) + } +} + +struct OCILinkBuilder { + links: HashMap>, + pax_link_builder: Rc, +} + +impl OCILinkBuilder { + fn new(pax_link_builder: Rc) -> Self { + OCILinkBuilder { links: HashMap::new(), - }) + pax_link_builder, + } } +} - fn chunk_reader(blob_path: &Path) -> Result> { - let config = LocalFsConfig { - blob_file: blob_path.to_str().unwrap().to_owned(), - readahead: false, - readahead_sec: Default::default(), - dir: Default::default(), - alt_dirs: Default::default(), - }; - let config = serde_json::to_value(config).map_err(|err| { - anyhow!( - "fail to create local backend config for {:?}, error: {:?}", - blob_path, - err - ) - })?; +impl SectionBuilder for OCILinkBuilder { + fn can_handle(&mut self, node: &dyn RafsInode, path: &Path) -> bool { + if !node.is_hardlink() || node.is_dir() { + return false; + } - let backend = LocalFs::new(config, Some("decompressor")).map_err(|err| { - anyhow!( - "fail to create local backend for {:?}, error: {:?}", - blob_path, - err - ) - })?; + let is_appeared = self.links.contains_key(&node.ino()); + if !is_appeared { + self.links + .insert(node.ino(), path.to_path_buf().into_boxed_path()); + } - let reader = backend.get_reader("").map_err(|err| { - anyhow!( - "fail to get chunk reader for {:?}, error: {:?}", - blob_path, - err - ) - })?; + return is_appeared; + } - Ok(reader) + fn build(&self, node: &dyn RafsInode, path: &Path) -> Result> { + let link = self.links.get(&node.ino()).unwrap(); + + self.pax_link_builder + .build(EntryType::hard_link(), node, path, link) + } +} + +struct OCIWhiteoutBuilder {} + +impl OCIWhiteoutBuilder { + fn new() -> Self { + OCIWhiteoutBuilder {} + } +} + +impl SectionBuilder for OCIWhiteoutBuilder { + fn can_handle(&mut self, _: &dyn RafsInode, path: &Path) -> bool { + path.file_name() + .unwrap() + .to_str() + .unwrap() + .starts_with(OCISPEC_WHITEOUT_PREFIX) } - fn build_whiteout_section(&self, inode: &dyn RafsInode, path: &Path) -> Result { + fn build(&self, inode: &dyn RafsInode, path: &Path) -> Result> { // In order to save access time and change time, gnu layout is required. let mut header = Header::new_gnu(); header.set_entry_type(EntryType::file()); @@ -308,21 +311,39 @@ impl DefaultTarBuilder { header.as_gnu_mut().unwrap().set_atime(node.mtime()); header.as_gnu_mut().unwrap().set_ctime(node.mtime()); - let mut requirements = Vec::with_capacity(1); - if let Some(requirement) = self.set_path_for_gnu_header(&mut header, path)? { - requirements.push(requirement); + let mut sections = Vec::with_capacity(2); + if let Some(sect) = GNUUtil::set_path(&mut header, path)? { + sections.push(sect); } header.set_cksum(); - Ok(TarSection { + let main_header = TarSection { header, data: Box::new(io::empty()), - presections: Vec::new(), - }) + }; + sections.push(main_header); + + Ok(sections) + } +} + +struct OCIDirBuilder { + ext_builder: Rc, +} + +impl OCIDirBuilder { + fn new(ext_builder: Rc) -> Self { + OCIDirBuilder { ext_builder } + } +} + +impl SectionBuilder for OCIDirBuilder { + fn can_handle(&mut self, node: &dyn RafsInode, _: &Path) -> bool { + node.is_dir() } - fn build_dir_section(&self, inode: &dyn RafsInode, path: &Path) -> Result { + fn build(&self, inode: &dyn RafsInode, path: &Path) -> Result> { let mut header = Header::new_ustar(); header.set_entry_type(EntryType::dir()); header.set_size(0); @@ -335,45 +356,75 @@ impl DefaultTarBuilder { header.set_gid(node.gid() as u64); header.set_mode(node.mode()); - let mut extensions = Vec::with_capacity(3); - if let Some(extension) = self.set_path_for_pax_header(&mut header, path)? { + let mut extensions = Vec::with_capacity(2); + if let Some(extension) = PAXUtil::set_path(&mut header, path)? { extensions.extend(extension); } - - header.set_cksum(); - - if let Some(extension) = self.get_xattr_as_extensions(inode) { + if let Some(extension) = PAXUtil::get_xattr_as_extensions(inode) { extensions.extend(extension); } - let mut requirements = Vec::with_capacity(1); - if let Some(xattr) = - self.build_extension_section(header.path().unwrap().borrow(), extensions)? - { - requirements.push(xattr); + header.set_cksum(); + + let mut sections = Vec::with_capacity(2); + if let Some(ext_sect) = self.ext_builder.build(&header, extensions)? { + sections.push(ext_sect); } - Ok(TarSection { + let main_header = TarSection { header, data: Box::new(io::empty()), - presections: requirements, - }) + }; + sections.push(main_header); + + Ok(sections) + } +} + +struct OCIRegBuilder { + ext_builder: Rc, + reader: Option>, + compressor: Option, +} + +impl OCIRegBuilder { + fn new( + ext_builder: Rc, + reader: Option>, + compressor: Option, + ) -> Self { + OCIRegBuilder { + ext_builder, + reader, + compressor, + } } - fn build_reg_section(&self, inode: &dyn RafsInode, path: &Path) -> Result { - if self.blob.is_none() || self.reader.is_none() { - bail!("miss blob meta or chunk reader for building regular header") + fn build_data(&self, inode: &dyn RafsInode) -> Box { + if self.reader.is_none() { + return Box::new(io::empty()); } - let chunks: Vec> = (0..inode.get_chunk_count()) + let chunks = (0..inode.get_chunk_count()) .map(|i| inode.get_chunk_info(i).unwrap()) .collect(); - let data = ChunkReader::new( - self.blob.as_ref().unwrap().compressor(), + + let reader = ChunkReader::new( + self.compressor.as_ref().unwrap().clone(), self.reader.as_ref().unwrap().clone(), chunks, ); + Box::new(reader) + } +} + +impl SectionBuilder for OCIRegBuilder { + fn can_handle(&mut self, node: &dyn RafsInode, _: &Path) -> bool { + node.is_reg() + } + + fn build(&self, inode: &dyn RafsInode, path: &Path) -> Result> { let mut header = Header::new_ustar(); header.set_entry_type(EntryType::file()); header.set_device_major(0).unwrap(); @@ -386,108 +437,138 @@ impl DefaultTarBuilder { header.set_mode(node.mode()); header.set_size(node.size()); - let mut extensions = Vec::with_capacity(3); - if let Some(extension) = self.set_path_for_pax_header(&mut header, path)? { + let mut extensions = Vec::with_capacity(2); + if let Some(extension) = PAXUtil::set_path(&mut header, path)? { extensions.extend(extension); } - - header.set_cksum(); - - if let Some(extension) = self.get_xattr_as_extensions(inode) { + if let Some(extension) = PAXUtil::get_xattr_as_extensions(inode) { extensions.extend(extension); } - let mut requirements = Vec::with_capacity(1); - if let Some(xattr) = - self.build_extension_section(header.path().unwrap().borrow(), extensions)? - { - requirements.push(xattr); + header.set_cksum(); + + let mut sections = Vec::with_capacity(2); + if let Some(ext_sect) = self.ext_builder.build(&header, extensions)? { + sections.push(ext_sect); } - Ok(TarSection { + let main_header = TarSection { header, - data: Box::new(data), - presections: requirements, - }) + data: Box::new(self.build_data(inode)), + }; + sections.push(main_header); + + Ok(sections) + } +} + +struct OCISymlinkBuilder { + pax_link_builder: Rc, +} + +impl OCISymlinkBuilder { + fn new(pax_link_builder: Rc) -> Self { + OCISymlinkBuilder { pax_link_builder } + } +} + +impl SectionBuilder for OCISymlinkBuilder { + fn can_handle(&mut self, node: &dyn RafsInode, _: &Path) -> bool { + node.is_symlink() } - fn build_symlink_section(&self, node: &dyn RafsInode, path: &Path) -> Result { + fn build(&self, node: &dyn RafsInode, path: &Path) -> Result> { let link = node.get_symlink().unwrap(); - self._build_link_section(EntryType::symlink(), node, path, &PathBuf::from(link)) + self.pax_link_builder + .build(EntryType::symlink(), node, path, &PathBuf::from(link)) } +} - fn build_link_section(&self, node: &dyn RafsInode, path: &Path) -> Result { - let link = self.links.get(&node.ino()).unwrap(); +struct OCIFifoBuilder { + pax_special_builder: Rc, +} + +impl OCIFifoBuilder { + fn new(pax_special_builder: Rc) -> Self { + OCIFifoBuilder { + pax_special_builder, + } + } +} - self._build_link_section(EntryType::hard_link(), node, path, link) +impl SectionBuilder for OCIFifoBuilder { + fn can_handle(&mut self, node: &dyn RafsInode, _: &Path) -> bool { + InodeWrapper::from_inode_info(node).is_fifo() } - fn build_fifo_section(&self, node: &dyn RafsInode, path: &Path) -> Result { - self.build_special_file_section(EntryType::Fifo, node, path) + fn build(&self, inode: &dyn RafsInode, path: &Path) -> Result> { + self.pax_special_builder + .build(EntryType::fifo(), inode, path) } +} + +struct OCICharBuilder { + pax_special_builder: Rc, +} - fn build_char_section(&self, node: &dyn RafsInode, path: &Path) -> Result { - self.build_special_file_section(EntryType::character_special(), node, path) +impl OCICharBuilder { + fn new(pax_special_builder: Rc) -> Self { + OCICharBuilder { + pax_special_builder, + } } +} - fn build_block_section(&self, node: &dyn RafsInode, path: &Path) -> Result { - self.build_special_file_section(EntryType::block_special(), node, path) +impl SectionBuilder for OCICharBuilder { + fn can_handle(&mut self, node: &dyn RafsInode, _: &Path) -> bool { + InodeWrapper::from_inode_info(node).is_chrdev() } - fn _build_link_section( - &self, - entry_type: EntryType, - inode: &dyn RafsInode, - path: &Path, - link: &Path, - ) -> Result { - let mut header = Header::new_ustar(); - header.set_entry_type(entry_type); - header.set_size(0); - header.set_device_major(0).unwrap(); - header.set_device_minor(0).unwrap(); + fn build(&self, inode: &dyn RafsInode, path: &Path) -> Result> { + self.pax_special_builder + .build(EntryType::character_special(), inode, path) + } +} - let node = InodeWrapper::from_inode_info(inode); - header.set_mtime(node.mtime()); - header.set_uid(node.uid() as u64); - header.set_gid(node.gid() as u64); - header.set_mode(node.mode()); +struct OCIBlockBuilder { + pax_special_builder: Rc, +} - let mut extensions = Vec::with_capacity(3); - if let Some(extension) = self.set_path_for_pax_header(&mut header, path)? { - extensions.extend(extension); - } - if let Some(extension) = self.set_link_for_pax_header(&mut header, link)? { - extensions.extend(extension); +impl OCIBlockBuilder { + fn new(pax_special_builder: Rc) -> Self { + OCIBlockBuilder { + pax_special_builder, } + } +} - header.set_cksum(); +impl SectionBuilder for OCIBlockBuilder { + fn can_handle(&mut self, node: &dyn RafsInode, _: &Path) -> bool { + InodeWrapper::from_inode_info(node).is_blkdev() + } - if let Some(extension) = self.get_xattr_as_extensions(inode) { - extensions.extend(extension); - } + fn build(&self, inode: &dyn RafsInode, path: &Path) -> Result> { + self.pax_special_builder + .build(EntryType::block_special(), inode, path) + } +} - let mut requirements = Vec::with_capacity(1); - if let Some(xattr) = - self.build_extension_section(header.path().unwrap().borrow(), extensions)? - { - requirements.push(xattr); - } +struct PAXSpecialSectionBuilder { + ext_builder: Rc, +} - Ok(TarSection { - header, - data: Box::new(io::empty()), - presections: requirements, - }) +impl PAXSpecialSectionBuilder { + fn new(ext_builder: Rc) -> Self { + PAXSpecialSectionBuilder { ext_builder } } - fn build_special_file_section( + fn build( &self, entry_type: EntryType, inode: &dyn RafsInode, path: &Path, - ) -> Result { + ) -> Result> { let mut header = Header::new_ustar(); header.set_entry_type(entry_type); @@ -498,47 +579,60 @@ impl DefaultTarBuilder { header.set_mode(node.mode()); header.set_size(node.size()); - let dev_id = inode.rdev() as u64; - let dev_major = ((dev_id >> 32) & 0xffff_f000) | ((dev_id >> 8) & 0x0000_0fff); - header.set_device_major(dev_major as u32)?; + let dev_id = self.cal_dev(inode.rdev() as u64); + header.set_device_major(dev_id.0)?; + header.set_device_minor(dev_id.1)?; - let dev_minor = ((dev_id >> 12) & 0xffff_ff00) | ((dev_id) & 0x0000_00ff); - header.set_device_minor(dev_minor as u32)?; - - let mut extensions = Vec::with_capacity(3); - if let Some(extension) = self.set_path_for_pax_header(&mut header, path)? { + let mut extensions = Vec::with_capacity(2); + if let Some(extension) = PAXUtil::set_path(&mut header, path)? { extensions.extend(extension); } - - header.set_cksum(); - - if let Some(extension) = self.get_xattr_as_extensions(inode) { + if let Some(extension) = PAXUtil::get_xattr_as_extensions(inode) { extensions.extend(extension); } - let mut requirements = Vec::with_capacity(1); - if let Some(xattr) = - self.build_extension_section(header.path().unwrap().borrow(), extensions)? - { - requirements.push(xattr); + header.set_cksum(); + + let mut sections = Vec::with_capacity(2); + if let Some(ext_sect) = self.ext_builder.build(&header, extensions)? { + sections.push(ext_sect); } - Ok(TarSection { + let main_header = TarSection { header, data: Box::new(io::empty()), - presections: requirements, - }) + }; + sections.push(main_header); + + Ok(sections) + } + + fn cal_dev(&self, dev_id: u64) -> (u32, u32) { + let major = ((dev_id >> 32) & 0xffff_f000) | ((dev_id >> 8) & 0x0000_0fff); + let minor = ((dev_id >> 12) & 0xffff_ff00) | ((dev_id) & 0x0000_00ff); + + (major as u32, minor as u32) + } +} + +struct PAXExtensionSectionBuilder {} + +impl PAXExtensionSectionBuilder { + fn new() -> Self { + PAXExtensionSectionBuilder {} } - fn build_extension_section( + fn build( &self, - path: &Path, - mut extensions: Vec<(Vec, Vec)>, + header: &Header, + extensions: Vec<(Vec, Vec)>, ) -> Result> { if extensions.len() == 0 { return Ok(None); } + let path = header.path().unwrap().into_owned(); + let mut header = Header::new_ustar(); header.set_entry_type(EntryType::XHeader); header.set_mode(0o644); @@ -546,26 +640,11 @@ impl DefaultTarBuilder { header.set_gid(0); header.set_mtime(0); - extensions.sort_by(|(k1, _), (k2, _)| { - let k1 = str::from_utf8(k1).unwrap(); - let k2 = str::from_utf8(k2).unwrap(); - - k1.cmp(k2) - }); - let data: Vec = extensions - .into_iter() - .map(|(k, v)| self.build_pax_record(&k, &v)) - .flatten() - .collect(); + let data = self.build_data(extensions); header.set_size(data.len() as u64); - let mut path = self.build_pax_name(path); - let max_len = header.as_old().name.len(); - if path.as_os_str().len() > max_len { - path = self.truncate_path(&path, max_len)?; - } header - .set_path(&path) + .set_path(&self.build_pax_name(&path, header.as_old().name.len())?) .map_err(|err| anyhow!("fail to set path for pax section, error {}", err))?; header.set_cksum(); @@ -573,65 +652,35 @@ impl DefaultTarBuilder { Ok(Some(TarSection { header, data: Box::new(Cursor::new(data)), - presections: Vec::new(), })) } - fn build_long_section(&self, entry_type: EntryType, mut data: Vec) -> TarSection { - // gnu requires, especially for long-link section and long-name section - let mut header = Header::new_gnu(); + fn build_data(&self, mut extensions: Vec<(Vec, Vec)>) -> Vec { + extensions.sort_by(|(k1, _), (k2, _)| { + let k1 = str::from_utf8(k1).unwrap(); + let k2 = str::from_utf8(k2).unwrap(); + k1.cmp(k2) + }); - data.push(0x00); - header.set_size(data.len() as u64); - let data = Cursor::new(data); + extensions + .into_iter() + .map(|(k, v)| self.build_pax_record(&k, &v)) + .flatten() + .collect() + } - let title = b"././@LongLink"; - header.as_gnu_mut().unwrap().name[..title.len()].clone_from_slice(&title[..]); + fn build_pax_name(&self, path: &Path, max_len: usize) -> Result { + let filename = path.file_name().unwrap().to_owned(); - header.set_entry_type(entry_type); - header.set_mode(0o644); - header.set_uid(0); - header.set_gid(0); - header.set_mtime(0); - - header.set_cksum(); - - TarSection { - header, - data: Box::new(data), - presections: Vec::new(), - } - } - - fn get_xattr_as_extensions(&self, inode: &dyn RafsInode) -> Option, Vec)>> { - if !inode.has_xattr() { - return None; - } - - let keys = inode.get_xattrs().unwrap(); - let mut extensions = Vec::with_capacity(keys.len()); - - for key in keys { - let value = inode - .get_xattr(OsStr::from_bytes(&key)) - .unwrap() - .unwrap_or_default(); + let mut path = path.to_path_buf(); + path.set_file_name("PaxHeaders.0"); + let mut path = path.join(filename); - let key = Vec::from(PAX_PREFIX.to_owned()) - .into_iter() - .chain(key.into_iter()) - .collect(); - extensions.push((key, value)); + if path.as_os_str().len() > max_len { + path = Util::truncate_path(&path, max_len)?; } - Some(extensions) - } - - fn build_pax_name(&self, path: &Path) -> PathBuf { - let mut path = path.to_path_buf(); - let filename = path.file_name().unwrap().to_owned(); - path.set_file_name("PaxHeaders.0"); - path.join(filename) + Ok(path) } fn build_pax_record(&self, k: &[u8], v: &[u8]) -> Vec { @@ -658,51 +707,69 @@ impl DefaultTarBuilder { record } +} - fn set_path_for_pax_header( +struct PAXLinkBuilder { + ext_builder: Rc, +} + +impl PAXLinkBuilder { + fn new(ext_builder: Rc) -> Self { + PAXLinkBuilder { ext_builder } + } + + fn build( &self, - header: &mut Header, + entry_type: EntryType, + inode: &dyn RafsInode, path: &Path, - ) -> Result, Vec)>>> { - let path = self - .normalize_path(&path, header.entry_type().is_dir()) - .map_err(|err| anyhow!("fail to normalize path, error {}", err))?; + link: &Path, + ) -> Result> { + let mut header = Header::new_ustar(); + header.set_entry_type(entry_type); + header.set_size(0); + header.set_device_major(0).unwrap(); + header.set_device_minor(0).unwrap(); - let max_len = header.as_old().name.len(); - if path.as_os_str().len() <= max_len { - return header - .set_path(path) - .map_err(|err| anyhow!("fail to set short path for pax header, error {}", err)) - .map(|_| None); + let node = InodeWrapper::from_inode_info(inode); + header.set_mtime(node.mtime()); + header.set_uid(node.uid() as u64); + header.set_gid(node.gid() as u64); + header.set_mode(node.mode()); + + let mut extensions = Vec::with_capacity(3); + if let Some(extension) = PAXUtil::set_path(&mut header, path)? { + extensions.extend(extension); + } + if let Some(extension) = PAXUtil::set_link(&mut header, link)? { + extensions.extend(extension); + } + if let Some(extension) = PAXUtil::get_xattr_as_extensions(inode) { + extensions.extend(extension); } - let extension = vec![( - "path".to_owned().into_bytes(), - path.to_owned().into_os_string().into_vec(), - )]; + header.set_cksum(); - let path = self - .truncate_path(&path, max_len) - .map_err(|err| anyhow!("fail to truncate path for pax header, error {}", err))?; + let mut sections = Vec::with_capacity(2); + if let Some(ext_sect) = self.ext_builder.build(&header, extensions)? { + sections.push(ext_sect); + } - header.set_path(&path).map_err(|err| { - anyhow!( - "fail to set header path again for {:?}, error {}", - path, - err - ) - })?; + let main_header = TarSection { + header, + data: Box::new(io::empty()), + }; + sections.push(main_header); - Ok(Some(extension)) + Ok(sections) } +} - fn set_path_for_gnu_header( - &self, - header: &mut Header, - path: &Path, - ) -> Result> { - let path = self - .normalize_path(&path, header.entry_type().is_dir()) +struct GNUUtil {} + +impl GNUUtil { + fn set_path(header: &mut Header, path: &Path) -> Result> { + let path = Util::normalize_path(&path, header.entry_type().is_dir()) .map_err(|err| anyhow!("fail to normalize path, error {}", err))?; let rs = match header.set_path(&path) { @@ -710,7 +777,7 @@ impl DefaultTarBuilder { Err(err) => Err(anyhow!("fail to set path for header, error {}", err)), }; - let path = match self.truncate_path(&path, header.as_old().name.len()) { + let path = match Util::truncate_path(&path, header.as_old().name.len()) { Ok(path) => path, Err(_) => return rs, }; @@ -723,37 +790,67 @@ impl DefaultTarBuilder { ) })?; - Ok(Some(self.build_long_section( + Ok(Some(Self::build_long_section( EntryType::GNULongName, path.into_os_string().into_vec(), ))) } - // path is required longer than max_len - fn truncate_path(&self, path: &Path, max_len: usize) -> Result { - let path = path.as_os_str().as_bytes(); - if path.len() < max_len { - bail!("path is shorter than limit") + fn build_long_section(entry_type: EntryType, mut data: Vec) -> TarSection { + // gnu requires, especially for long-link section and long-name section + let mut header = Header::new_gnu(); + + data.push(0x00); + header.set_size(data.len() as u64); + let data = Cursor::new(data); + + let title = b"././@LongLink"; + header.as_gnu_mut().unwrap().name[..title.len()].clone_from_slice(&title[..]); + + header.set_entry_type(entry_type); + header.set_mode(0o644); + header.set_uid(0); + header.set_gid(0); + header.set_mtime(0); + + header.set_cksum(); + + TarSection { + header, + data: Box::new(data), + } + } +} + +struct PAXUtil {} + +impl PAXUtil { + fn get_xattr_as_extensions(inode: &dyn RafsInode) -> Option, Vec)>> { + if !inode.has_xattr() { + return None; } - let path = match str::from_utf8(&path[..max_len]) { - Ok(s) => Ok(s), - Err(err) => str::from_utf8(&path[..err.valid_up_to()]) - .map_err(|err| anyhow!("fail to convert bytes to utf8 str, error: {}", err)), - }?; + let keys = inode.get_xattrs().unwrap(); + let mut extensions = Vec::with_capacity(keys.len()); - Ok(PathBuf::from(path)) + for key in keys { + let value = inode + .get_xattr(OsStr::from_bytes(&key)) + .unwrap() + .unwrap_or_default(); + + let key = Vec::from(PAX_PREFIX.to_owned()) + .into_iter() + .chain(key.into_iter()) + .collect(); + extensions.push((key, value)); + } + + Some(extensions) } - /// If link size is too long, a long-link-header is returned as precondition of input - /// header. - fn set_link_for_pax_header( - &self, - header: &mut Header, - path: &Path, - ) -> Result, Vec)>>> { - let path = self - .normalize_path(path, header.entry_type().is_dir()) + fn set_link(header: &mut Header, path: &Path) -> Result, Vec)>>> { + let path = Util::normalize_path(path, header.entry_type().is_dir()) .map_err(|err| anyhow!("fail to normalize path, error {}", err))?; let max_len = header.as_old().linkname.len(); @@ -769,8 +866,7 @@ impl DefaultTarBuilder { path.to_owned().into_os_string().into_vec(), )]; - let path = self - .truncate_path(&path, max_len) + let path = Util::truncate_path(&path, max_len) .map_err(|err| anyhow!("fail to truncate path for pax header, error {}", err))?; header.set_link_name(&path).map_err(|err| { @@ -784,7 +880,42 @@ impl DefaultTarBuilder { Ok(Some(extension)) } - fn normalize_path(&self, path: &Path, is_dir: bool) -> Result { + fn set_path(header: &mut Header, path: &Path) -> Result, Vec)>>> { + let path = Util::normalize_path(&path, header.entry_type().is_dir()) + .map_err(|err| anyhow!("fail to normalize path, error {}", err))?; + + let max_len = header.as_old().name.len(); + if path.as_os_str().len() <= max_len { + return header + .set_path(path) + .map_err(|err| anyhow!("fail to set short path for pax header, error {}", err)) + .map(|_| None); + } + + let extension = vec![( + "path".to_owned().into_bytes(), + path.to_owned().into_os_string().into_vec(), + )]; + + let path = Util::truncate_path(&path, max_len) + .map_err(|err| anyhow!("fail to truncate path for pax header, error {}", err))?; + + header.set_path(&path).map_err(|err| { + anyhow!( + "fail to set header path again for {:?}, error {}", + path, + err + ) + })?; + + Ok(Some(extension)) + } +} + +struct Util {} + +impl Util { + fn normalize_path(path: &Path, is_dir: bool) -> Result { fn has_trailing_slash(p: &Path) -> bool { p.as_os_str().as_bytes().last() == Some(&b'/') } @@ -808,90 +939,155 @@ impl DefaultTarBuilder { Ok(normalized) } - fn is_link_section(&mut self, node: &dyn RafsInode, path: &Path) -> bool { - if !node.is_hardlink() || node.is_dir() { - return false; - } - - let is_appeared = self.links.contains_key(&node.ino()); - if !is_appeared { - self.links - .insert(node.ino(), path.to_path_buf().into_boxed_path()); + // path is required longer than max_len + fn truncate_path(path: &Path, max_len: usize) -> Result { + let path = path.as_os_str().as_bytes(); + if path.len() < max_len { + bail!("path is shorter than limit") } - return is_appeared; - } - - fn is_symlink_section(&self, node: &dyn RafsInode) -> bool { - node.is_symlink() - } + let path = match str::from_utf8(&path[..max_len]) { + Ok(s) => Ok(s), + Err(err) => str::from_utf8(&path[..err.valid_up_to()]) + .map_err(|err| anyhow!("fail to convert bytes to utf8 str, error: {}", err)), + }?; - fn is_socket_section(&self, node: &dyn RafsInode) -> bool { - InodeWrapper::from_inode_info(node).is_sock() + Ok(PathBuf::from(path)) } +} - fn is_dir_section(&self, node: &dyn RafsInode) -> bool { - node.is_dir() - } +struct OCITarBuilder { + writer: Builder, + builders: Vec>, +} - fn is_reg_section(&self, node: &dyn RafsInode) -> bool { - node.is_reg() - } +impl OCITarBuilder { + fn new(meta: &RafsSuper, blob_dir: Option<&Path>, output_path: &Path) -> Result { + let writer = Builder::new( + OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .read(false) + .open(output_path) + .map_err(|err| { + anyhow!( + "fail to open output file {:?}, error: {:?}", + output_path, + err + ) + })?, + ); - fn is_fifo_section(&self, node: &dyn RafsInode) -> bool { - InodeWrapper::from_inode_info(node).is_fifo() - } + let builders = Self::builders(meta, blob_dir)?; - fn is_char_section(&self, node: &dyn RafsInode) -> bool { - InodeWrapper::from_inode_info(node).is_chrdev() + Ok(Self { builders, writer }) } - fn is_block_section(&self, node: &dyn RafsInode) -> bool { - InodeWrapper::from_inode_info(node).is_blkdev() + fn builders(meta: &RafsSuper, blob_dir: Option<&Path>) -> Result>> { + let pax_ext_builder = Rc::new(PAXExtensionSectionBuilder::new()); + let pax_link_builder = Rc::new(PAXLinkBuilder::new(pax_ext_builder.clone())); + let pax_special_builder = Rc::new(PAXSpecialSectionBuilder::new(pax_ext_builder.clone())); + + let sock_builder = OCISocketBuilder::new(); + let link_builder = OCILinkBuilder::new(pax_link_builder.clone()); + let symlink_builder = OCISymlinkBuilder::new(pax_link_builder.clone()); + let whiteout_builder = OCIWhiteoutBuilder::new(); + let dir_builder = OCIDirBuilder::new(pax_ext_builder); + let fifo_builder = OCIFifoBuilder::new(pax_special_builder.clone()); + let char_builder = OCICharBuilder::new(pax_special_builder.clone()); + let block_builder = OCIBlockBuilder::new(pax_special_builder); + let reg_builder = Self::reg_builder(meta, blob_dir)?; + + let builders = vec![ + Box::new(sock_builder) as Box, + Box::new(link_builder), + Box::new(whiteout_builder), + Box::new(dir_builder), + Box::new(reg_builder), + Box::new(symlink_builder), + Box::new(fifo_builder), + Box::new(char_builder), + Box::new(block_builder), + ]; + + Ok(builders) } - fn is_white_out_section(&self, path: &Path) -> bool { - path.file_name() - .unwrap() - .to_str() - .unwrap() - .starts_with(OCISPEC_WHITEOUT_PREFIX) - } -} + fn reg_builder(meta: &RafsSuper, blob_dir: Option<&Path>) -> Result { + let blob = meta.superblock.get_blob_infos().pop(); + let (reader, compressor) = match blob { + None => (None, None), + Some(ref blob) => { + if blob_dir.is_none() { + bail!("miss blob dir") + } -impl TarBuilder for DefaultTarBuilder { - fn append(&mut self, inode: &dyn RafsInode, path: &Path) -> Result<()> { - let tar_sect = match inode { - node if self.is_socket_section(node) => return Ok(()), + let reader = Self::blob_reader(&blob_dir.unwrap().join(blob.blob_id()))?; + let compressor = blob.compressor(); - node if self.is_link_section(node, path) => self.build_link_section(node, path)?, + (Some(reader), Some(compressor)) + } + }; - node if self.is_white_out_section(path) => self.build_whiteout_section(node, path)?, + let pax_ext_builder = Rc::new(PAXExtensionSectionBuilder::new()); - node if self.is_dir_section(node) => self.build_dir_section(node, path)?, + Ok(OCIRegBuilder::new( + pax_ext_builder.clone(), + reader, + compressor, + )) + } - node if self.is_reg_section(node) => self.build_reg_section(node, path)?, + fn blob_reader(blob_path: &Path) -> Result> { + let config = LocalFsConfig { + blob_file: blob_path.to_str().unwrap().to_owned(), + readahead: false, + readahead_sec: Default::default(), + dir: Default::default(), + alt_dirs: Default::default(), + }; + let config = serde_json::to_value(config).map_err(|err| { + anyhow!( + "fail to create local backend config for {:?}, error: {:?}", + blob_path, + err + ) + })?; - node if self.is_symlink_section(node) => self.build_symlink_section(node, path)?, + let backend = LocalFs::new(config, Some("decompressor")).map_err(|err| { + anyhow!( + "fail to create local backend for {:?}, error: {:?}", + blob_path, + err + ) + })?; - node if self.is_fifo_section(node) => self.build_fifo_section(node, path)?, + let reader = backend + .get_reader("") + .map_err(|err| anyhow!("fail to get reader, error {:?}", err))?; - node if self.is_char_section(node) => self.build_char_section(node, path)?, + Ok(reader) + } +} - node if self.is_block_section(node) => self.build_block_section(node, path)?, +impl TarBuilder for OCITarBuilder { + fn append(&mut self, inode: &dyn RafsInode, path: &Path) -> Result<()> { + let mut is_known_type = true; - _ => bail!("unknow inde type"), - }; + for builder in &mut self.builders { + if builder.can_handle(inode, path) { + is_known_type = false; - for sect in tar_sect.presections { - self.writer - .append(§.header, sect.data) - .map_err(|err| anyhow!("fail to append inode {:?}, error: {}", path, err))?; + for sect in builder.build(inode, path)? { + self.writer.append(§.header, sect.data)?; + } + } } - self.writer - .append(&tar_sect.header, tar_sect.data) - .map_err(|err| anyhow!("fail to append inode {:?}, error: {}", path, err))?; + if is_known_type { + bail!("node {:?} can not be decompressed", path) + } Ok(()) } diff --git a/src/bin/nydus-image/main.rs b/src/bin/nydus-image/main.rs index 259b6a391fe..051e6a99735 100644 --- a/src/bin/nydus-image/main.rs +++ b/src/bin/nydus-image/main.rs @@ -43,7 +43,7 @@ use crate::core::context::{ use crate::core::node::{self, WhiteoutSpec}; use crate::core::prefetch::Prefetch; use crate::core::tree; -use crate::decompress::DefaultDecompressor; +use crate::decompress::OCIDecompressor; use crate::merge::Merger; use crate::trace::{EventTracerClass, TimingTracerClass, TraceClass}; use crate::validator::Validator; @@ -523,11 +523,11 @@ fn prepare_cmd_args(bti_string: String) -> ArgMatches<'static> { .required(true) .takes_value(true)) .arg( - Arg::with_name("blob") - .long("blob") + Arg::with_name("blob_dir") + .long("blob_dir") .short("b") - .help("path to blob file") - .required(true) + .help("path to folder that holds blob file") + .required(false) .takes_value(true) ) .arg( @@ -799,10 +799,10 @@ impl Command { fn decompress(args: &clap::ArgMatches) -> Result<()> { let bootstrap = args.value_of("bootstrap").expect("pass in bootstrap"); - let blob = args.value_of("blob").expect("pass in blob"); + let blob_dir = args.value_of("blob_dir"); let output = args.value_of("output").expect("pass in output"); - let decompressor = Box::new(DefaultDecompressor::new(bootstrap, blob, output).map_err( + let decompressor = Box::new(OCIDecompressor::new(bootstrap, blob_dir, output).map_err( |err| { error!("fail to create decompressor, error: {}", err); anyhow!("fail to create decompressor, error: {}", err) diff --git a/tests/builder.rs b/tests/builder.rs index 56a102f613d..6b023c9c93d 100644 --- a/tests/builder.rs +++ b/tests/builder.rs @@ -453,7 +453,7 @@ impl<'a> Builder<'a> { pub fn decompress(&self, bootstrap: &str, blob: &str, output: &str) { let cmd = format!( - "{:?} decompress --bootstrap {:?} --blob {:?} --output {:?}", + "{:?} decompress --bootstrap {:?} --blob_dir {:?} --output {:?}", self.builder, self.work_dir.join(bootstrap), self.work_dir.join(blob), From d771c12fa9a9fbff2a0ac0770d9ed4305071fd61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B3=B0=E5=8F=8B?= Date: Thu, 23 Jun 2022 14:05:34 +0800 Subject: [PATCH 4/7] refact: try best to restore same bytes as original OCI image. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 泰友 --- src/bin/nydus-image/decompress.rs | 189 ++++++++++++++-------- src/bin/nydus-image/main.rs | 10 +- tests/builder.rs | 26 ++- tests/smoke.rs | 16 +- tests/texture/directory/decompress.result | 40 +++++ 5 files changed, 203 insertions(+), 78 deletions(-) create mode 100644 tests/texture/directory/decompress.result diff --git a/src/bin/nydus-image/decompress.rs b/src/bin/nydus-image/decompress.rs index e9c2603ecb9..4443248925d 100644 --- a/src/bin/nydus-image/decompress.rs +++ b/src/bin/nydus-image/decompress.rs @@ -15,14 +15,15 @@ use std::{ }; use anyhow::Result; +use nydus_api::http::LocalFsConfig; use nydus_rafs::{ metadata::{layout::RAFS_ROOT_INODE, RafsInode, RafsMode, RafsSuper}, RafsIoReader, }; use nydus_utils::compress::{self, Algorithm}; use storage::{ - backend::{localfs::LocalFs, BlobBackend, BlobReader, LocalFsConfig}, - device::BlobChunkInfo, + backend::{localfs::LocalFs, BlobBackend, BlobReader}, + device::{BlobChunkInfo, BlobInfo}, utils::alloc_buf, }; use tar::{Builder, EntryType, Header}; @@ -41,23 +42,24 @@ pub trait Decompressor { /// A decompressor with the ability to convert bootstrap file and blob file to tar pub struct OCIDecompressor { bootstrap: Box, - blob_dir: Option>, + blob: Option>, output: Box, + + builder_factory: OCITarBuilderFactory, } impl OCIDecompressor { - pub fn new(bootstrap: &str, blob_dir: Option<&str>, output: &str) -> Result { + pub fn new(bootstrap: &str, blob: Option<&str>, output: &str) -> Result { let bootstrap = PathBuf::from(bootstrap).into_boxed_path(); let output = PathBuf::from(output).into_boxed_path(); + let blob = blob.map(|v| PathBuf::from(v).into_boxed_path()); - let blob_dir = match blob_dir { - Some(dir) => Some(PathBuf::from(dir).canonicalize()?.into_boxed_path()), - None => None, - }; + let builder_factory = OCITarBuilderFactory::new(); Ok(OCIDecompressor { + builder_factory, bootstrap, - blob_dir, + blob, output, }) } @@ -183,16 +185,14 @@ impl Decompressor for OCIDecompressor { fn decompress(&self) -> Result<()> { info!( "default decompressor, bootstrap file: {:?}, blob file: {:?}, output file: {:?}", - self.bootstrap, self.blob_dir, self.output + self.bootstrap, self.blob, self.output ); let rafs = self.load_rafs()?; - let mut builder = Box::new(OCITarBuilder::new( - &rafs, - self.blob_dir.as_ref().map(|dir| dir.as_ref()), - &*self.output, - )?) as Box; + let mut builder = + self.builder_factory + .create(&rafs, self.blob.as_deref(), &*self.output)?; for (node, path) in self.iterator(&rafs) { if node.name() == OsStr::from_bytes(ROOT_PATH_NAME) { @@ -303,7 +303,7 @@ impl SectionBuilder for OCIWhiteoutBuilder { header.set_mtime(node.mtime()); header.set_uid(node.uid() as u64); header.set_gid(node.gid() as u64); - header.set_mode(node.mode()); + header.set_mode(Util::mask_mode(node.mode())); // Rafs loses the access time and change time. // It's required by certain tools, such 7-zip. @@ -354,7 +354,7 @@ impl SectionBuilder for OCIDirBuilder { header.set_mtime(node.mtime()); header.set_uid(node.uid() as u64); header.set_gid(node.gid() as u64); - header.set_mode(node.mode()); + header.set_mode(Util::mask_mode(node.mode())); let mut extensions = Vec::with_capacity(2); if let Some(extension) = PAXUtil::set_path(&mut header, path)? { @@ -434,7 +434,7 @@ impl SectionBuilder for OCIRegBuilder { header.set_mtime(node.mtime()); header.set_uid(node.uid() as u64); header.set_gid(node.gid() as u64); - header.set_mode(node.mode()); + header.set_mode(Util::mask_mode(node.mode())); header.set_size(node.size()); let mut extensions = Vec::with_capacity(2); @@ -576,7 +576,7 @@ impl PAXSpecialSectionBuilder { header.set_mtime(node.mtime()); header.set_uid(node.uid() as u64); header.set_gid(node.gid() as u64); - header.set_mode(node.mode()); + header.set_mode(Util::mask_mode(node.mode())); header.set_size(node.size()); let dev_id = self.cal_dev(inode.rdev() as u64); @@ -735,7 +735,7 @@ impl PAXLinkBuilder { header.set_mtime(node.mtime()); header.set_uid(node.uid() as u64); header.set_gid(node.gid() as u64); - header.set_mode(node.mode()); + header.set_mode(Util::mask_mode(node.mode())); let mut extensions = Vec::with_capacity(3); if let Some(extension) = PAXUtil::set_path(&mut header, path)? { @@ -769,7 +769,7 @@ struct GNUUtil {} impl GNUUtil { fn set_path(header: &mut Header, path: &Path) -> Result> { - let path = Util::normalize_path(&path, header.entry_type().is_dir()) + let path = Util::normalize_path(&path) .map_err(|err| anyhow!("fail to normalize path, error {}", err))?; let rs = match header.set_path(&path) { @@ -850,7 +850,7 @@ impl PAXUtil { } fn set_link(header: &mut Header, path: &Path) -> Result, Vec)>>> { - let path = Util::normalize_path(path, header.entry_type().is_dir()) + let path = Util::normalize_path(path) .map_err(|err| anyhow!("fail to normalize path, error {}", err))?; let max_len = header.as_old().linkname.len(); @@ -881,7 +881,7 @@ impl PAXUtil { } fn set_path(header: &mut Header, path: &Path) -> Result, Vec)>>> { - let path = Util::normalize_path(&path, header.entry_type().is_dir()) + let path = Util::normalize_path(&path) .map_err(|err| anyhow!("fail to normalize path, error {}", err))?; let max_len = header.as_old().name.len(); @@ -915,12 +915,11 @@ impl PAXUtil { struct Util {} impl Util { - fn normalize_path(path: &Path, is_dir: bool) -> Result { - fn has_trailing_slash(p: &Path) -> bool { + fn normalize_path(path: &Path) -> Result { + fn end_with_slash(p: &Path) -> bool { p.as_os_str().as_bytes().last() == Some(&b'/') } - // remove root let mut normalized = if path.has_root() { path.strip_prefix("/") .map_err(|err| anyhow!("fail to strip prefix /, error {}", err))? @@ -929,10 +928,7 @@ impl Util { path.to_path_buf() }; - // handle trailing slash - if is_dir { - normalized.push(""); - } else if has_trailing_slash(path) { + if end_with_slash(path) { normalized.set_file_name(normalized.file_name().unwrap().to_owned()) } @@ -954,16 +950,53 @@ impl Util { Ok(PathBuf::from(path)) } -} -struct OCITarBuilder { - writer: Builder, - builders: Vec>, + // Common Unix mode constants; these are not defined in any common tar standard. + // + // c_ISDIR = 040000 // Directory + // c_ISFIFO = 010000 // FIFO + // c_ISREG = 0100000 // Regular file + // c_ISLNK = 0120000 // Symbolic link + // c_ISBLK = 060000 // Block special file + // c_ISCHR = 020000 // Character special file + // c_ISSOCK = 0140000 // Socket + // + // Although many readers bear it, such as Go standard library and tar tool in ubuntu, truncate to last four bytes. The four consists of below: + // + // c_ISUID = 04000 // Set uid + // c_ISGID = 02000 // Set gid + // c_ISVTX = 01000 // Sticky bit + // MODE_PERM = 0777 // Owner:Group:Other R/W + fn mask_mode(st_mode: u32) -> u32 { + st_mode & 0o7777 + } } -impl OCITarBuilder { - fn new(meta: &RafsSuper, blob_dir: Option<&Path>, output_path: &Path) -> Result { - let writer = Builder::new( +struct OCITarBuilderFactory {} + +impl OCITarBuilderFactory { + fn new() -> Self { + OCITarBuilderFactory {} + } + + fn create( + &self, + meta: &RafsSuper, + blob_path: Option<&Path>, + output_path: &Path, + ) -> Result> { + let writer = self.create_writer(output_path)?; + + let blob = meta.superblock.get_blob_infos().pop(); + let builders = self.create_builders(blob, blob_path)?; + + let builder = OCITarBuilder::new(builders, writer); + + Ok(Box::new(builder) as Box) + } + + fn create_writer(&self, output_path: &Path) -> Result> { + let builder = Builder::new( OpenOptions::new() .write(true) .create(true) @@ -979,29 +1012,34 @@ impl OCITarBuilder { })?, ); - let builders = Self::builders(meta, blob_dir)?; - - Ok(Self { builders, writer }) + Ok(builder) } - fn builders(meta: &RafsSuper, blob_dir: Option<&Path>) -> Result>> { - let pax_ext_builder = Rc::new(PAXExtensionSectionBuilder::new()); - let pax_link_builder = Rc::new(PAXLinkBuilder::new(pax_ext_builder.clone())); - let pax_special_builder = Rc::new(PAXSpecialSectionBuilder::new(pax_ext_builder.clone())); - + fn create_builders( + &self, + blob: Option>, + blob_path: Option<&Path>, + ) -> Result>> { + // PAX basic builders + let ext_builder = Rc::new(PAXExtensionSectionBuilder::new()); + let link_builder = Rc::new(PAXLinkBuilder::new(ext_builder.clone())); + let special_builder = Rc::new(PAXSpecialSectionBuilder::new(ext_builder.clone())); + + // OCI builders let sock_builder = OCISocketBuilder::new(); - let link_builder = OCILinkBuilder::new(pax_link_builder.clone()); - let symlink_builder = OCISymlinkBuilder::new(pax_link_builder.clone()); + let hard_link_builder = OCILinkBuilder::new(link_builder.clone()); + let symlink_builder = OCISymlinkBuilder::new(link_builder.clone()); let whiteout_builder = OCIWhiteoutBuilder::new(); - let dir_builder = OCIDirBuilder::new(pax_ext_builder); - let fifo_builder = OCIFifoBuilder::new(pax_special_builder.clone()); - let char_builder = OCICharBuilder::new(pax_special_builder.clone()); - let block_builder = OCIBlockBuilder::new(pax_special_builder); - let reg_builder = Self::reg_builder(meta, blob_dir)?; + let dir_builder = OCIDirBuilder::new(ext_builder); + let fifo_builder = OCIFifoBuilder::new(special_builder.clone()); + let char_builder = OCICharBuilder::new(special_builder.clone()); + let block_builder = OCIBlockBuilder::new(special_builder); + let reg_builder = self.create_reg_builder(blob, blob_path)?; + // The order counts. let builders = vec![ Box::new(sock_builder) as Box, - Box::new(link_builder), + Box::new(hard_link_builder), Box::new(whiteout_builder), Box::new(dir_builder), Box::new(reg_builder), @@ -1014,32 +1052,33 @@ impl OCITarBuilder { Ok(builders) } - fn reg_builder(meta: &RafsSuper, blob_dir: Option<&Path>) -> Result { - let blob = meta.superblock.get_blob_infos().pop(); + fn create_reg_builder( + &self, + blob: Option>, + blob_path: Option<&Path>, + ) -> Result { let (reader, compressor) = match blob { None => (None, None), Some(ref blob) => { - if blob_dir.is_none() { - bail!("miss blob dir") + if blob_path.is_none() { + bail!("miss blob path") } - let reader = Self::blob_reader(&blob_dir.unwrap().join(blob.blob_id()))?; + let reader = self.create_blob_reader(blob_path.unwrap())?; let compressor = blob.compressor(); (Some(reader), Some(compressor)) } }; - let pax_ext_builder = Rc::new(PAXExtensionSectionBuilder::new()); - Ok(OCIRegBuilder::new( - pax_ext_builder.clone(), + Rc::new(PAXExtensionSectionBuilder::new()), reader, compressor, )) } - fn blob_reader(blob_path: &Path) -> Result> { + fn create_blob_reader(&self, blob_path: &Path) -> Result> { let config = LocalFsConfig { blob_file: blob_path.to_str().unwrap().to_owned(), readahead: false, @@ -1071,18 +1110,34 @@ impl OCITarBuilder { } } +struct OCITarBuilder { + writer: Builder, + builders: Vec>, +} + +impl OCITarBuilder { + fn new(builders: Vec>, writer: Builder) -> Self { + Self { builders, writer } + } +} + impl TarBuilder for OCITarBuilder { fn append(&mut self, inode: &dyn RafsInode, path: &Path) -> Result<()> { let mut is_known_type = true; for builder in &mut self.builders { - if builder.can_handle(inode, path) { - is_known_type = false; + // Useless one, just go !!!!! + if !builder.can_handle(inode, path) { + continue; + } - for sect in builder.build(inode, path)? { - self.writer.append(§.header, sect.data)?; - } + for sect in builder.build(inode, path)? { + self.writer.append(§.header, sect.data)?; } + + is_known_type = false; + + break; } if is_known_type { diff --git a/src/bin/nydus-image/main.rs b/src/bin/nydus-image/main.rs index 051e6a99735..8fc890bf93d 100644 --- a/src/bin/nydus-image/main.rs +++ b/src/bin/nydus-image/main.rs @@ -523,10 +523,10 @@ fn prepare_cmd_args(bti_string: String) -> ArgMatches<'static> { .required(true) .takes_value(true)) .arg( - Arg::with_name("blob_dir") - .long("blob_dir") + Arg::with_name("blob") + .long("blob") .short("b") - .help("path to folder that holds blob file") + .help("path to blob file") .required(false) .takes_value(true) ) @@ -799,10 +799,10 @@ impl Command { fn decompress(args: &clap::ArgMatches) -> Result<()> { let bootstrap = args.value_of("bootstrap").expect("pass in bootstrap"); - let blob_dir = args.value_of("blob_dir"); + let blob = args.value_of("blob"); let output = args.value_of("output").expect("pass in output"); - let decompressor = Box::new(OCIDecompressor::new(bootstrap, blob_dir, output).map_err( + let decompressor = Box::new(OCIDecompressor::new(bootstrap, blob, output).map_err( |err| { error!("fail to create decompressor, error: {}", err); anyhow!("fail to create decompressor, error: {}", err) diff --git a/tests/builder.rs b/tests/builder.rs index 6b023c9c93d..f10c2ee4d7d 100644 --- a/tests/builder.rs +++ b/tests/builder.rs @@ -453,12 +453,34 @@ impl<'a> Builder<'a> { pub fn decompress(&self, bootstrap: &str, blob: &str, output: &str) { let cmd = format!( - "{:?} decompress --bootstrap {:?} --blob_dir {:?} --output {:?}", + "{:?} decompress --bootstrap {:?} --blob {:?} --output {:?}", self.builder, self.work_dir.join(bootstrap), self.work_dir.join(blob), self.work_dir.join(output) ); - exec(&cmd, false).unwrap(); + + exec(&cmd, false, b"").unwrap(); + } + + pub fn compress(&mut self, compressor: &str, rafs_version: &str) { + let lower_dir = self.work_dir.join("lower"); + self.create_dir(&self.work_dir.join("blobs")); + + exec( + format!( + "{:?} create --bootstrap {:?} --blob-dir {:?} --log-level info --compressor {} --whiteout-spec {} --fs-version {} {:?}", + self.builder, + self.work_dir.join("bootstrap-lower"), + self.work_dir.join("blobs"), + compressor, + self.whiteout_spec, + rafs_version, + lower_dir, + ) + .as_str(), + false, + b"" + ).unwrap(); } } diff --git a/tests/smoke.rs b/tests/smoke.rs index 7fe9c719621..843c4616c19 100644 --- a/tests/smoke.rs +++ b/tests/smoke.rs @@ -5,7 +5,7 @@ extern crate log; use std::env::var; -use std::fs::File; +use std::fs::{self, File}; use std::io::Read; use std::path::{Path, PathBuf}; @@ -341,6 +341,7 @@ fn test_inline(rafs_version: &str) { builder.check_inline_layout(); } +#[test] fn integration_test_decompress1() { let mut prefix = PathBuf::from(var("TEST_WORKDIR_PREFIX").expect("Please specify TEST_WORKDIR_PREFIX env")); @@ -350,10 +351,17 @@ fn integration_test_decompress1() { let mut builder = builder::new(wk_dir.as_path(), "oci"); builder.make_lower(); - builder.build_lower("lz4_block", "5"); + builder.compress("lz4_block", "5"); + + let mut blob_dir = fs::read_dir(wk_dir.as_path().join("blobs")).unwrap(); + let blob_path = blob_dir.next().unwrap().unwrap().path(); let tar_name = wk_dir.as_path().join("oci.tar"); - builder.decompress("bootstrap-lower", "blobs", tar_name.to_str().unwrap()); + builder.decompress( + "bootstrap-lower", + blob_path.to_str().unwrap(), + tar_name.to_str().unwrap(), + ); let unpack_dir = wk_dir.as_path().join("output"); exec(&format!("mkdir {:?}", unpack_dir), false, b"").unwrap(); @@ -378,7 +386,7 @@ fn integration_test_decompress1() { md5_ret.replace(unpack_dir.to_str().unwrap(), "") ); - let mut texture = File::open("./tests/texture/directory/lower.result").unwrap(); + let mut texture = File::open("./tests/texture/directory/decompress.result").unwrap(); let mut expected = String::new(); texture.read_to_string(&mut expected).unwrap(); diff --git a/tests/texture/directory/decompress.result b/tests/texture/directory/decompress.result new file mode 100644 index 00000000000..e199485cb3a --- /dev/null +++ b/tests/texture/directory/decompress.result @@ -0,0 +1,40 @@ +[ + {"type":"directory","name":"","contents":[ + {"type":"file","name":"root-1"}, + {"type":"file","name":"root-2"}, + {"type":"file","name":"root-large"}, + {"type":"file","name":"root-large-copy"}, + {"type":"directory","name":"sub","contents":[ + {"type":"directory","name":"more","contents":[ + {"type":"file","name":"more-1"}, + {"type":"directory","name":"more-sub","contents":[ + {"type":"file","name":"more-sub-1"} + ]} + ]}, + {"type":"directory","name":"some","contents":[ + {"type":"file","name":"some-1"} + ]}, + {"type":"file","name":"sub-1"}, + {"type":"file","name":"sub-2"}, + {"type":"file","name":"sub-root-large-copy-hardlink"}, + {"type":"file","name":"sub-root-large-copy-hardlink-1"}, + {"type":"file","name":"sub-root-large-hardlink"}, + {"type":"link","name":"sub-root-large-symlink","target":"../root-large","contents":[]} + ]}, + {"type":"file","name":"test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name."} + ]}, + {"type":"report","directories":4,"files":14} +] +1db49cdc205866ae03c1bf1c0c569964 /sub/some/some-1 +1fc133b570f60d1a13f2c97e179a7e8b /root-1 +2655622de6a29f06e0684f55ddc45cc5 /test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name. +34fd810de15e465479ff2d6ff859b5f2 /sub/sub-2 +8593f734ad273015e2376857d02e6b0d /root-2 +9eb85482ce4c00a585f1dc96070b7c6d /sub/more/more-1 +a832eab60623f115363cc100a8862c23 /root-large +a832eab60623f115363cc100a8862c23 /root-large-copy +a832eab60623f115363cc100a8862c23 /sub/sub-root-large-copy-hardlink +a832eab60623f115363cc100a8862c23 /sub/sub-root-large-copy-hardlink-1 +a832eab60623f115363cc100a8862c23 /sub/sub-root-large-hardlink +cca7b08b6c36f4adce54108e0ec23fe7 /sub/more/more-sub/more-sub-1 +fb99fe44d7d0c4486fdb707dc260b2c8 /sub/sub-1 From b128730206c7b9d0be9b13d110a1dad4ef9460ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B3=B0=E5=8F=8B?= Date: Mon, 27 Jun 2022 14:11:22 +0800 Subject: [PATCH 5/7] feat: decompress also works with rafs v6 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 泰友 --- src/bin/nydus-image/decompress.rs | 166 ++++++---------------- src/bin/nydus-image/decompress/test.rs | 117 --------------- tests/builder.rs | 65 ++++++++- tests/smoke.rs | 28 ++-- tests/texture/directory/decompress.result | 9 +- 5 files changed, 124 insertions(+), 261 deletions(-) diff --git a/src/bin/nydus-image/decompress.rs b/src/bin/nydus-image/decompress.rs index 4443248925d..96ffc789f8b 100644 --- a/src/bin/nydus-image/decompress.rs +++ b/src/bin/nydus-image/decompress.rs @@ -5,7 +5,7 @@ use std::{ ffi::OsStr, fs::{File, OpenOptions}, io::{self, Cursor, Error, ErrorKind, Read}, - iter::from_fn, + iter::{self, from_fn, repeat}, os::unix::prelude::{OsStrExt, OsStringExt}, path::{Path, PathBuf}, rc::Rc, @@ -17,7 +17,7 @@ use std::{ use anyhow::Result; use nydus_api::http::LocalFsConfig; use nydus_rafs::{ - metadata::{layout::RAFS_ROOT_INODE, RafsInode, RafsMode, RafsSuper}, + metadata::{RafsInode, RafsMode, RafsSuper}, RafsIoReader, }; use nydus_utils::compress::{self, Algorithm}; @@ -28,7 +28,7 @@ use storage::{ }; use tar::{Builder, EntryType, Header}; -use crate::core::node::{InodeWrapper, OCISPEC_WHITEOUT_PREFIX, ROOT_PATH_NAME}; +use crate::core::node::InodeWrapper; static PAX_SEP1: &'static [u8; 1] = b" "; static PAX_SEP2: &'static [u8; 1] = b"="; @@ -171,7 +171,7 @@ impl OCIDecompressor { } has_more = false; - let node = rs.get_inode(RAFS_ROOT_INODE, false).unwrap(); + let node = rs.get_inode(rs.superblock.root_ino(), false).unwrap(); let path = PathBuf::from("/").join(node.name()).into_boxed_path(); Some((node, path)) @@ -195,7 +195,7 @@ impl Decompressor for OCIDecompressor { .create(&rafs, self.blob.as_deref(), &*self.output)?; for (node, path) in self.iterator(&rafs) { - if node.name() == OsStr::from_bytes(ROOT_PATH_NAME) { + if Util::is_root(&*path) { continue; } builder.append(&*node, &*path)?; @@ -274,60 +274,6 @@ impl SectionBuilder for OCILinkBuilder { } } -struct OCIWhiteoutBuilder {} - -impl OCIWhiteoutBuilder { - fn new() -> Self { - OCIWhiteoutBuilder {} - } -} - -impl SectionBuilder for OCIWhiteoutBuilder { - fn can_handle(&mut self, _: &dyn RafsInode, path: &Path) -> bool { - path.file_name() - .unwrap() - .to_str() - .unwrap() - .starts_with(OCISPEC_WHITEOUT_PREFIX) - } - - fn build(&self, inode: &dyn RafsInode, path: &Path) -> Result> { - // In order to save access time and change time, gnu layout is required. - let mut header = Header::new_gnu(); - header.set_entry_type(EntryType::file()); - header.set_device_major(0).unwrap(); - header.set_device_minor(0).unwrap(); - header.set_size(0); - - let node = InodeWrapper::from_inode_info(inode); - header.set_mtime(node.mtime()); - header.set_uid(node.uid() as u64); - header.set_gid(node.gid() as u64); - header.set_mode(Util::mask_mode(node.mode())); - - // Rafs loses the access time and change time. - // It's required by certain tools, such 7-zip. - // Fill modify time as access time and change time. - header.as_gnu_mut().unwrap().set_atime(node.mtime()); - header.as_gnu_mut().unwrap().set_ctime(node.mtime()); - - let mut sections = Vec::with_capacity(2); - if let Some(sect) = GNUUtil::set_path(&mut header, path)? { - sections.push(sect); - } - - header.set_cksum(); - - let main_header = TarSection { - header, - data: Box::new(io::empty()), - }; - sections.push(main_header); - - Ok(sections) - } -} - struct OCIDirBuilder { ext_builder: Rc, } @@ -364,7 +310,7 @@ impl SectionBuilder for OCIDirBuilder { extensions.extend(extension); } - header.set_cksum(); + Util::set_cksum(&mut header); let mut sections = Vec::with_capacity(2); if let Some(ext_sect) = self.ext_builder.build(&header, extensions)? { @@ -445,7 +391,7 @@ impl SectionBuilder for OCIRegBuilder { extensions.extend(extension); } - header.set_cksum(); + Util::set_cksum(&mut header); let mut sections = Vec::with_capacity(2); if let Some(ext_sect) = self.ext_builder.build(&header, extensions)? { @@ -591,7 +537,7 @@ impl PAXSpecialSectionBuilder { extensions.extend(extension); } - header.set_cksum(); + Util::set_cksum(&mut header); let mut sections = Vec::with_capacity(2); if let Some(ext_sect) = self.ext_builder.build(&header, extensions)? { @@ -647,7 +593,7 @@ impl PAXExtensionSectionBuilder { .set_path(&self.build_pax_name(&path, header.as_old().name.len())?) .map_err(|err| anyhow!("fail to set path for pax section, error {}", err))?; - header.set_cksum(); + Util::set_cksum(&mut header); Ok(Some(TarSection { header, @@ -748,7 +694,7 @@ impl PAXLinkBuilder { extensions.extend(extension); } - header.set_cksum(); + Util::set_cksum(&mut header); let mut sections = Vec::with_capacity(2); if let Some(ext_sect) = self.ext_builder.build(&header, extensions)? { @@ -765,63 +711,6 @@ impl PAXLinkBuilder { } } -struct GNUUtil {} - -impl GNUUtil { - fn set_path(header: &mut Header, path: &Path) -> Result> { - let path = Util::normalize_path(&path) - .map_err(|err| anyhow!("fail to normalize path, error {}", err))?; - - let rs = match header.set_path(&path) { - Ok(_) => return Ok(None), - Err(err) => Err(anyhow!("fail to set path for header, error {}", err)), - }; - - let path = match Util::truncate_path(&path, header.as_old().name.len()) { - Ok(path) => path, - Err(_) => return rs, - }; - - header.set_path(&path).map_err(|err| { - anyhow!( - "fail to set header path again for {:?}, error {}", - path, - err - ) - })?; - - Ok(Some(Self::build_long_section( - EntryType::GNULongName, - path.into_os_string().into_vec(), - ))) - } - - fn build_long_section(entry_type: EntryType, mut data: Vec) -> TarSection { - // gnu requires, especially for long-link section and long-name section - let mut header = Header::new_gnu(); - - data.push(0x00); - header.set_size(data.len() as u64); - let data = Cursor::new(data); - - let title = b"././@LongLink"; - header.as_gnu_mut().unwrap().name[..title.len()].clone_from_slice(&title[..]); - - header.set_entry_type(entry_type); - header.set_mode(0o644); - header.set_uid(0); - header.set_gid(0); - header.set_mtime(0); - - header.set_cksum(); - - TarSection { - header, - data: Box::new(data), - } - } -} - struct PAXUtil {} impl PAXUtil { @@ -915,6 +804,10 @@ impl PAXUtil { struct Util {} impl Util { + fn is_root(path: &Path) -> bool { + path.is_absolute() && path.file_name().is_none() + } + fn normalize_path(path: &Path) -> Result { fn end_with_slash(p: &Path) -> bool { p.as_os_str().as_bytes().last() == Some(&b'/') @@ -928,7 +821,7 @@ impl Util { path.to_path_buf() }; - if end_with_slash(path) { + if end_with_slash(&normalized) { normalized.set_file_name(normalized.file_name().unwrap().to_owned()) } @@ -970,6 +863,33 @@ impl Util { fn mask_mode(st_mode: u32) -> u32 { st_mode & 0o7777 } + + // The checksum is calculated by taking the sum of the unsigned byte values of the header record with the eight checksum bytes taken to be ASCII spaces (decimal value 32). It is stored as a six digit octal number with leading zeroes followed by a NUL and then a space. + // The wiki and Go standard library adhere to this format. Stay with them~~~. + fn set_cksum(header: &mut Header) { + let old = header.as_old(); + let start = old as *const _ as usize; + let cksum_start = old.cksum.as_ptr() as *const _ as usize; + let offset = cksum_start - start; + let len = old.cksum.len(); + + let bs = header.as_bytes(); + let sum = bs[0..offset] + .iter() + .chain(iter::repeat(&b' ').take(len)) + .chain(&bs[offset + len..]) + .fold(0, |a, b| a + (*b as u32)); + + let bs = &mut header.as_old_mut().cksum; + bs[bs.len() - 1] = b' '; + bs[bs.len() - 2] = 0o0; + + let o = format!("{:o}", sum); + let value = o.bytes().rev().chain(repeat(b'0')); + for (slot, value) in bs.iter_mut().rev().skip(2).zip(value) { + *slot = value; + } + } } struct OCITarBuilderFactory {} @@ -1029,7 +949,6 @@ impl OCITarBuilderFactory { let sock_builder = OCISocketBuilder::new(); let hard_link_builder = OCILinkBuilder::new(link_builder.clone()); let symlink_builder = OCISymlinkBuilder::new(link_builder.clone()); - let whiteout_builder = OCIWhiteoutBuilder::new(); let dir_builder = OCIDirBuilder::new(ext_builder); let fifo_builder = OCIFifoBuilder::new(special_builder.clone()); let char_builder = OCICharBuilder::new(special_builder.clone()); @@ -1040,7 +959,6 @@ impl OCITarBuilderFactory { let builders = vec![ Box::new(sock_builder) as Box, Box::new(hard_link_builder), - Box::new(whiteout_builder), Box::new(dir_builder), Box::new(reg_builder), Box::new(symlink_builder), diff --git a/src/bin/nydus-image/decompress/test.rs b/src/bin/nydus-image/decompress/test.rs index e1384bcd5f5..fb9b4a6ad10 100644 --- a/src/bin/nydus-image/decompress/test.rs +++ b/src/bin/nydus-image/decompress/test.rs @@ -198,123 +198,6 @@ fn test_read_chunk_compress() { assert_eq!(buf, [4u8; 256]); } -//#[test] -//fn test_set_path_for_reg_header() { -// let builder = create_default_tar_build(TempFile::new().unwrap().as_path()); -// let mut header = create_default_reg_header(); -// -// let rs = builder.set_path(&mut header, &PathBuf::from("/dir/short_file_name")); -// -// assert!(rs.unwrap().is_none(), "expect no long section returned"); -// assert_eq!(header.path_bytes().into_owned(), b"dir/short_file_name"); -//} -// -//#[test] -//fn test_set_path_for_long_reg_header() { -// let builder = create_default_tar_build(TempFile::new().unwrap().as_path()); -// let mut header = create_default_reg_header(); -// -// let path = PathBuf::from("/").join("path".repeat(1000)); -// let rs = builder.set_path(&mut header, &path); -// -// assert!( -// rs.is_err(), -// "should fail to set long path for regular file header" -// ); -//} -// -//#[test] -//fn test_set_path_for_whiteout_header() { -// let builder = create_default_tar_build(TempFile::new().unwrap().as_path()); -// let mut header = create_default_whiteout_header(); -// -// let path = PathBuf::from("/dir/.wh.file_name"); -// let rs = builder.set_path(&mut header, &path); -// -// assert!( -// builder.is_white_out_section(&path), -// "expect whiteout type assertion" -// ); -// assert!(rs.unwrap().is_none(), "expect no long section returned"); -// assert_eq!(header.path_bytes().into_owned(), b"dir/.wh.file_name"); -//} -// -//#[test] -//fn test_set_path_for_long_whiteout_header() { -// let builder = create_default_tar_build(TempFile::new().unwrap().as_path()); -// let mut header = create_default_whiteout_header(); -// -// let mut path = OsString::from("/.wh."); -// path.push("path".repeat(1000)); -// let path = PathBuf::from(path); -// -// let rs = builder.set_path(&mut header, &path); -// -// let mut sect = rs.unwrap().unwrap(); -// -// let header = sect.header; -// assert!(header.entry_type().is_gnu_longname()); -// assert_eq!(header.path_bytes().into_owned(), b"././@LongLink"); -// -// let mut data = Vec::new(); -// let size = sect.data.read_to_end(&mut data).unwrap(); -// assert_eq!(header.size().unwrap() as usize, size); -// -// let mut path = Vec::new(); -// path.extend_from_slice(b".wh."); -// path.extend_from_slice(&b"path".repeat(1000)); -// path.push(0x00); -// assert_eq!(data, path); -//} -// -//#[test] -//fn test_set_path_for_dir_header() { -// let builder = create_default_tar_build(TempFile::new().unwrap().as_path()); -// let mut header = create_default_dir_header(); -// -// let rs = builder.set_path(&mut header, &PathBuf::from("/dir")); -// -// assert!(rs.unwrap().is_none(), "expect no long section"); -// assert_eq!(header.path_bytes().into_owned(), b"dir/"); -//} - -//fn create_default_dir_header() -> Header { -// let mut header = Header::new_ustar(); -// header.set_entry_type(EntryType::dir()); -// -// header -//} -// -//fn create_default_whiteout_header() -> Header { -// let mut header = Header::new_gnu(); -// header.set_entry_type(EntryType::file()); -// -// header -//} -// -//fn create_default_reg_header() -> Header { -// let mut header = Header::new_ustar(); -// header.set_entry_type(EntryType::file()); -// -// header -//} -// -//fn create_default_tar_build(output: &Path) -> DefaultTarBuilder { -// let output = OpenOptions::new() -// .write(true) -// .truncate(true) -// .read(false) -// .open(output) -// .unwrap(); -// -// DefaultTarBuilder { -// blob: None, -// reader: None, -// writer: Builder::new(output), -// links: HashMap::new(), -// } -//} - fn creater_compress_chunk_reader() -> ChunkReader { let chunk = [[1u8; 256], [2u8; 256], [3u8; 256], [4u8; 256]].concat(); diff --git a/tests/builder.rs b/tests/builder.rs index f10c2ee4d7d..8f8ed0692bc 100644 --- a/tests/builder.rs +++ b/tests/builder.rs @@ -451,11 +451,11 @@ impl<'a> Builder<'a> { assert_eq!(cur, header.size().unwrap()); } - pub fn decompress(&self, bootstrap: &str, blob: &str, output: &str) { + pub fn decompress(&self, blob: &str, output: &str) { let cmd = format!( "{:?} decompress --bootstrap {:?} --blob {:?} --output {:?}", self.builder, - self.work_dir.join(bootstrap), + self.work_dir.join("bootstrap"), self.work_dir.join(blob), self.work_dir.join(output) ); @@ -464,23 +464,76 @@ impl<'a> Builder<'a> { } pub fn compress(&mut self, compressor: &str, rafs_version: &str) { - let lower_dir = self.work_dir.join("lower"); self.create_dir(&self.work_dir.join("blobs")); exec( format!( "{:?} create --bootstrap {:?} --blob-dir {:?} --log-level info --compressor {} --whiteout-spec {} --fs-version {} {:?}", self.builder, - self.work_dir.join("bootstrap-lower"), + self.work_dir.join("bootstrap"), self.work_dir.join("blobs"), compressor, - self.whiteout_spec, + "none", // Use "none" instead of "oci". Otherwise whiteout and opaque files are no longer exist in result. rafs_version, - lower_dir, + self.work_dir.join("compress"), ) .as_str(), false, b"" ).unwrap(); } + + pub fn make_compress(&mut self) { + let dir = self.work_dir.join("compress"); + self.create_dir(&dir); + + self.create_file(&dir.join("root-1"), b"lower:root-1"); + self.create_file(&dir.join("root-2"), b"lower:root-2"); + self.create_large_file(&dir.join("root-large"), 13); + self.copy_file(&dir.join("root-large"), &dir.join("root-large-copy")); + + self.create_dir(&dir.join("sub")); + self.create_file(&dir.join("sub/sub-1"), b"lower:sub-1"); + self.create_file(&dir.join("sub/sub-2"), b"lower:sub-2"); + self.create_hardlink( + &dir.join("root-large"), + &dir.join("sub/sub-root-large-hardlink"), + ); + self.create_hardlink( + &dir.join("root-large-copy"), + &dir.join("sub/sub-root-large-copy-hardlink"), + ); + self.create_hardlink( + &dir.join("root-large-copy"), + &dir.join("sub/sub-root-large-copy-hardlink-1"), + ); + self.create_symlink( + Path::new("../root-large"), + &dir.join("sub/sub-root-large-symlink"), + ); + + self.create_dir(&dir.join("sub/some")); + self.create_file(&dir.join("sub/some/some-1"), b"lower:some-1"); + + self.create_dir(&dir.join("sub/more")); + self.create_file(&dir.join("sub/more/more-1"), b"lower:more-1"); + self.create_dir(&dir.join("sub/more/more-sub")); + self.create_file( + &dir.join("sub/more/more-sub/more-sub-1"), + b"lower:more-sub-1", + ); + + let long_name = &"test-😉-name.".repeat(100)[..255]; + self.create_file(&dir.join(long_name), b"lower:long-name"); + + self.set_xattr(&dir.join("sub/sub-1"), "user.key-foo", b"value-foo"); + self.set_xattr(&dir.join("sub/sub-1"), "user.key-bar", b"value-bar"); + + self.create_whiteout_file(&dir.join("sub/some")); + self.create_opaque_entry(&dir.join("sub/more")); + + self.create_special_file(&dir.join("block-file"), "block"); + self.create_special_file(&dir.join("char-file"), "char"); + self.create_special_file(&dir.join("fifo-file"), "fifo"); + } } diff --git a/tests/smoke.rs b/tests/smoke.rs index 843c4616c19..4e166c86234 100644 --- a/tests/smoke.rs +++ b/tests/smoke.rs @@ -342,28 +342,30 @@ fn test_inline(rafs_version: &str) { } #[test] -fn integration_test_decompress1() { +fn integration_test_decompress() { let mut prefix = PathBuf::from(var("TEST_WORKDIR_PREFIX").expect("Please specify TEST_WORKDIR_PREFIX env")); prefix.push(""); - let wk_dir = TempDir::new_with_prefix(prefix).unwrap(); + let wk_dir = TempDir::new_with_prefix(&prefix).unwrap(); + test_decompress(wk_dir.as_path(), "5"); - let mut builder = builder::new(wk_dir.as_path(), "oci"); - builder.make_lower(); - builder.compress("lz4_block", "5"); + let wk_dir = TempDir::new_with_prefix(&prefix).unwrap(); + test_decompress(wk_dir.as_path(), "6"); +} - let mut blob_dir = fs::read_dir(wk_dir.as_path().join("blobs")).unwrap(); +fn test_decompress(work_dir: &Path, version: &str) { + let mut builder = builder::new(work_dir, "oci"); + builder.make_compress(); + builder.compress("lz4_block", version); + + let mut blob_dir = fs::read_dir(work_dir.join("blobs")).unwrap(); let blob_path = blob_dir.next().unwrap().unwrap().path(); - let tar_name = wk_dir.as_path().join("oci.tar"); - builder.decompress( - "bootstrap-lower", - blob_path.to_str().unwrap(), - tar_name.to_str().unwrap(), - ); + let tar_name = work_dir.join("oci.tar"); + builder.decompress(blob_path.to_str().unwrap(), tar_name.to_str().unwrap()); - let unpack_dir = wk_dir.as_path().join("output"); + let unpack_dir = work_dir.join("output"); exec(&format!("mkdir {:?}", unpack_dir), false, b"").unwrap(); exec( &format!("tar --xattrs -xf {:?} -C {:?}", tar_name, unpack_dir), diff --git a/tests/texture/directory/decompress.result b/tests/texture/directory/decompress.result index e199485cb3a..34fb826ea8f 100644 --- a/tests/texture/directory/decompress.result +++ b/tests/texture/directory/decompress.result @@ -1,11 +1,16 @@ [ {"type":"directory","name":"","contents":[ + {"type":"block","name":"block-file"}, + {"type":"char","name":"char-file"}, + {"type":"fifo","name":"fifo-file"}, {"type":"file","name":"root-1"}, {"type":"file","name":"root-2"}, {"type":"file","name":"root-large"}, {"type":"file","name":"root-large-copy"}, {"type":"directory","name":"sub","contents":[ + {"type":"file","name":".wh.some"}, {"type":"directory","name":"more","contents":[ + {"type":"file","name":".wh..wh..opq"}, {"type":"file","name":"more-1"}, {"type":"directory","name":"more-sub","contents":[ {"type":"file","name":"more-sub-1"} @@ -23,7 +28,7 @@ ]}, {"type":"file","name":"test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name.test-😉-name."} ]}, - {"type":"report","directories":4,"files":14} + {"type":"report","directories":4,"files":19} ] 1db49cdc205866ae03c1bf1c0c569964 /sub/some/some-1 1fc133b570f60d1a13f2c97e179a7e8b /root-1 @@ -37,4 +42,6 @@ a832eab60623f115363cc100a8862c23 /sub/sub-root-large-copy-hardlink a832eab60623f115363cc100a8862c23 /sub/sub-root-large-copy-hardlink-1 a832eab60623f115363cc100a8862c23 /sub/sub-root-large-hardlink cca7b08b6c36f4adce54108e0ec23fe7 /sub/more/more-sub/more-sub-1 +d41d8cd98f00b204e9800998ecf8427e /sub/more/.wh..wh..opq +d41d8cd98f00b204e9800998ecf8427e /sub/.wh.some fb99fe44d7d0c4486fdb707dc260b2c8 /sub/sub-1 From 0c3e38c559d9eb30154a24ef03833129739805c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B3=B0=E5=8F=8B?= Date: Mon, 27 Jun 2022 18:02:32 +0800 Subject: [PATCH 6/7] refact: chore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 泰友 --- src/bin/nydus-image/main.rs | 27 +- .../nydus-image/{decompress.rs => unpack.rs} | 241 +++++++----------- .../{decompress => unpack}/test.rs | 6 +- tests/builder.rs | 8 +- tests/smoke.rs | 18 +- .../{decompress.result => unpack.result} | 2 +- 6 files changed, 126 insertions(+), 176 deletions(-) rename src/bin/nydus-image/{decompress.rs => unpack.rs} (81%) rename src/bin/nydus-image/{decompress => unpack}/test.rs (97%) rename tests/texture/directory/{decompress.result => unpack.result} (100%) diff --git a/src/bin/nydus-image/main.rs b/src/bin/nydus-image/main.rs index 8fc890bf93d..6130d936992 100644 --- a/src/bin/nydus-image/main.rs +++ b/src/bin/nydus-image/main.rs @@ -22,7 +22,6 @@ use std::path::{Path, PathBuf}; use anyhow::{bail, Context, Result}; use clap::{App, Arg, ArgMatches, SubCommand}; -use decompress::Decompressor; use nix::unistd::{getegid, geteuid}; use serde::{Deserialize, Serialize}; @@ -43,19 +42,19 @@ use crate::core::context::{ use crate::core::node::{self, WhiteoutSpec}; use crate::core::prefetch::Prefetch; use crate::core::tree; -use crate::decompress::OCIDecompressor; use crate::merge::Merger; use crate::trace::{EventTracerClass, TimingTracerClass, TraceClass}; +use crate::unpack::{OCIUnpacker, Unpacker}; use crate::validator::Validator; #[macro_use] mod trace; mod builder; mod core; -mod decompress; mod inspect; mod merge; mod stat; +mod unpack; mod validator; const BLOB_ID_MAXIMUM_LENGTH: usize = 255; @@ -513,7 +512,7 @@ fn prepare_cmd_args(bti_string: String) -> ArgMatches<'static> { .takes_value(true)) ) .subcommand( - SubCommand::with_name("decompress") + SubCommand::with_name("unpack") .about("Decompress nydus bootstrap and blob files to tar") .arg( Arg::with_name("bootstrap") @@ -597,8 +596,8 @@ fn main() -> Result<()> { Command::stat(matches) } else if let Some(matches) = cmd.subcommand_matches("compact") { Command::compact(matches, &build_info) - } else if let Some(matches) = cmd.subcommand_matches("decompress") { - Command::decompress(matches) + } else if let Some(matches) = cmd.subcommand_matches("unpack") { + Command::unpack(matches) } else { println!("{}", cmd.usage()); Ok(()) @@ -797,22 +796,16 @@ impl Command { Ok(()) } - fn decompress(args: &clap::ArgMatches) -> Result<()> { + fn unpack(args: &clap::ArgMatches) -> Result<()> { let bootstrap = args.value_of("bootstrap").expect("pass in bootstrap"); let blob = args.value_of("blob"); let output = args.value_of("output").expect("pass in output"); - let decompressor = Box::new(OCIDecompressor::new(bootstrap, blob, output).map_err( - |err| { - error!("fail to create decompressor, error: {}", err); - anyhow!("fail to create decompressor, error: {}", err) - }, - )?) as Box; + let unpacker = Box::new( + OCIUnpacker::new(bootstrap, blob, output).with_context(|| "fail to create unpacker")?, + ) as Box; - decompressor.decompress().map_err(|err| { - error!("fail to decompress, error: {}", err); - anyhow!("fail to decompress, error: {}", err) - }) + unpacker.unpack().with_context(|| "fail to unpack") } fn check(matches: &clap::ArgMatches, build_info: &BuildTimeInfo) -> Result<()> { diff --git a/src/bin/nydus-image/decompress.rs b/src/bin/nydus-image/unpack.rs similarity index 81% rename from src/bin/nydus-image/decompress.rs rename to src/bin/nydus-image/unpack.rs index 96ffc789f8b..96bd9d5e477 100644 --- a/src/bin/nydus-image/decompress.rs +++ b/src/bin/nydus-image/unpack.rs @@ -1,5 +1,3 @@ -extern crate tar; - use std::{ collections::HashMap, ffi::OsStr, @@ -14,7 +12,7 @@ use std::{ vec::IntoIter, }; -use anyhow::Result; +use anyhow::{Context, Result}; use nydus_api::http::LocalFsConfig; use nydus_rafs::{ metadata::{RafsInode, RafsMode, RafsSuper}, @@ -30,33 +28,33 @@ use tar::{Builder, EntryType, Header}; use crate::core::node::InodeWrapper; -static PAX_SEP1: &'static [u8; 1] = b" "; -static PAX_SEP2: &'static [u8; 1] = b"="; -static PAX_PREFIX: &'static [u8; 13] = b"SCHILY.xattr."; -static PAX_DELIMITER: &'static [u8; 1] = b"\n"; +static PAX_SEP1: &[u8; 1] = b" "; +static PAX_SEP2: &[u8; 1] = b"="; +static PAX_PREFIX: &[u8; 13] = b"SCHILY.xattr."; +static PAX_DELIMITER: &[u8; 1] = b"\n"; -pub trait Decompressor { - fn decompress(&self) -> Result<()>; +pub trait Unpacker { + fn unpack(&self) -> Result<()>; } -/// A decompressor with the ability to convert bootstrap file and blob file to tar -pub struct OCIDecompressor { - bootstrap: Box, - blob: Option>, - output: Box, +/// A unpacker with the ability to convert bootstrap file and blob file to tar +pub struct OCIUnpacker { + bootstrap: PathBuf, + blob: Option, + output: PathBuf, builder_factory: OCITarBuilderFactory, } -impl OCIDecompressor { +impl OCIUnpacker { pub fn new(bootstrap: &str, blob: Option<&str>, output: &str) -> Result { - let bootstrap = PathBuf::from(bootstrap).into_boxed_path(); - let output = PathBuf::from(output).into_boxed_path(); - let blob = blob.map(|v| PathBuf::from(v).into_boxed_path()); + let bootstrap = PathBuf::from(bootstrap); + let output = PathBuf::from(output); + let blob = blob.map(PathBuf::from); let builder_factory = OCITarBuilderFactory::new(); - Ok(OCIDecompressor { + Ok(OCIUnpacker { builder_factory, bootstrap, blob, @@ -69,35 +67,16 @@ impl OCIDecompressor { .read(true) .write(false) .open(&*self.bootstrap) - .map_err(|err| { - error!( - "fail to load bootstrap {:?}, error: {}", - self.bootstrap, err - ); - anyhow!( - "fail to load bootstrap {:?}, error: {}", - self.bootstrap, - err - ) - })?; + .with_context(|| format!("fail to open bootstrap {:?}", self.bootstrap))?; let mut rs = RafsSuper { mode: RafsMode::Direct, - validate_digest: true, + validate_digest: false, ..Default::default() }; + rs.load(&mut (Box::new(bootstrap) as RafsIoReader)) - .map_err(|err| { - error!( - "fail to load bootstrap {:?}, error: {}", - self.bootstrap, err - ); - anyhow!( - "fail to load bootstrap {:?}, error: {}", - self.bootstrap, - err - ) - })?; + .with_context(|| format!("fail to load bootstrap {:?}", self.bootstrap))?; Ok(rs) } @@ -106,8 +85,9 @@ impl OCIDecompressor { fn iterator<'a>( &'a self, rs: &'a RafsSuper, - ) -> Box, Box)> + 'a> { - // A cursor means the next node to visit of certain height in the tree. It always starts with the first one of level. + ) -> Box, PathBuf)> + 'a> { + // A cursor means the next node to be visited at same height in the tree. + // It always starts with the first one of level. // A cursor stack is of cursors from root to leaf. let mut cursor_stack = Vec::with_capacity(32); @@ -142,7 +122,7 @@ impl OCIDecompressor { &self, node: Arc, path: &Path, - ) -> Box, Box)>> { + ) -> Box, PathBuf)>> { let base = path.to_path_buf(); let mut next_idx = 0..node.get_child_count(); @@ -152,7 +132,7 @@ impl OCIDecompressor { } let child = node.get_child_by_index(next_idx.next().unwrap()).unwrap(); - let child_path = base.join(child.name()).into_boxed_path(); + let child_path = base.join(child.name()); Some((child, child_path)) }; @@ -163,7 +143,7 @@ impl OCIDecompressor { fn cursor_of_root<'a>( &self, rs: &'a RafsSuper, - ) -> Box, Box)> + 'a> { + ) -> Box, PathBuf)> + 'a> { let mut has_more = true; let visitor = from_fn(move || { if !has_more { @@ -172,7 +152,7 @@ impl OCIDecompressor { has_more = false; let node = rs.get_inode(rs.superblock.root_ino(), false).unwrap(); - let path = PathBuf::from("/").join(node.name()).into_boxed_path(); + let path = PathBuf::from("/").join(node.name()); Some((node, path)) }); @@ -181,24 +161,24 @@ impl OCIDecompressor { } } -impl Decompressor for OCIDecompressor { - fn decompress(&self) -> Result<()> { +impl Unpacker for OCIUnpacker { + fn unpack(&self) -> Result<()> { info!( - "default decompressor, bootstrap file: {:?}, blob file: {:?}, output file: {:?}", + "oci unpacker, bootstrap file: {:?}, blob file: {:?}, output file: {:?}", self.bootstrap, self.blob, self.output ); let rafs = self.load_rafs()?; - let mut builder = - self.builder_factory - .create(&rafs, self.blob.as_deref(), &*self.output)?; + let mut builder = self + .builder_factory + .create(&rafs, self.blob.as_deref(), &self.output)?; for (node, path) in self.iterator(&rafs) { - if Util::is_root(&*path) { + if Util::is_root(&path) { continue; } - builder.append(&*node, &*path)?; + builder.append(&*node, &path)?; } Ok(()) @@ -263,7 +243,7 @@ impl SectionBuilder for OCILinkBuilder { .insert(node.ino(), path.to_path_buf().into_boxed_path()); } - return is_appeared; + is_appeared } fn build(&self, node: &dyn RafsInode, path: &Path) -> Result> { @@ -304,7 +284,7 @@ impl SectionBuilder for OCIDirBuilder { let mut extensions = Vec::with_capacity(2); if let Some(extension) = PAXUtil::set_path(&mut header, path)? { - extensions.extend(extension); + extensions.push(extension); } if let Some(extension) = PAXUtil::get_xattr_as_extensions(inode) { extensions.extend(extension); @@ -356,7 +336,7 @@ impl OCIRegBuilder { .collect(); let reader = ChunkReader::new( - self.compressor.as_ref().unwrap().clone(), + *self.compressor.as_ref().unwrap(), self.reader.as_ref().unwrap().clone(), chunks, ); @@ -385,7 +365,7 @@ impl SectionBuilder for OCIRegBuilder { let mut extensions = Vec::with_capacity(2); if let Some(extension) = PAXUtil::set_path(&mut header, path)? { - extensions.extend(extension); + extensions.push(extension); } if let Some(extension) = PAXUtil::get_xattr_as_extensions(inode) { extensions.extend(extension); @@ -531,7 +511,7 @@ impl PAXSpecialSectionBuilder { let mut extensions = Vec::with_capacity(2); if let Some(extension) = PAXUtil::set_path(&mut header, path)? { - extensions.extend(extension); + extensions.push(extension); } if let Some(extension) = PAXUtil::get_xattr_as_extensions(inode) { extensions.extend(extension); @@ -561,6 +541,11 @@ impl PAXSpecialSectionBuilder { } } +struct PAXRecord { + k: Vec, + v: Vec, +} + struct PAXExtensionSectionBuilder {} impl PAXExtensionSectionBuilder { @@ -568,12 +553,8 @@ impl PAXExtensionSectionBuilder { PAXExtensionSectionBuilder {} } - fn build( - &self, - header: &Header, - extensions: Vec<(Vec, Vec)>, - ) -> Result> { - if extensions.len() == 0 { + fn build(&self, header: &Header, extensions: Vec) -> Result> { + if extensions.is_empty() { return Ok(None); } @@ -591,7 +572,7 @@ impl PAXExtensionSectionBuilder { header .set_path(&self.build_pax_name(&path, header.as_old().name.len())?) - .map_err(|err| anyhow!("fail to set path for pax section, error {}", err))?; + .with_context(|| "fail to set path for pax section")?; Util::set_cksum(&mut header); @@ -601,17 +582,16 @@ impl PAXExtensionSectionBuilder { })) } - fn build_data(&self, mut extensions: Vec<(Vec, Vec)>) -> Vec { - extensions.sort_by(|(k1, _), (k2, _)| { - let k1 = str::from_utf8(k1).unwrap(); - let k2 = str::from_utf8(k2).unwrap(); + fn build_data(&self, mut extensions: Vec) -> Vec { + extensions.sort_by(|r1, r2| { + let k1 = str::from_utf8(&r1.k).unwrap(); + let k2 = str::from_utf8(&r2.k).unwrap(); k1.cmp(k2) }); extensions .into_iter() - .map(|(k, v)| self.build_pax_record(&k, &v)) - .flatten() + .flat_map(|r| self.build_pax_record(&r.k, &r.v)) .collect() } @@ -685,10 +665,10 @@ impl PAXLinkBuilder { let mut extensions = Vec::with_capacity(3); if let Some(extension) = PAXUtil::set_path(&mut header, path)? { - extensions.extend(extension); + extensions.push(extension); } if let Some(extension) = PAXUtil::set_link(&mut header, link)? { - extensions.extend(extension); + extensions.push(extension); } if let Some(extension) = PAXUtil::get_xattr_as_extensions(inode) { extensions.extend(extension); @@ -714,7 +694,7 @@ impl PAXLinkBuilder { struct PAXUtil {} impl PAXUtil { - fn get_xattr_as_extensions(inode: &dyn RafsInode) -> Option, Vec)>> { + fn get_xattr_as_extensions(inode: &dyn RafsInode) -> Option> { if !inode.has_xattr() { return None; } @@ -732,70 +712,60 @@ impl PAXUtil { .into_iter() .chain(key.into_iter()) .collect(); - extensions.push((key, value)); + extensions.push(PAXRecord { k: key, v: value }); } Some(extensions) } - fn set_link(header: &mut Header, path: &Path) -> Result, Vec)>>> { - let path = Util::normalize_path(path) - .map_err(|err| anyhow!("fail to normalize path, error {}", err))?; + fn set_link(header: &mut Header, path: &Path) -> Result> { + let path = Util::normalize_path(path).with_context(|| "fail to normalize path")?; let max_len = header.as_old().linkname.len(); if path.as_os_str().len() <= max_len { return header .set_link_name(&path) - .map_err(|err| anyhow!("fail to set short path for pax header, error {}", err)) + .with_context(|| "fail to set short link for pax header") .map(|_| None); } - let extension = vec![( - "linkpath".to_owned().into_bytes(), - path.to_owned().into_os_string().into_vec(), - )]; + let extension = PAXRecord { + k: "linkpath".to_owned().into_bytes(), + v: path.to_owned().into_os_string().into_vec(), + }; let path = Util::truncate_path(&path, max_len) - .map_err(|err| anyhow!("fail to truncate path for pax header, error {}", err))?; + .with_context(|| "fail to truncate link for pax header")?; - header.set_link_name(&path).map_err(|err| { - anyhow!( - "fail to set header path again for {:?}, error {}", - path, - err - ) - })?; + header + .set_link_name(&path) + .with_context(|| format!("fail to set header link again for {:?}", path))?; Ok(Some(extension)) } - fn set_path(header: &mut Header, path: &Path) -> Result, Vec)>>> { - let path = Util::normalize_path(&path) - .map_err(|err| anyhow!("fail to normalize path, error {}", err))?; + fn set_path(header: &mut Header, path: &Path) -> Result> { + let path = Util::normalize_path(path).with_context(|| "fail to normalize path")?; let max_len = header.as_old().name.len(); if path.as_os_str().len() <= max_len { return header .set_path(path) - .map_err(|err| anyhow!("fail to set short path for pax header, error {}", err)) + .with_context(|| "fail to set short path for pax header") .map(|_| None); } - let extension = vec![( - "path".to_owned().into_bytes(), - path.to_owned().into_os_string().into_vec(), - )]; + let extension = PAXRecord { + k: "path".to_owned().into_bytes(), + v: path.to_owned().into_os_string().into_vec(), + }; let path = Util::truncate_path(&path, max_len) - .map_err(|err| anyhow!("fail to truncate path for pax header, error {}", err))?; + .with_context(|| "fail to truncate path for pax header")?; - header.set_path(&path).map_err(|err| { - anyhow!( - "fail to set header path again for {:?}, error {}", - path, - err - ) - })?; + header + .set_path(&path) + .with_context(|| format!("fail to set header path again for {:?}", path))?; Ok(Some(extension)) } @@ -815,14 +785,15 @@ impl Util { let mut normalized = if path.has_root() { path.strip_prefix("/") - .map_err(|err| anyhow!("fail to strip prefix /, error {}", err))? + .with_context(|| "fail to strip prefix /")? .to_path_buf() } else { path.to_path_buf() }; if end_with_slash(&normalized) { - normalized.set_file_name(normalized.file_name().unwrap().to_owned()) + let name = normalized.file_name().unwrap().to_owned(); + normalized.set_file_name(name); } Ok(normalized) @@ -838,7 +809,7 @@ impl Util { let path = match str::from_utf8(&path[..max_len]) { Ok(s) => Ok(s), Err(err) => str::from_utf8(&path[..err.valid_up_to()]) - .map_err(|err| anyhow!("fail to convert bytes to utf8 str, error: {}", err)), + .with_context(|| "fail to convert bytes to utf8 str"), }?; Ok(PathBuf::from(path)) @@ -854,7 +825,8 @@ impl Util { // c_ISCHR = 020000 // Character special file // c_ISSOCK = 0140000 // Socket // - // Although many readers bear it, such as Go standard library and tar tool in ubuntu, truncate to last four bytes. The four consists of below: + // Although many readers bear it, such as Go standard library and tar tool in ubuntu + // Truncate to last four bytes. The four consists of below: // // c_ISUID = 04000 // Set uid // c_ISGID = 02000 // Set gid @@ -864,7 +836,9 @@ impl Util { st_mode & 0o7777 } - // The checksum is calculated by taking the sum of the unsigned byte values of the header record with the eight checksum bytes taken to be ASCII spaces (decimal value 32). It is stored as a six digit octal number with leading zeroes followed by a NUL and then a space. + // The checksum is calculated by taking the sum of the unsigned byte values of + // the header record with the eight checksum bytes taken to be ASCII spaces (decimal value 32). + // It is stored as a six digit octal number with leading zeroes followed by a NUL and then a space. // The wiki and Go standard library adhere to this format. Stay with them~~~. fn set_cksum(header: &mut Header) { let old = header.as_old(); @@ -923,13 +897,7 @@ impl OCITarBuilderFactory { .truncate(true) .read(false) .open(output_path) - .map_err(|err| { - anyhow!( - "fail to open output file {:?}, error: {:?}", - output_path, - err - ) - })?, + .with_context(|| format!("fail to open output file {:?}", output_path))?, ); Ok(builder) @@ -948,7 +916,7 @@ impl OCITarBuilderFactory { // OCI builders let sock_builder = OCISocketBuilder::new(); let hard_link_builder = OCILinkBuilder::new(link_builder.clone()); - let symlink_builder = OCISymlinkBuilder::new(link_builder.clone()); + let symlink_builder = OCISymlinkBuilder::new(link_builder); let dir_builder = OCIDirBuilder::new(ext_builder); let fifo_builder = OCIFifoBuilder::new(special_builder.clone()); let char_builder = OCICharBuilder::new(special_builder.clone()); @@ -1004,21 +972,11 @@ impl OCITarBuilderFactory { dir: Default::default(), alt_dirs: Default::default(), }; - let config = serde_json::to_value(config).map_err(|err| { - anyhow!( - "fail to create local backend config for {:?}, error: {:?}", - blob_path, - err - ) - })?; - - let backend = LocalFs::new(config, Some("decompressor")).map_err(|err| { - anyhow!( - "fail to create local backend for {:?}, error: {:?}", - blob_path, - err - ) - })?; + let config = serde_json::to_value(config) + .with_context(|| format!("fail to create local backend config for {:?}", blob_path))?; + + let backend = LocalFs::new(config, Some("unpacker")) + .with_context(|| format!("fail to create local backend for {:?}", blob_path))?; let reader = backend .get_reader("") @@ -1059,7 +1017,7 @@ impl TarBuilder for OCITarBuilder { } if is_known_type { - bail!("node {:?} can not be decompressed", path) + bail!("node {:?} can not be unpacked", path) } Ok(()) @@ -1109,10 +1067,7 @@ impl ChunkReader { data.as_mut_slice(), self.compressor, ) - .map_err(|err| { - error!("fail to decompress, error: {:?}", err); - anyhow!("fail to decompress, error: {:?}", err) - })?; + .with_context(|| "fail to decompress")?; self.chunk = Cursor::new(data); diff --git a/src/bin/nydus-image/decompress/test.rs b/src/bin/nydus-image/unpack/test.rs similarity index 97% rename from src/bin/nydus-image/decompress/test.rs rename to src/bin/nydus-image/unpack/test.rs index fb9b4a6ad10..04d4682b8ad 100644 --- a/src/bin/nydus-image/decompress/test.rs +++ b/src/bin/nydus-image/unpack/test.rs @@ -25,7 +25,7 @@ impl BlobReader for MockBlobReader { fn try_read(&self, buf: &mut [u8], offset: u64) -> storage::backend::BackendResult { let offset = offset as usize; if offset >= self.data.len() { - return Ok(0 as usize); + return Ok(0_usize); } let end = self.data.len().min(offset as usize + buf.len()); @@ -179,7 +179,7 @@ fn test_read_chunk_zero_buffer() { #[test] fn test_read_chunk_compress() { - let mut reader = creater_compress_chunk_reader(); + let mut reader = create_compress_chunk_reader(); let mut buf = [0u8; 256]; assert_eq!(256, reader.read(&mut buf).unwrap()); @@ -198,7 +198,7 @@ fn test_read_chunk_compress() { assert_eq!(buf, [4u8; 256]); } -fn creater_compress_chunk_reader() -> ChunkReader { +fn create_compress_chunk_reader() -> ChunkReader { let chunk = [[1u8; 256], [2u8; 256], [3u8; 256], [4u8; 256]].concat(); let (compressed_chunk, is_compressed) = compress::compress(&chunk, Algorithm::GZip).unwrap(); diff --git a/tests/builder.rs b/tests/builder.rs index 8f8ed0692bc..0ab22b3db72 100644 --- a/tests/builder.rs +++ b/tests/builder.rs @@ -451,9 +451,9 @@ impl<'a> Builder<'a> { assert_eq!(cur, header.size().unwrap()); } - pub fn decompress(&self, blob: &str, output: &str) { + pub fn unpack(&self, blob: &str, output: &str) { let cmd = format!( - "{:?} decompress --bootstrap {:?} --blob {:?} --output {:?}", + "{:?} unpack --bootstrap {:?} --blob {:?} --output {:?}", self.builder, self.work_dir.join("bootstrap"), self.work_dir.join(blob), @@ -463,7 +463,7 @@ impl<'a> Builder<'a> { exec(&cmd, false, b"").unwrap(); } - pub fn compress(&mut self, compressor: &str, rafs_version: &str) { + pub fn pack(&mut self, compressor: &str, rafs_version: &str) { self.create_dir(&self.work_dir.join("blobs")); exec( @@ -483,7 +483,7 @@ impl<'a> Builder<'a> { ).unwrap(); } - pub fn make_compress(&mut self) { + pub fn make_pack(&mut self) { let dir = self.work_dir.join("compress"); self.create_dir(&dir); diff --git a/tests/smoke.rs b/tests/smoke.rs index 4e166c86234..f83a567d99c 100644 --- a/tests/smoke.rs +++ b/tests/smoke.rs @@ -342,28 +342,30 @@ fn test_inline(rafs_version: &str) { } #[test] -fn integration_test_decompress() { +fn integration_test_unpack() { let mut prefix = PathBuf::from(var("TEST_WORKDIR_PREFIX").expect("Please specify TEST_WORKDIR_PREFIX env")); + + // A trailing slash is required. prefix.push(""); let wk_dir = TempDir::new_with_prefix(&prefix).unwrap(); - test_decompress(wk_dir.as_path(), "5"); + test_unpack(wk_dir.as_path(), "5"); let wk_dir = TempDir::new_with_prefix(&prefix).unwrap(); - test_decompress(wk_dir.as_path(), "6"); + test_unpack(wk_dir.as_path(), "6"); } -fn test_decompress(work_dir: &Path, version: &str) { +fn test_unpack(work_dir: &Path, version: &str) { let mut builder = builder::new(work_dir, "oci"); - builder.make_compress(); - builder.compress("lz4_block", version); + builder.make_pack(); + builder.pack("lz4_block", version); let mut blob_dir = fs::read_dir(work_dir.join("blobs")).unwrap(); let blob_path = blob_dir.next().unwrap().unwrap().path(); let tar_name = work_dir.join("oci.tar"); - builder.decompress(blob_path.to_str().unwrap(), tar_name.to_str().unwrap()); + builder.unpack(blob_path.to_str().unwrap(), tar_name.to_str().unwrap()); let unpack_dir = work_dir.join("output"); exec(&format!("mkdir {:?}", unpack_dir), false, b"").unwrap(); @@ -388,7 +390,7 @@ fn test_decompress(work_dir: &Path, version: &str) { md5_ret.replace(unpack_dir.to_str().unwrap(), "") ); - let mut texture = File::open("./tests/texture/directory/decompress.result").unwrap(); + let mut texture = File::open("./tests/texture/directory/unpack.result").unwrap(); let mut expected = String::new(); texture.read_to_string(&mut expected).unwrap(); diff --git a/tests/texture/directory/decompress.result b/tests/texture/directory/unpack.result similarity index 100% rename from tests/texture/directory/decompress.result rename to tests/texture/directory/unpack.result index 34fb826ea8f..7b1caab96fb 100644 --- a/tests/texture/directory/decompress.result +++ b/tests/texture/directory/unpack.result @@ -42,6 +42,6 @@ a832eab60623f115363cc100a8862c23 /sub/sub-root-large-copy-hardlink a832eab60623f115363cc100a8862c23 /sub/sub-root-large-copy-hardlink-1 a832eab60623f115363cc100a8862c23 /sub/sub-root-large-hardlink cca7b08b6c36f4adce54108e0ec23fe7 /sub/more/more-sub/more-sub-1 -d41d8cd98f00b204e9800998ecf8427e /sub/more/.wh..wh..opq d41d8cd98f00b204e9800998ecf8427e /sub/.wh.some +d41d8cd98f00b204e9800998ecf8427e /sub/more/.wh..wh..opq fb99fe44d7d0c4486fdb707dc260b2c8 /sub/sub-1 From e246ee1e28a5e71e1d7fdc8a813d69d10699acca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B3=B0=E5=8F=8B?= Date: Wed, 29 Jun 2022 14:44:40 +0800 Subject: [PATCH 7/7] refact: code organization of unpack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 泰友 --- src/bin/nydus-image/main.rs | 9 +- src/bin/nydus-image/unpack/mod.rs | 342 ++++++++++++++ .../nydus-image/{unpack.rs => unpack/pax.rs} | 424 ++---------------- src/bin/nydus-image/unpack/{ => pax}/test.rs | 0 4 files changed, 393 insertions(+), 382 deletions(-) create mode 100644 src/bin/nydus-image/unpack/mod.rs rename src/bin/nydus-image/{unpack.rs => unpack/pax.rs} (65%) rename src/bin/nydus-image/unpack/{ => pax}/test.rs (100%) diff --git a/src/bin/nydus-image/main.rs b/src/bin/nydus-image/main.rs index 6130d936992..7a37951ebaf 100644 --- a/src/bin/nydus-image/main.rs +++ b/src/bin/nydus-image/main.rs @@ -513,7 +513,7 @@ fn prepare_cmd_args(bti_string: String) -> ArgMatches<'static> { ) .subcommand( SubCommand::with_name("unpack") - .about("Decompress nydus bootstrap and blob files to tar") + .about("Unpack nydus image layer to a tar file") .arg( Arg::with_name("bootstrap") .long("bootstrap") @@ -533,7 +533,7 @@ fn prepare_cmd_args(bti_string: String) -> ArgMatches<'static> { Arg::with_name("output") .long("output") .short("o") - .help("path to output") + .help("path to output tar file") .required(true) .takes_value(true) ) @@ -801,9 +801,8 @@ impl Command { let blob = args.value_of("blob"); let output = args.value_of("output").expect("pass in output"); - let unpacker = Box::new( - OCIUnpacker::new(bootstrap, blob, output).with_context(|| "fail to create unpacker")?, - ) as Box; + let unpacker = + OCIUnpacker::new(bootstrap, blob, output).with_context(|| "fail to create unpacker")?; unpacker.unpack().with_context(|| "fail to unpack") } diff --git a/src/bin/nydus-image/unpack/mod.rs b/src/bin/nydus-image/unpack/mod.rs new file mode 100644 index 00000000000..a58bc104313 --- /dev/null +++ b/src/bin/nydus-image/unpack/mod.rs @@ -0,0 +1,342 @@ +use std::{ + fs::{File, OpenOptions}, + io::Read, + iter::from_fn, + path::{Path, PathBuf}, + rc::Rc, + str, + sync::Arc, +}; + +use anyhow::{Context, Result}; +use nydus_api::http::LocalFsConfig; +use nydus_rafs::{ + metadata::{RafsInode, RafsMode, RafsSuper}, + RafsIoReader, +}; +use storage::{ + backend::{localfs::LocalFs, BlobBackend, BlobReader}, + device::BlobInfo, +}; +use tar::{Builder, Header}; + +use self::pax::{ + OCIBlockBuilder, OCICharBuilder, OCIDirBuilder, OCIFifoBuilder, OCILinkBuilder, OCIRegBuilder, + OCISocketBuilder, OCISymlinkBuilder, PAXExtensionSectionBuilder, PAXLinkBuilder, + PAXSpecialSectionBuilder, +}; + +mod pax; + +pub trait Unpacker { + fn unpack(&self) -> Result<()>; +} + +/// A unpacker with the ability to convert bootstrap file and blob file to tar +pub struct OCIUnpacker { + bootstrap: PathBuf, + blob: Option, + output: PathBuf, + + builder_factory: OCITarBuilderFactory, +} + +impl OCIUnpacker { + pub fn new(bootstrap: &str, blob: Option<&str>, output: &str) -> Result { + let bootstrap = PathBuf::from(bootstrap); + let output = PathBuf::from(output); + let blob = blob.map(PathBuf::from); + + let builder_factory = OCITarBuilderFactory::new(); + + Ok(OCIUnpacker { + builder_factory, + bootstrap, + blob, + output, + }) + } + + fn load_rafs(&self) -> Result { + let bootstrap = OpenOptions::new() + .read(true) + .write(false) + .open(&*self.bootstrap) + .with_context(|| format!("fail to open bootstrap {:?}", self.bootstrap))?; + + let mut rs = RafsSuper { + mode: RafsMode::Direct, + validate_digest: false, + ..Default::default() + }; + + rs.load(&mut (Box::new(bootstrap) as RafsIoReader)) + .with_context(|| format!("fail to load bootstrap {:?}", self.bootstrap))?; + + Ok(rs) + } + + /// A lazy iterator of RafsInode in DFS, which travels in preorder. + fn iterator<'a>( + &'a self, + rs: &'a RafsSuper, + ) -> Box, PathBuf)> + 'a> { + // A cursor means the next node to be visited at same height in the tree. + // It always starts with the first one of level. + // A cursor stack is of cursors from root to leaf. + let mut cursor_stack = Vec::with_capacity(32); + + cursor_stack.push(self.cursor_of_root(rs)); + + let dfs = move || { + while !cursor_stack.is_empty() { + let mut cursor = cursor_stack.pop().unwrap(); + + let (node, path) = match cursor.next() { + None => continue, + Some(point) => { + cursor_stack.push(cursor); + point + } + }; + + if node.is_dir() { + cursor_stack.push(self.cursor_of_children(node.clone(), &*path)) + } + + return Some((node, path)); + } + + None + }; + + Box::new(from_fn(dfs)) + } + + fn cursor_of_children( + &self, + node: Arc, + path: &Path, + ) -> Box, PathBuf)>> { + let base = path.to_path_buf(); + let mut next_idx = 0..node.get_child_count(); + + let visitor = move || { + if next_idx.is_empty() { + return None; + } + + let child = node.get_child_by_index(next_idx.next().unwrap()).unwrap(); + let child_path = base.join(child.name()); + + Some((child, child_path)) + }; + + Box::new(from_fn(visitor)) + } + + fn cursor_of_root<'a>( + &self, + rs: &'a RafsSuper, + ) -> Box, PathBuf)> + 'a> { + let mut has_more = true; + let visitor = from_fn(move || { + if !has_more { + return None; + } + has_more = false; + + let node = rs.get_inode(rs.superblock.root_ino(), false).unwrap(); + let path = PathBuf::from("/").join(node.name()); + + Some((node, path)) + }); + + Box::new(visitor) + } +} + +impl Unpacker for OCIUnpacker { + fn unpack(&self) -> Result<()> { + info!( + "oci unpacker, bootstrap file: {:?}, blob file: {:?}, output file: {:?}", + self.bootstrap, self.blob, self.output + ); + + let rafs = self.load_rafs()?; + + let mut builder = self + .builder_factory + .create(&rafs, self.blob.as_deref(), &self.output)?; + + for (node, path) in self.iterator(&rafs) { + builder.append(&*node, &path)?; + } + + Ok(()) + } +} + +trait TarBuilder { + fn append(&mut self, node: &dyn RafsInode, path: &Path) -> Result<()>; +} + +struct TarSection { + header: Header, + data: Box, +} + +trait SectionBuilder { + fn can_handle(&mut self, inode: &dyn RafsInode, path: &Path) -> bool; + fn build(&self, inode: &dyn RafsInode, path: &Path) -> Result>; +} + +struct OCITarBuilderFactory {} + +impl OCITarBuilderFactory { + fn new() -> Self { + OCITarBuilderFactory {} + } + + fn create( + &self, + meta: &RafsSuper, + blob_path: Option<&Path>, + output_path: &Path, + ) -> Result> { + let writer = self.create_writer(output_path)?; + + let blob = meta.superblock.get_blob_infos().pop(); + let builders = self.create_builders(blob, blob_path)?; + + let builder = OCITarBuilder::new(builders, writer); + + Ok(Box::new(builder) as Box) + } + + fn create_writer(&self, output_path: &Path) -> Result> { + let builder = Builder::new( + OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .read(false) + .open(output_path) + .with_context(|| format!("fail to open output file {:?}", output_path))?, + ); + + Ok(builder) + } + + fn create_builders( + &self, + blob: Option>, + blob_path: Option<&Path>, + ) -> Result>> { + // PAX basic builders + let ext_builder = Rc::new(PAXExtensionSectionBuilder::new()); + let link_builder = Rc::new(PAXLinkBuilder::new(ext_builder.clone())); + let special_builder = Rc::new(PAXSpecialSectionBuilder::new(ext_builder.clone())); + + // OCI builders + let sock_builder = OCISocketBuilder::new(); + let hard_link_builder = OCILinkBuilder::new(link_builder.clone()); + let symlink_builder = OCISymlinkBuilder::new(link_builder); + let dir_builder = OCIDirBuilder::new(ext_builder); + let fifo_builder = OCIFifoBuilder::new(special_builder.clone()); + let char_builder = OCICharBuilder::new(special_builder.clone()); + let block_builder = OCIBlockBuilder::new(special_builder); + let reg_builder = self.create_reg_builder(blob, blob_path)?; + + // The order counts. + let builders = vec![ + Box::new(sock_builder) as Box, + Box::new(hard_link_builder), + Box::new(dir_builder), + Box::new(reg_builder), + Box::new(symlink_builder), + Box::new(fifo_builder), + Box::new(char_builder), + Box::new(block_builder), + ]; + + Ok(builders) + } + + fn create_reg_builder( + &self, + blob: Option>, + blob_path: Option<&Path>, + ) -> Result { + let (reader, compressor) = match blob { + None => (None, None), + Some(ref blob) => { + if blob_path.is_none() { + bail!("miss blob path") + } + + let reader = self.create_blob_reader(blob_path.unwrap())?; + let compressor = blob.compressor(); + + (Some(reader), Some(compressor)) + } + }; + + Ok(OCIRegBuilder::new( + Rc::new(PAXExtensionSectionBuilder::new()), + reader, + compressor, + )) + } + + fn create_blob_reader(&self, blob_path: &Path) -> Result> { + let config = LocalFsConfig { + blob_file: blob_path.to_str().unwrap().to_owned(), + readahead: false, + readahead_sec: Default::default(), + dir: Default::default(), + alt_dirs: Default::default(), + }; + let config = serde_json::to_value(config) + .with_context(|| format!("fail to create local backend config for {:?}", blob_path))?; + + let backend = LocalFs::new(config, Some("unpacker")) + .with_context(|| format!("fail to create local backend for {:?}", blob_path))?; + + let reader = backend + .get_reader("") + .map_err(|err| anyhow!("fail to get reader, error {:?}", err))?; + + Ok(reader) + } +} + +struct OCITarBuilder { + writer: Builder, + builders: Vec>, +} + +impl OCITarBuilder { + fn new(builders: Vec>, writer: Builder) -> Self { + Self { builders, writer } + } +} + +impl TarBuilder for OCITarBuilder { + fn append(&mut self, inode: &dyn RafsInode, path: &Path) -> Result<()> { + for builder in &mut self.builders { + // Useless one, just go !!!!! + if !builder.can_handle(inode, path) { + continue; + } + + for sect in builder.build(inode, path)? { + self.writer.append(§.header, sect.data)?; + } + + return Ok(()); + } + + bail!("node {:?} can not be unpacked", path) + } +} diff --git a/src/bin/nydus-image/unpack.rs b/src/bin/nydus-image/unpack/pax.rs similarity index 65% rename from src/bin/nydus-image/unpack.rs rename to src/bin/nydus-image/unpack/pax.rs index 96bd9d5e477..df68e70742a 100644 --- a/src/bin/nydus-image/unpack.rs +++ b/src/bin/nydus-image/unpack/pax.rs @@ -1,9 +1,8 @@ use std::{ collections::HashMap, ffi::OsStr, - fs::{File, OpenOptions}, io::{self, Cursor, Error, ErrorKind, Read}, - iter::{self, from_fn, repeat}, + iter::{self, repeat}, os::unix::prelude::{OsStrExt, OsStringExt}, path::{Path, PathBuf}, rc::Rc, @@ -13,196 +12,24 @@ use std::{ }; use anyhow::{Context, Result}; -use nydus_api::http::LocalFsConfig; -use nydus_rafs::{ - metadata::{RafsInode, RafsMode, RafsSuper}, - RafsIoReader, -}; +use nydus_rafs::metadata::RafsInode; use nydus_utils::compress::{self, Algorithm}; -use storage::{ - backend::{localfs::LocalFs, BlobBackend, BlobReader}, - device::{BlobChunkInfo, BlobInfo}, - utils::alloc_buf, -}; -use tar::{Builder, EntryType, Header}; +use storage::{backend::BlobReader, device::BlobChunkInfo, utils::alloc_buf}; +use tar::{EntryType, Header}; use crate::core::node::InodeWrapper; +use super::{SectionBuilder, TarSection}; + static PAX_SEP1: &[u8; 1] = b" "; static PAX_SEP2: &[u8; 1] = b"="; static PAX_PREFIX: &[u8; 13] = b"SCHILY.xattr."; static PAX_DELIMITER: &[u8; 1] = b"\n"; -pub trait Unpacker { - fn unpack(&self) -> Result<()>; -} - -/// A unpacker with the ability to convert bootstrap file and blob file to tar -pub struct OCIUnpacker { - bootstrap: PathBuf, - blob: Option, - output: PathBuf, - - builder_factory: OCITarBuilderFactory, -} - -impl OCIUnpacker { - pub fn new(bootstrap: &str, blob: Option<&str>, output: &str) -> Result { - let bootstrap = PathBuf::from(bootstrap); - let output = PathBuf::from(output); - let blob = blob.map(PathBuf::from); - - let builder_factory = OCITarBuilderFactory::new(); - - Ok(OCIUnpacker { - builder_factory, - bootstrap, - blob, - output, - }) - } - - fn load_rafs(&self) -> Result { - let bootstrap = OpenOptions::new() - .read(true) - .write(false) - .open(&*self.bootstrap) - .with_context(|| format!("fail to open bootstrap {:?}", self.bootstrap))?; - - let mut rs = RafsSuper { - mode: RafsMode::Direct, - validate_digest: false, - ..Default::default() - }; - - rs.load(&mut (Box::new(bootstrap) as RafsIoReader)) - .with_context(|| format!("fail to load bootstrap {:?}", self.bootstrap))?; - - Ok(rs) - } - - /// A lazy iterator of RafsInode in DFS, which travels in preorder. - fn iterator<'a>( - &'a self, - rs: &'a RafsSuper, - ) -> Box, PathBuf)> + 'a> { - // A cursor means the next node to be visited at same height in the tree. - // It always starts with the first one of level. - // A cursor stack is of cursors from root to leaf. - let mut cursor_stack = Vec::with_capacity(32); - - cursor_stack.push(self.cursor_of_root(rs)); - - let dfs = move || { - while !cursor_stack.is_empty() { - let mut cursor = cursor_stack.pop().unwrap(); - - let (node, path) = match cursor.next() { - None => continue, - Some(point) => { - cursor_stack.push(cursor); - point - } - }; - - if node.is_dir() { - cursor_stack.push(self.cursor_of_children(node.clone(), &*path)) - } - - return Some((node, path)); - } - - None - }; - - Box::new(from_fn(dfs)) - } - - fn cursor_of_children( - &self, - node: Arc, - path: &Path, - ) -> Box, PathBuf)>> { - let base = path.to_path_buf(); - let mut next_idx = 0..node.get_child_count(); - - let visitor = move || { - if next_idx.is_empty() { - return None; - } - - let child = node.get_child_by_index(next_idx.next().unwrap()).unwrap(); - let child_path = base.join(child.name()); - - Some((child, child_path)) - }; - - Box::new(from_fn(visitor)) - } - - fn cursor_of_root<'a>( - &self, - rs: &'a RafsSuper, - ) -> Box, PathBuf)> + 'a> { - let mut has_more = true; - let visitor = from_fn(move || { - if !has_more { - return None; - } - has_more = false; - - let node = rs.get_inode(rs.superblock.root_ino(), false).unwrap(); - let path = PathBuf::from("/").join(node.name()); - - Some((node, path)) - }); - - Box::new(visitor) - } -} - -impl Unpacker for OCIUnpacker { - fn unpack(&self) -> Result<()> { - info!( - "oci unpacker, bootstrap file: {:?}, blob file: {:?}, output file: {:?}", - self.bootstrap, self.blob, self.output - ); - - let rafs = self.load_rafs()?; - - let mut builder = self - .builder_factory - .create(&rafs, self.blob.as_deref(), &self.output)?; - - for (node, path) in self.iterator(&rafs) { - if Util::is_root(&path) { - continue; - } - builder.append(&*node, &path)?; - } - - Ok(()) - } -} - -trait TarBuilder { - fn append(&mut self, node: &dyn RafsInode, path: &Path) -> Result<()>; -} - -struct TarSection { - header: Header, - data: Box, -} - -trait SectionBuilder { - fn can_handle(&mut self, inode: &dyn RafsInode, path: &Path) -> bool; - fn build(&self, inode: &dyn RafsInode, path: &Path) -> Result>; -} - -struct OCISocketBuilder {} +pub struct OCISocketBuilder {} impl OCISocketBuilder { - fn new() -> Self { + pub fn new() -> Self { OCISocketBuilder {} } } @@ -217,13 +44,13 @@ impl SectionBuilder for OCISocketBuilder { } } -struct OCILinkBuilder { - links: HashMap>, +pub struct OCILinkBuilder { + links: HashMap, pax_link_builder: Rc, } impl OCILinkBuilder { - fn new(pax_link_builder: Rc) -> Self { + pub fn new(pax_link_builder: Rc) -> Self { OCILinkBuilder { links: HashMap::new(), pax_link_builder, @@ -239,8 +66,7 @@ impl SectionBuilder for OCILinkBuilder { let is_appeared = self.links.contains_key(&node.ino()); if !is_appeared { - self.links - .insert(node.ino(), path.to_path_buf().into_boxed_path()); + self.links.insert(node.ino(), path.to_path_buf()); } is_appeared @@ -254,14 +80,18 @@ impl SectionBuilder for OCILinkBuilder { } } -struct OCIDirBuilder { +pub struct OCIDirBuilder { ext_builder: Rc, } impl OCIDirBuilder { - fn new(ext_builder: Rc) -> Self { + pub fn new(ext_builder: Rc) -> Self { OCIDirBuilder { ext_builder } } + + fn is_root(&self, path: &Path) -> bool { + path.is_absolute() && path.file_name().is_none() + } } impl SectionBuilder for OCIDirBuilder { @@ -270,6 +100,10 @@ impl SectionBuilder for OCIDirBuilder { } fn build(&self, inode: &dyn RafsInode, path: &Path) -> Result> { + if self.is_root(path) { + return Ok(Vec::new()); + } + let mut header = Header::new_ustar(); header.set_entry_type(EntryType::dir()); header.set_size(0); @@ -307,14 +141,14 @@ impl SectionBuilder for OCIDirBuilder { } } -struct OCIRegBuilder { +pub struct OCIRegBuilder { ext_builder: Rc, reader: Option>, compressor: Option, } impl OCIRegBuilder { - fn new( + pub fn new( ext_builder: Rc, reader: Option>, compressor: Option, @@ -388,12 +222,12 @@ impl SectionBuilder for OCIRegBuilder { } } -struct OCISymlinkBuilder { +pub struct OCISymlinkBuilder { pax_link_builder: Rc, } impl OCISymlinkBuilder { - fn new(pax_link_builder: Rc) -> Self { + pub fn new(pax_link_builder: Rc) -> Self { OCISymlinkBuilder { pax_link_builder } } } @@ -411,12 +245,12 @@ impl SectionBuilder for OCISymlinkBuilder { } } -struct OCIFifoBuilder { +pub struct OCIFifoBuilder { pax_special_builder: Rc, } impl OCIFifoBuilder { - fn new(pax_special_builder: Rc) -> Self { + pub fn new(pax_special_builder: Rc) -> Self { OCIFifoBuilder { pax_special_builder, } @@ -434,12 +268,12 @@ impl SectionBuilder for OCIFifoBuilder { } } -struct OCICharBuilder { +pub struct OCICharBuilder { pax_special_builder: Rc, } impl OCICharBuilder { - fn new(pax_special_builder: Rc) -> Self { + pub fn new(pax_special_builder: Rc) -> Self { OCICharBuilder { pax_special_builder, } @@ -457,12 +291,12 @@ impl SectionBuilder for OCICharBuilder { } } -struct OCIBlockBuilder { +pub struct OCIBlockBuilder { pax_special_builder: Rc, } impl OCIBlockBuilder { - fn new(pax_special_builder: Rc) -> Self { + pub fn new(pax_special_builder: Rc) -> Self { OCIBlockBuilder { pax_special_builder, } @@ -480,12 +314,12 @@ impl SectionBuilder for OCIBlockBuilder { } } -struct PAXSpecialSectionBuilder { +pub struct PAXSpecialSectionBuilder { ext_builder: Rc, } impl PAXSpecialSectionBuilder { - fn new(ext_builder: Rc) -> Self { + pub fn new(ext_builder: Rc) -> Self { PAXSpecialSectionBuilder { ext_builder } } @@ -546,10 +380,10 @@ struct PAXRecord { v: Vec, } -struct PAXExtensionSectionBuilder {} +pub struct PAXExtensionSectionBuilder {} impl PAXExtensionSectionBuilder { - fn new() -> Self { + pub fn new() -> Self { PAXExtensionSectionBuilder {} } @@ -635,12 +469,12 @@ impl PAXExtensionSectionBuilder { } } -struct PAXLinkBuilder { +pub struct PAXLinkBuilder { ext_builder: Rc, } impl PAXLinkBuilder { - fn new(ext_builder: Rc) -> Self { + pub fn new(ext_builder: Rc) -> Self { PAXLinkBuilder { ext_builder } } @@ -771,13 +605,9 @@ impl PAXUtil { } } -struct Util {} +pub struct Util {} impl Util { - fn is_root(path: &Path) -> bool { - path.is_absolute() && path.file_name().is_none() - } - fn normalize_path(path: &Path) -> Result { fn end_with_slash(p: &Path) -> bool { p.as_os_str().as_bytes().last() == Some(&b'/') @@ -866,164 +696,6 @@ impl Util { } } -struct OCITarBuilderFactory {} - -impl OCITarBuilderFactory { - fn new() -> Self { - OCITarBuilderFactory {} - } - - fn create( - &self, - meta: &RafsSuper, - blob_path: Option<&Path>, - output_path: &Path, - ) -> Result> { - let writer = self.create_writer(output_path)?; - - let blob = meta.superblock.get_blob_infos().pop(); - let builders = self.create_builders(blob, blob_path)?; - - let builder = OCITarBuilder::new(builders, writer); - - Ok(Box::new(builder) as Box) - } - - fn create_writer(&self, output_path: &Path) -> Result> { - let builder = Builder::new( - OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .read(false) - .open(output_path) - .with_context(|| format!("fail to open output file {:?}", output_path))?, - ); - - Ok(builder) - } - - fn create_builders( - &self, - blob: Option>, - blob_path: Option<&Path>, - ) -> Result>> { - // PAX basic builders - let ext_builder = Rc::new(PAXExtensionSectionBuilder::new()); - let link_builder = Rc::new(PAXLinkBuilder::new(ext_builder.clone())); - let special_builder = Rc::new(PAXSpecialSectionBuilder::new(ext_builder.clone())); - - // OCI builders - let sock_builder = OCISocketBuilder::new(); - let hard_link_builder = OCILinkBuilder::new(link_builder.clone()); - let symlink_builder = OCISymlinkBuilder::new(link_builder); - let dir_builder = OCIDirBuilder::new(ext_builder); - let fifo_builder = OCIFifoBuilder::new(special_builder.clone()); - let char_builder = OCICharBuilder::new(special_builder.clone()); - let block_builder = OCIBlockBuilder::new(special_builder); - let reg_builder = self.create_reg_builder(blob, blob_path)?; - - // The order counts. - let builders = vec![ - Box::new(sock_builder) as Box, - Box::new(hard_link_builder), - Box::new(dir_builder), - Box::new(reg_builder), - Box::new(symlink_builder), - Box::new(fifo_builder), - Box::new(char_builder), - Box::new(block_builder), - ]; - - Ok(builders) - } - - fn create_reg_builder( - &self, - blob: Option>, - blob_path: Option<&Path>, - ) -> Result { - let (reader, compressor) = match blob { - None => (None, None), - Some(ref blob) => { - if blob_path.is_none() { - bail!("miss blob path") - } - - let reader = self.create_blob_reader(blob_path.unwrap())?; - let compressor = blob.compressor(); - - (Some(reader), Some(compressor)) - } - }; - - Ok(OCIRegBuilder::new( - Rc::new(PAXExtensionSectionBuilder::new()), - reader, - compressor, - )) - } - - fn create_blob_reader(&self, blob_path: &Path) -> Result> { - let config = LocalFsConfig { - blob_file: blob_path.to_str().unwrap().to_owned(), - readahead: false, - readahead_sec: Default::default(), - dir: Default::default(), - alt_dirs: Default::default(), - }; - let config = serde_json::to_value(config) - .with_context(|| format!("fail to create local backend config for {:?}", blob_path))?; - - let backend = LocalFs::new(config, Some("unpacker")) - .with_context(|| format!("fail to create local backend for {:?}", blob_path))?; - - let reader = backend - .get_reader("") - .map_err(|err| anyhow!("fail to get reader, error {:?}", err))?; - - Ok(reader) - } -} - -struct OCITarBuilder { - writer: Builder, - builders: Vec>, -} - -impl OCITarBuilder { - fn new(builders: Vec>, writer: Builder) -> Self { - Self { builders, writer } - } -} - -impl TarBuilder for OCITarBuilder { - fn append(&mut self, inode: &dyn RafsInode, path: &Path) -> Result<()> { - let mut is_known_type = true; - - for builder in &mut self.builders { - // Useless one, just go !!!!! - if !builder.can_handle(inode, path) { - continue; - } - - for sect in builder.build(inode, path)? { - self.writer.append(§.header, sect.data)?; - } - - is_known_type = false; - - break; - } - - if is_known_type { - bail!("node {:?} can not be unpacked", path) - } - - Ok(()) - } -} - struct ChunkReader { compressor: Algorithm, reader: Arc, @@ -1085,17 +757,15 @@ impl Read for ChunkReader { loop { if self.is_chunk_empty() { - let chunk = self.chunks.next(); - if chunk.is_none() { - break; + match self.chunks.next() { + None => break, + Some(chunk) => self.load_chunk(chunk.as_ref()).map_err(|err| { + Error::new( + ErrorKind::InvalidData, + format!("fail to load chunk, error: {}", err), + ) + })?, } - - self.load_chunk(chunk.unwrap().as_ref()).map_err(|err| { - Error::new( - ErrorKind::InvalidData, - format!("fail to load chunk, error: {}", err), - ) - })?; } size += Read::read(&mut self.chunk, &mut buf[size..])?; diff --git a/src/bin/nydus-image/unpack/test.rs b/src/bin/nydus-image/unpack/pax/test.rs similarity index 100% rename from src/bin/nydus-image/unpack/test.rs rename to src/bin/nydus-image/unpack/pax/test.rs