From dc54beea4d280090b7aa9353eac645b6e001db0f Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Fri, 3 Mar 2023 15:58:42 +0800 Subject: [PATCH 01/10] rafs: optimize ChunkWrapper to reduce memory consumption Optimize ChunkWrapper to reduce memory consumption by only instantialing the chunk info object when needed. Signed-off-by: Jiang Liu --- rafs/src/metadata/chunk.rs | 146 ++++++++++++++++++----- src/bin/nydus-image/core/blob_compact.rs | 1 + src/bin/nydus-image/core/chunk_dict.rs | 2 +- src/bin/nydus-image/core/tree.rs | 2 +- 4 files changed, 117 insertions(+), 34 deletions(-) diff --git a/rafs/src/metadata/chunk.rs b/rafs/src/metadata/chunk.rs index 1d04236afe1..35e2a644356 100644 --- a/rafs/src/metadata/chunk.rs +++ b/rafs/src/metadata/chunk.rs @@ -3,7 +3,9 @@ // // SPDX-License-Identifier: Apache-2.0 -use std::fmt::{self, Display, Formatter}; +use std::fmt::{self, Debug, Display, Formatter}; +use std::ops::Deref; +use std::sync::Arc; use anyhow::{Context, Result}; use nydus_storage::device::v5::BlobV5ChunkInfo; @@ -18,13 +20,45 @@ use crate::metadata::layout::v5::RafsV5ChunkInfo; use crate::metadata::{RafsStore, RafsVersion}; use crate::RafsIoWrite; -/// A ChunkInfo object wrapper for different RAFS versions. -#[derive(Clone, Debug)] +/// A wrapper to encapsulate different versions of chunk information objects. +#[derive(Clone)] pub enum ChunkWrapper { - /// Chunk info structure for RAFS v5. + /// Chunk info for RAFS v5. V5(RafsV5ChunkInfo), - /// Chunk info structure for RAFS v6, reuse `RafsV5ChunkInfo` as IR for v6. + /// Chunk info RAFS v6, reuse `RafsV5ChunkInfo` as IR for v6. V6(RafsV5ChunkInfo), + /// Reference to a `BlobChunkInfo` object. + Ref(Arc), +} + +impl Debug for ChunkWrapper { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + Self::V5(c) => write!(f, "{:?}", c), + Self::V6(c) => write!(f, "{:?}", c), + Self::Ref(c) => { + let chunk = to_rafs_v5_chunk_info(as_blob_v5_chunk_info(c.deref())); + write!(f, "{:?}", chunk) + } + } + } +} + +impl Display for ChunkWrapper { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!( + f, + "id {}, index {}, blob_index {}, file_offset {}, compressed {}/{}, uncompressed {}/{}", + self.id(), + self.index(), + self.blob_index(), + self.file_offset(), + self.compressed_offset(), + self.compressed_size(), + self.uncompressed_offset(), + self.uncompressed_size(), + ) + } } impl ChunkWrapper { @@ -37,18 +71,8 @@ impl ChunkWrapper { } /// Create a `ChunkWrapper` object from a `BlobChunkInfo` trait object. - pub fn from_chunk_info(cki: &dyn BlobChunkInfo) -> Self { - if let Some(cki) = cki.as_any().downcast_ref::() { - ChunkWrapper::V6(to_rafsv5_chunk_info(cki)) - } else if let Some(cki_v5) = cki.as_any().downcast_ref::() { - ChunkWrapper::V5(to_rafsv5_chunk_info(cki_v5)) - } else if let Some(cki_v5) = cki.as_any().downcast_ref::() { - ChunkWrapper::V5(to_rafsv5_chunk_info(cki_v5)) - } else if let Some(cki_v6) = cki.as_any().downcast_ref::() { - ChunkWrapper::V6(to_rafsv5_chunk_info(cki_v6)) - } else { - panic!("unknown chunk information struct"); - } + pub fn from_chunk_info(cki: Arc) -> Self { + Self::Ref(cki) } /// Get digest of chunk data, which is also used as chunk ID. @@ -56,14 +80,17 @@ impl ChunkWrapper { match self { ChunkWrapper::V5(c) => &c.block_id, ChunkWrapper::V6(c) => &c.block_id, + ChunkWrapper::Ref(c) => c.chunk_id(), } } /// Set digest of chunk data, which is also used as chunk ID. pub fn set_id(&mut self, id: RafsDigest) { + self.ensure_owned(); match self { ChunkWrapper::V5(c) => c.block_id = id, ChunkWrapper::V6(c) => c.block_id = id, + ChunkWrapper::Ref(_c) => panic!("unexpected"), } } @@ -72,14 +99,17 @@ impl ChunkWrapper { match self { ChunkWrapper::V5(c) => c.blob_index, ChunkWrapper::V6(c) => c.blob_index, + ChunkWrapper::Ref(c) => c.blob_index(), } } /// Set index of the data blob associated with the chunk. pub fn set_blob_index(&mut self, index: u32) { + self.ensure_owned(); match self { ChunkWrapper::V5(c) => c.blob_index = index, ChunkWrapper::V6(c) => c.blob_index = index, + ChunkWrapper::Ref(_c) => panic!("unexpected"), } } @@ -88,14 +118,17 @@ impl ChunkWrapper { match self { ChunkWrapper::V5(c) => c.compressed_offset, ChunkWrapper::V6(c) => c.compressed_offset, + ChunkWrapper::Ref(c) => c.compressed_offset(), } } /// Set offset into the compressed data blob to fetch chunk data. pub fn set_compressed_offset(&mut self, offset: u64) { + self.ensure_owned(); match self { ChunkWrapper::V5(c) => c.compressed_offset = offset, ChunkWrapper::V6(c) => c.compressed_offset = offset, + ChunkWrapper::Ref(_c) => panic!("unexpected"), } } @@ -104,14 +137,17 @@ impl ChunkWrapper { match self { ChunkWrapper::V5(c) => c.compressed_size, ChunkWrapper::V6(c) => c.compressed_size, + ChunkWrapper::Ref(c) => c.compressed_size(), } } /// Set size of compressed chunk data. pub fn set_compressed_size(&mut self, size: u32) { + self.ensure_owned(); match self { ChunkWrapper::V5(c) => c.compressed_size = size, ChunkWrapper::V6(c) => c.compressed_size = size, + ChunkWrapper::Ref(_c) => panic!("unexpected"), } } @@ -120,14 +156,17 @@ impl ChunkWrapper { match self { ChunkWrapper::V5(c) => c.uncompressed_offset, ChunkWrapper::V6(c) => c.uncompressed_offset, + ChunkWrapper::Ref(c) => c.uncompressed_offset(), } } /// Set offset into the uncompressed data blob file to get chunk data. pub fn set_uncompressed_offset(&mut self, offset: u64) { + self.ensure_owned(); match self { ChunkWrapper::V5(c) => c.uncompressed_offset = offset, ChunkWrapper::V6(c) => c.uncompressed_offset = offset, + ChunkWrapper::Ref(_c) => panic!("unexpected"), } } @@ -136,14 +175,17 @@ impl ChunkWrapper { match self { ChunkWrapper::V5(c) => c.uncompressed_size, ChunkWrapper::V6(c) => c.uncompressed_size, + ChunkWrapper::Ref(c) => c.uncompressed_size(), } } /// Set size of uncompressed chunk data. pub fn set_uncompressed_size(&mut self, size: u32) { + self.ensure_owned(); match self { ChunkWrapper::V5(c) => c.uncompressed_size = size, ChunkWrapper::V6(c) => c.uncompressed_size = size, + ChunkWrapper::Ref(_c) => panic!("unexpected"), } } @@ -152,14 +194,17 @@ impl ChunkWrapper { match self { ChunkWrapper::V5(c) => c.index, ChunkWrapper::V6(c) => c.index, + ChunkWrapper::Ref(c) => as_blob_v5_chunk_info(c.deref()).index(), } } /// Set chunk index into the RAFS chunk information array, used by RAFS v5. pub fn set_index(&mut self, index: u32) { + self.ensure_owned(); match self { ChunkWrapper::V5(c) => c.index = index, ChunkWrapper::V6(c) => c.index = index, + ChunkWrapper::Ref(_c) => panic!("unexpected"), } } @@ -168,14 +213,17 @@ impl ChunkWrapper { match self { ChunkWrapper::V5(c) => c.file_offset, ChunkWrapper::V6(c) => c.file_offset, + ChunkWrapper::Ref(c) => as_blob_v5_chunk_info(c.deref()).file_offset(), } } /// Set chunk offset in the file it belongs to, RAFS v5. pub fn set_file_offset(&mut self, offset: u64) { + self.ensure_owned(); match self { ChunkWrapper::V5(c) => c.file_offset = offset, ChunkWrapper::V6(c) => c.file_offset = offset, + ChunkWrapper::Ref(_c) => panic!("unexpected"), } } @@ -184,14 +232,19 @@ impl ChunkWrapper { match self { ChunkWrapper::V5(c) => c.flags.contains(BlobChunkFlags::COMPRESSED), ChunkWrapper::V6(c) => c.flags.contains(BlobChunkFlags::COMPRESSED), + ChunkWrapper::Ref(c) => as_blob_v5_chunk_info(c.deref()) + .flags() + .contains(BlobChunkFlags::COMPRESSED), } } /// Set flag for whether chunk is compressed. pub fn set_compressed(&mut self, compressed: bool) { + self.ensure_owned(); match self { ChunkWrapper::V5(c) => c.flags.set(BlobChunkFlags::COMPRESSED, compressed), ChunkWrapper::V6(c) => c.flags.set(BlobChunkFlags::COMPRESSED, compressed), + ChunkWrapper::Ref(_c) => panic!("unexpected"), } } @@ -208,6 +261,7 @@ impl ChunkWrapper { compressed_size: u32, is_compressed: bool, ) -> Result<()> { + self.ensure_owned(); match self { ChunkWrapper::V5(c) => { c.index = chunk_index; @@ -233,6 +287,7 @@ impl ChunkWrapper { c.flags |= BlobChunkFlags::COMPRESSED; } } + ChunkWrapper::Ref(_c) => panic!("unexpected"), } Ok(()) @@ -240,11 +295,21 @@ impl ChunkWrapper { /// Copy chunk information from another `ChunkWrapper` object. pub fn copy_from(&mut self, other: &Self) { + self.ensure_owned(); match (self, other) { (ChunkWrapper::V5(s), ChunkWrapper::V5(o)) => s.clone_from(o), (ChunkWrapper::V6(s), ChunkWrapper::V6(o)) => s.clone_from(o), (ChunkWrapper::V5(s), ChunkWrapper::V6(o)) => s.clone_from(o), (ChunkWrapper::V6(s), ChunkWrapper::V5(o)) => s.clone_from(o), + (ChunkWrapper::V5(s), ChunkWrapper::Ref(o)) => { + s.clone_from(&to_rafs_v5_chunk_info(as_blob_v5_chunk_info(o.deref()))) + } + (ChunkWrapper::V6(s), ChunkWrapper::Ref(o)) => { + s.clone_from(&to_rafs_v5_chunk_info(as_blob_v5_chunk_info(o.deref()))) + } + (ChunkWrapper::Ref(_s), ChunkWrapper::V5(_o)) => panic!("unexpected"), + (ChunkWrapper::Ref(_s), ChunkWrapper::V6(_o)) => panic!("unexpected"), + (ChunkWrapper::Ref(_s), ChunkWrapper::Ref(_o)) => panic!("unexpected"), } } @@ -253,29 +318,46 @@ impl ChunkWrapper { match self { ChunkWrapper::V5(c) => c.store(w).context("failed to store rafs v5 chunk"), ChunkWrapper::V6(c) => c.store(w).context("failed to store rafs v6 chunk"), + ChunkWrapper::Ref(c) => { + let chunk = to_rafs_v5_chunk_info(as_blob_v5_chunk_info(c.deref())); + chunk.store(w).context("failed to store rafs v6 chunk") + } + } + } + + fn ensure_owned(&mut self) { + if let Self::Ref(cki) = self { + if let Some(cki_v6) = cki.as_any().downcast_ref::() { + *self = Self::V6(to_rafs_v5_chunk_info(cki_v6)); + } else if let Some(cki_v6) = cki.as_any().downcast_ref::() { + *self = Self::V6(to_rafs_v5_chunk_info(cki_v6)); + } else if let Some(cki_v5) = cki.as_any().downcast_ref::() { + *self = Self::V6(to_rafs_v5_chunk_info(cki_v5)); + } else if let Some(cki_v5) = cki.as_any().downcast_ref::() { + *self = Self::V6(to_rafs_v5_chunk_info(cki_v5)); + } else { + panic!("unknown chunk information struct"); + } } } } -impl Display for ChunkWrapper { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!( - f, - "id {}, index {}, blob_index {}, file_offset {}, compressed {}/{}, uncompressed {}/{}", - self.id(), - self.index(), - self.blob_index(), - self.file_offset(), - self.compressed_offset(), - self.compressed_size(), - self.uncompressed_offset(), - self.uncompressed_size(), - ) +fn as_blob_v5_chunk_info(cki: &dyn BlobChunkInfo) -> &dyn BlobV5ChunkInfo { + if let Some(cki_v6) = cki.as_any().downcast_ref::() { + cki_v6 + } else if let Some(cki_v6) = cki.as_any().downcast_ref::() { + cki_v6 + } else if let Some(cki_v5) = cki.as_any().downcast_ref::() { + cki_v5 + } else if let Some(cki_v5) = cki.as_any().downcast_ref::() { + cki_v5 + } else { + panic!("unknown chunk information struct"); } } /// Construct a `RafsV5ChunkInfo` object from a `dyn BlobChunkInfo` object. -fn to_rafsv5_chunk_info(cki: &dyn BlobV5ChunkInfo) -> RafsV5ChunkInfo { +fn to_rafs_v5_chunk_info(cki: &dyn BlobV5ChunkInfo) -> RafsV5ChunkInfo { RafsV5ChunkInfo { block_id: *cki.chunk_id(), blob_index: cki.blob_index(), diff --git a/src/bin/nydus-image/core/blob_compact.rs b/src/bin/nydus-image/core/blob_compact.rs index ca190b45c16..8ba7cda57a3 100644 --- a/src/bin/nydus-image/core/blob_compact.rs +++ b/src/bin/nydus-image/core/blob_compact.rs @@ -79,6 +79,7 @@ impl ChunkKey { match c { ChunkWrapper::V5(_) => Self::Digest(*c.id()), ChunkWrapper::V6(_) => Self::Offset(c.blob_index(), c.compressed_offset()), + ChunkWrapper::Ref(_) => unimplemented!("unsupport ChunkWrapper::Ref(c)"), } } } diff --git a/src/bin/nydus-image/core/chunk_dict.rs b/src/bin/nydus-image/core/chunk_dict.rs index 5ae3fb3def9..223b004f5a6 100644 --- a/src/bin/nydus-image/core/chunk_dict.rs +++ b/src/bin/nydus-image/core/chunk_dict.rs @@ -163,7 +163,7 @@ impl HashChunkDict { for idx in 0..(size / unit_size) { let chunk = rs.superblock.get_chunk_info(idx)?; - self.add_chunk(ChunkWrapper::from_chunk_info(chunk.as_ref()), self.digester); + self.add_chunk(ChunkWrapper::from_chunk_info(chunk), self.digester); } Ok(()) diff --git a/src/bin/nydus-image/core/tree.rs b/src/bin/nydus-image/core/tree.rs index e0e28d904f9..9d82c881ca5 100644 --- a/src/bin/nydus-image/core/tree.rs +++ b/src/bin/nydus-image/core/tree.rs @@ -307,7 +307,7 @@ impl<'a> MetadataTreeBuilder<'a> { let cki = inode.get_chunk_info(i)?; chunks.push(NodeChunk { source: ChunkSource::Parent, - inner: ChunkWrapper::from_chunk_info(cki.as_ref()), + inner: ChunkWrapper::from_chunk_info(cki), }); } chunks From b971551a14cfcbcf715cc938590886e248d7a8ba Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Sat, 4 Mar 2023 00:39:00 +0800 Subject: [PATCH 02/10] rafs: optimize InodeWrapper to reduce memory consumption Optimize InodeWrapper to reduce memory consumption by only instantialing the inode object when needed. Signed-off-by: Jiang Liu --- rafs/src/metadata/inode.rs | 137 +++++++++++++++++++++++++++--- rafs/src/metadata/mod.rs | 14 +-- src/bin/nydus-image/core/tree.rs | 11 ++- src/bin/nydus-image/inspect.rs | 2 +- src/bin/nydus-image/merge.rs | 5 +- src/bin/nydus-image/unpack/mod.rs | 14 +-- src/bin/nydus-image/unpack/pax.rs | 57 +++++++------ 7 files changed, 174 insertions(+), 66 deletions(-) diff --git a/rafs/src/metadata/inode.rs b/rafs/src/metadata/inode.rs index d60b5eb7fb8..f19b935ed25 100644 --- a/rafs/src/metadata/inode.rs +++ b/rafs/src/metadata/inode.rs @@ -3,7 +3,10 @@ // // SPDX-License-Identifier: Apache-2.0 +use std::fmt::{Debug, Formatter}; use std::mem::size_of; +use std::ops::Deref; +use std::sync::Arc; use nydus_utils::digest::RafsDigest; @@ -18,12 +21,27 @@ use crate::metadata::{Inode, RafsVersion}; use crate::RafsInodeExt; /// An inode object wrapper for different RAFS versions. -#[derive(Clone, Debug)] +#[derive(Clone)] pub enum InodeWrapper { /// Inode info structure for RAFS v5. V5(RafsV5Inode), /// Inode info structure for RAFS v6, reuse `RafsV5Inode` as IR for v6. V6(RafsV5Inode), + /// A reference to a `RafsInodeExt` object. + Ref(Arc), +} + +impl Debug for InodeWrapper { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::V5(i) => write!(f, "{:?}", i), + Self::V6(i) => write!(f, "{:?}", i), + Self::Ref(i) => { + let i = to_rafsv5_inode(i.deref()); + write!(f, "{:?}", i) + } + } + } } impl InodeWrapper { @@ -36,16 +54,8 @@ impl InodeWrapper { } /// Create an `InodeWrapper` object from a `RafsInodeExt` trait object. - pub fn from_inode_info(inode: &dyn RafsInodeExt) -> Self { - if let Some(inode) = inode.as_any().downcast_ref::() { - InodeWrapper::V5(to_rafsv5_inode(inode)) - } else if let Some(inode) = inode.as_any().downcast_ref::() { - InodeWrapper::V5(to_rafsv5_inode(inode)) - } else if let Some(inode) = inode.as_any().downcast_ref::() { - InodeWrapper::V6(to_rafsv5_inode(inode)) - } else { - panic!("unknown inode information struct"); - } + pub fn from_inode_info(inode: Arc) -> Self { + Self::Ref(inode) } /// Check whether is a RAFS V5 inode. @@ -53,6 +63,28 @@ impl InodeWrapper { match self { InodeWrapper::V5(_i) => true, InodeWrapper::V6(_i) => false, + InodeWrapper::Ref(inode) => { + if let Some(_inode) = inode.as_any().downcast_ref::() { + true + } else { + inode + .as_any() + .downcast_ref::() + .is_some() + } + } + } + } + + /// Check whether is a RAFS V6 inode. + pub fn is_v6(&self) -> bool { + match self { + InodeWrapper::V5(_i) => true, + InodeWrapper::V6(_i) => false, + InodeWrapper::Ref(inode) => inode + .as_any() + .downcast_ref::() + .is_some(), } } @@ -60,7 +92,7 @@ impl InodeWrapper { pub fn inode_size(&self) -> usize { match self { InodeWrapper::V5(i) => i.size(), - InodeWrapper::V6(i) => i.size(), + _ => panic!("only available for RAFS v5 inode"), } } @@ -69,14 +101,17 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => i.mode(), InodeWrapper::V6(i) => i.mode(), + InodeWrapper::Ref(i) => i.get_attr().mode, } } /// Set access permission/mode for the inode. pub fn set_mode(&mut self, mode: u32) { + self.ensure_owned(); match self { InodeWrapper::V5(i) => i.i_mode = mode, InodeWrapper::V6(i) => i.i_mode = mode, + InodeWrapper::Ref(_i) => panic!("unexpected"), } } @@ -85,6 +120,7 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => i.is_dir(), InodeWrapper::V6(i) => i.is_dir(), + InodeWrapper::Ref(i) => i.is_dir(), } } @@ -93,6 +129,7 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => i.is_reg(), InodeWrapper::V6(i) => i.is_reg(), + InodeWrapper::Ref(i) => i.is_reg(), } } @@ -101,6 +138,7 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => i.is_hardlink(), InodeWrapper::V6(i) => i.is_hardlink(), + InodeWrapper::Ref(i) => i.is_hardlink(), } } @@ -109,6 +147,7 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => i.is_symlink(), InodeWrapper::V6(i) => i.is_symlink(), + InodeWrapper::Ref(i) => i.is_symlink(), } } @@ -117,6 +156,7 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => i.is_chrdev(), InodeWrapper::V6(i) => i.is_chrdev(), + InodeWrapper::Ref(_i) => unimplemented!(), } } @@ -125,6 +165,7 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => i.is_blkdev(), InodeWrapper::V6(i) => i.is_blkdev(), + InodeWrapper::Ref(_i) => unimplemented!(), } } @@ -133,6 +174,7 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => i.is_fifo(), InodeWrapper::V6(i) => i.is_fifo(), + InodeWrapper::Ref(_i) => unimplemented!(), } } @@ -141,6 +183,7 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => i.is_sock(), InodeWrapper::V6(i) => i.is_sock(), + InodeWrapper::Ref(_i) => unimplemented!(), } } @@ -154,6 +197,7 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => i.has_hardlink(), InodeWrapper::V6(i) => i.has_hardlink(), + InodeWrapper::Ref(_i) => unimplemented!(), } } @@ -162,11 +206,13 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => i.has_xattr(), InodeWrapper::V6(i) => i.has_xattr(), + InodeWrapper::Ref(i) => i.has_xattr(), } } /// Set whether the inode has associated xattrs. pub fn set_has_xattr(&mut self, enable: bool) { + self.ensure_owned(); match self { InodeWrapper::V5(i) => { if enable { @@ -182,6 +228,7 @@ impl InodeWrapper { i.i_flags &= !RafsV5InodeFlags::XATTR; } } + InodeWrapper::Ref(_i) => panic!("unexpected"), } } @@ -190,14 +237,17 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => i.i_ino, InodeWrapper::V6(i) => i.i_ino, + InodeWrapper::Ref(i) => i.ino(), } } /// Set inode number. pub fn set_ino(&mut self, ino: Inode) { + self.ensure_owned(); match self { InodeWrapper::V5(i) => i.i_ino = ino, InodeWrapper::V6(i) => i.i_ino = ino, + InodeWrapper::Ref(_i) => panic!("unexpected"), } } @@ -206,14 +256,17 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => i.i_parent, InodeWrapper::V6(i) => i.i_parent, + InodeWrapper::Ref(i) => i.parent(), } } /// Set parent inode number. pub fn set_parent(&mut self, parent: Inode) { + self.ensure_owned(); match self { InodeWrapper::V5(i) => i.i_parent = parent, InodeWrapper::V6(i) => i.i_parent = parent, + InodeWrapper::Ref(_i) => panic!("unexpected"), } } @@ -222,14 +275,17 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => i.i_size, InodeWrapper::V6(i) => i.i_size, + InodeWrapper::Ref(i) => i.size(), } } /// Get inode content size. pub fn set_size(&mut self, size: u64) { + self.ensure_owned(); match self { InodeWrapper::V5(i) => i.i_size = size, InodeWrapper::V6(i) => i.i_size = size, + InodeWrapper::Ref(_i) => panic!("unexpected"), } } @@ -238,14 +294,17 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => i.i_uid, InodeWrapper::V6(i) => i.i_uid, + InodeWrapper::Ref(_i) => unimplemented!(), } } /// Set user id associated with the inode. pub fn set_uid(&mut self, uid: u32) { + self.ensure_owned(); match self { InodeWrapper::V5(i) => i.i_uid = uid, InodeWrapper::V6(i) => i.i_uid = uid, + InodeWrapper::Ref(_i) => panic!("unexpected"), } } @@ -254,14 +313,17 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => i.i_gid, InodeWrapper::V6(i) => i.i_gid, + InodeWrapper::Ref(_i) => unimplemented!(), } } /// Set group id associated with the inode. pub fn set_gid(&mut self, gid: u32) { + self.ensure_owned(); match self { InodeWrapper::V5(i) => i.i_gid = gid, InodeWrapper::V6(i) => i.i_gid = gid, + InodeWrapper::Ref(_i) => panic!("unexpected"), } } @@ -270,14 +332,17 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => i.i_mtime, InodeWrapper::V6(i) => i.i_mtime, + InodeWrapper::Ref(_i) => unimplemented!(), } } /// Set modified time. pub fn set_mtime(&mut self, mtime: u64) { + self.ensure_owned(); match self { InodeWrapper::V5(i) => i.i_mtime = mtime, InodeWrapper::V6(i) => i.i_mtime = mtime, + InodeWrapper::Ref(_i) => panic!("unexpected"), } } @@ -286,14 +351,17 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => i.i_mtime_nsec, InodeWrapper::V6(i) => i.i_mtime_nsec, + InodeWrapper::Ref(_i) => unimplemented!(), } } /// Set nsec part of modified time. pub fn set_mtime_nsec(&mut self, mtime_nsec: u32) { + self.ensure_owned(); match self { InodeWrapper::V5(i) => i.i_mtime_nsec = mtime_nsec, InodeWrapper::V6(i) => i.i_mtime_nsec = mtime_nsec, + InodeWrapper::Ref(_i) => panic!("unexpected"), } } @@ -302,14 +370,17 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => i.i_blocks, InodeWrapper::V6(i) => i.i_blocks, + InodeWrapper::Ref(_i) => unimplemented!(), } } /// Set data blocks of file content, in unit of 512 bytes. pub fn set_blocks(&mut self, blocks: u64) { + self.ensure_owned(); match self { InodeWrapper::V5(i) => i.i_blocks = blocks, InodeWrapper::V6(i) => i.i_blocks = blocks, + InodeWrapper::Ref(_i) => panic!("unexpected"), } } @@ -318,22 +389,27 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => i.i_rdev, InodeWrapper::V6(i) => i.i_rdev, + InodeWrapper::Ref(i) => i.rdev(), } } /// Set real device id associated with the inode. pub fn set_rdev(&mut self, rdev: u32) { + self.ensure_owned(); match self { InodeWrapper::V5(i) => i.i_rdev = rdev, InodeWrapper::V6(i) => i.i_rdev = rdev, + InodeWrapper::Ref(_i) => panic!("unexpected"), } } /// Set project ID associated with the inode. pub fn set_projid(&mut self, projid: u32) { + self.ensure_owned(); match self { InodeWrapper::V5(i) => i.i_projid = projid, InodeWrapper::V6(i) => i.i_projid = projid, + InodeWrapper::Ref(_i) => panic!("unexpected"), } } @@ -342,14 +418,17 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => i.i_nlink, InodeWrapper::V6(i) => i.i_nlink, + InodeWrapper::Ref(_i) => unimplemented!(), } } /// Set number of hardlinks. pub fn set_nlink(&mut self, nlink: u32) { + self.ensure_owned(); match self { InodeWrapper::V5(i) => i.i_nlink = nlink, InodeWrapper::V6(i) => i.i_nlink = nlink, + InodeWrapper::Ref(_i) => panic!("unexpected"), } } @@ -358,14 +437,17 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => &i.i_digest, InodeWrapper::V6(i) => &i.i_digest, + InodeWrapper::Ref(_i) => unimplemented!(), } } /// Set digest of inode metadata, RAFS v5 only. pub fn set_digest(&mut self, digest: RafsDigest) { + self.ensure_owned(); match self { InodeWrapper::V5(i) => i.i_digest = digest, InodeWrapper::V6(i) => i.i_digest = digest, + InodeWrapper::Ref(_i) => panic!("unexpected"), } } @@ -374,15 +456,18 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => i.i_name_size, InodeWrapper::V6(i) => i.i_name_size, + InodeWrapper::Ref(i) => i.get_name_size(), } } /// Set size of inode name. pub fn set_name_size(&mut self, size: usize) { + self.ensure_owned(); debug_assert!(size < u16::MAX as usize); match self { InodeWrapper::V5(i) => i.i_name_size = size as u16, InodeWrapper::V6(i) => i.i_name_size = size as u16, + InodeWrapper::Ref(_i) => panic!("unexpected"), } } @@ -391,12 +476,14 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => i.i_symlink_size, InodeWrapper::V6(i) => i.i_symlink_size, + InodeWrapper::Ref(i) => i.get_symlink_size(), } } /// Set size of symlink. pub fn set_symlink_size(&mut self, size: usize) { debug_assert!(size <= u16::MAX as usize); + self.ensure_owned(); match self { InodeWrapper::V5(i) => { i.i_flags |= RafsV5InodeFlags::SYMLINK; @@ -406,22 +493,26 @@ impl InodeWrapper { i.i_flags |= RafsV5InodeFlags::SYMLINK; i.i_symlink_size = size as u16; } + InodeWrapper::Ref(_i) => panic!("unexpected"), } } - /// Set child inode index. + /// Get child inode index. pub fn child_index(&self) -> u32 { match self { InodeWrapper::V5(i) => i.i_child_index, InodeWrapper::V6(i) => i.i_child_index, + InodeWrapper::Ref(_i) => unimplemented!(), } } - /// Get child inode index. + /// Set child inode index. pub fn set_child_index(&mut self, index: u32) { + self.ensure_owned(); match self { InodeWrapper::V5(i) => i.i_child_index = index, InodeWrapper::V6(i) => i.i_child_index = index, + InodeWrapper::Ref(_i) => panic!("unexpected"), } } @@ -430,14 +521,17 @@ impl InodeWrapper { match self { InodeWrapper::V5(i) => i.i_child_count, InodeWrapper::V6(i) => i.i_child_count, + InodeWrapper::Ref(i) => i.get_child_count(), } } /// Set child/chunk count. pub fn set_child_count(&mut self, count: u32) { + self.ensure_owned(); match self { InodeWrapper::V5(i) => i.i_child_count = count, InodeWrapper::V6(i) => i.i_child_count = count, + InodeWrapper::Ref(_i) => panic!("unexpected"), } } @@ -446,6 +540,7 @@ impl InodeWrapper { match self { InodeWrapper::V5(_) => ChunkWrapper::V5(RafsV5ChunkInfo::new()), InodeWrapper::V6(_) => ChunkWrapper::V6(RafsV5ChunkInfo::new()), + InodeWrapper::Ref(_i) => unimplemented!(), } } @@ -461,6 +556,20 @@ impl InodeWrapper { }; inode_size + xattrs.aligned_size_v6() } + InodeWrapper::Ref(_i) => unimplemented!(), + } + } + + fn ensure_owned(&mut self) { + if let Self::Ref(i) = self { + let i = i.clone(); + if self.is_v6() { + *self = Self::V6(to_rafsv5_inode(i.deref().deref())); + } else if self.is_v5() { + *self = Self::V5(to_rafsv5_inode(i.deref().deref())); + } else { + panic!("inode is neither v5 nor v6"); + } } } } diff --git a/rafs/src/metadata/mod.rs b/rafs/src/metadata/mod.rs index af1775e8dd0..94641e7305c 100644 --- a/rafs/src/metadata/mod.rs +++ b/rafs/src/metadata/mod.rs @@ -12,7 +12,6 @@ use std::ffi::{OsStr, OsString}; use std::fmt::{Debug, Display, Formatter, Result as FmtResult}; use std::fs::OpenOptions; use std::io::{Error, ErrorKind, Result}; -use std::ops::Deref; use std::os::unix::ffi::OsStrExt; use std::path::{Component, Path, PathBuf}; use std::str::FromStr; @@ -67,6 +66,7 @@ pub const DOTDOT: &str = ".."; /// Type for RAFS filesystem inode number. pub type Inode = u64; +pub type ArcRafsInodeExt = Arc; /// Trait to access filesystem inodes managed by a RAFS filesystem. pub trait RafsSuperInodes { @@ -1001,32 +1001,32 @@ impl RafsSuper { &self, ino: Inode, parent: Option

, - cb: &mut dyn FnMut(&dyn RafsInodeExt, &Path) -> anyhow::Result<()>, + cb: &mut dyn FnMut(ArcRafsInodeExt, &Path) -> anyhow::Result<()>, ) -> anyhow::Result<()> { let inode = self.get_extended_inode(ino, false)?; if !inode.is_dir() { bail!("inode {} is not a directory", ino); } - self.do_walk_directory(inode.deref(), parent, cb) + self.do_walk_directory(inode, parent, cb) } #[allow(clippy::only_used_in_recursion)] fn do_walk_directory>( &self, - inode: &dyn RafsInodeExt, + inode: Arc, parent: Option

, - cb: &mut dyn FnMut(&dyn RafsInodeExt, &Path) -> anyhow::Result<()>, + cb: &mut dyn FnMut(ArcRafsInodeExt, &Path) -> anyhow::Result<()>, ) -> anyhow::Result<()> { let path = if let Some(parent) = parent { parent.as_ref().join(inode.name()) } else { PathBuf::from("/") }; - cb(inode, &path)?; + cb(inode.clone(), &path)?; if inode.is_dir() { for idx in 0..inode.get_child_count() { let child = inode.get_child_by_index(idx)?; - self.do_walk_directory(child.deref(), Some(&path), cb)?; + self.do_walk_directory(child, Some(&path), cb)?; } } Ok(()) diff --git a/src/bin/nydus-image/core/tree.rs b/src/bin/nydus-image/core/tree.rs index 9d82c881ca5..537997dd351 100644 --- a/src/bin/nydus-image/core/tree.rs +++ b/src/bin/nydus-image/core/tree.rs @@ -17,8 +17,8 @@ use std::ffi::OsStr; use std::ffi::OsString; -use std::ops::Deref; use std::path::{Path, PathBuf}; +use std::sync::Arc; use anyhow::Result; use nydus_rafs::metadata::chunk::ChunkWrapper; @@ -51,8 +51,7 @@ impl Tree { pub fn from_bootstrap(rs: &RafsSuper, chunk_dict: &mut T) -> Result { let tree_builder = MetadataTreeBuilder::new(rs); let root_inode = rs.get_extended_inode(rs.superblock.root_ino(), true)?; - let root_node = - MetadataTreeBuilder::parse_node(rs, root_inode.deref(), PathBuf::from("/"))?; + let root_node = MetadataTreeBuilder::parse_node(rs, root_inode, PathBuf::from("/"))?; let mut tree = Tree::new(root_node); tree.children = timing_tracer!( @@ -276,7 +275,7 @@ impl<'a> MetadataTreeBuilder<'a> { let child = inode.get_child_by_index(idx)?; let child_ino = child.ino(); let child_path = parent_path.join(child.name()); - let child = Self::parse_node(self.rs, child.deref(), child_path)?; + let child = Self::parse_node(self.rs, child.clone(), child_path)?; if child.is_reg() { for chunk in &child.chunks { @@ -299,7 +298,7 @@ impl<'a> MetadataTreeBuilder<'a> { } /// Convert a `RafsInode` object to an in-memory `Node` object. - pub fn parse_node(rs: &RafsSuper, inode: &dyn RafsInodeExt, path: PathBuf) -> Result { + pub fn parse_node(rs: &RafsSuper, inode: Arc, path: PathBuf) -> Result { let chunks = if inode.is_reg() { let chunk_count = inode.get_chunk_count(); let mut chunks = Vec::with_capacity(chunk_count as usize); @@ -332,7 +331,7 @@ impl<'a> MetadataTreeBuilder<'a> { // to avoid breaking hardlink detecting logic. let src_dev = u64::MAX; - let inode_wrapper = InodeWrapper::from_inode_info(inode); + let inode_wrapper = InodeWrapper::from_inode_info(inode.clone()); let source = PathBuf::from("/"); let target = Node::generate_target(&path, &source); let target_vec = Node::generate_target_vec(&target); diff --git a/src/bin/nydus-image/inspect.rs b/src/bin/nydus-image/inspect.rs index 7e348da81ef..2f491a32ec6 100644 --- a/src/bin/nydus-image/inspect.rs +++ b/src/bin/nydus-image/inspect.rs @@ -431,7 +431,7 @@ RAFS Blob Size: {rafs_size} self.rafs_meta.walk_directory::( self.rafs_meta.superblock.root_ino(), None, - &mut |inode: &dyn RafsInodeExt, _path: &Path| -> anyhow::Result<()> { + &mut |inode: Arc, _path: &Path| -> anyhow::Result<()> { // only regular file has data chunks if !inode.is_reg() { return Ok(()); diff --git a/src/bin/nydus-image/merge.rs b/src/bin/nydus-image/merge.rs index 01ecfd174b4..802b2cad8a5 100644 --- a/src/bin/nydus-image/merge.rs +++ b/src/bin/nydus-image/merge.rs @@ -4,7 +4,6 @@ use std::collections::HashSet; use std::convert::TryFrom; -use std::ops::Deref; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -209,9 +208,9 @@ impl Merger { rs.walk_directory::( rs.superblock.root_ino(), None, - &mut |inode: &dyn RafsInodeExt, path: &Path| -> Result<()> { + &mut |inode: Arc, path: &Path| -> Result<()> { let mut node = - MetadataTreeBuilder::parse_node(&rs, inode.deref(), path.to_path_buf()) + MetadataTreeBuilder::parse_node(&rs, inode, path.to_path_buf()) .context(format!( "parse node from bootstrap {:?}", bootstrap_path diff --git a/src/bin/nydus-image/unpack/mod.rs b/src/bin/nydus-image/unpack/mod.rs index 184b771ecbe..4f1136331aa 100644 --- a/src/bin/nydus-image/unpack/mod.rs +++ b/src/bin/nydus-image/unpack/mod.rs @@ -84,7 +84,7 @@ impl Unpacker for OCIUnpacker { .create(&rafs, &self.blob_backend, &self.output)?; for (node, path) in RafsIterator::new(&rafs) { - builder.append(&*node, &path)?; + builder.append(node, &path)?; } Ok(()) @@ -92,7 +92,7 @@ impl Unpacker for OCIUnpacker { } trait TarBuilder { - fn append(&mut self, node: &dyn RafsInodeExt, path: &Path) -> Result<()>; + fn append(&mut self, node: Arc, path: &Path) -> Result<()>; } struct TarSection { @@ -101,8 +101,8 @@ struct TarSection { } trait SectionBuilder { - fn can_handle(&mut self, inode: &dyn RafsInodeExt, path: &Path) -> bool; - fn build(&self, inode: &dyn RafsInodeExt, path: &Path) -> Result>; + fn can_handle(&mut self, inode: Arc, path: &Path) -> bool; + fn build(&self, inode: Arc, path: &Path) -> Result>; } struct OCITarBuilderFactory {} @@ -218,14 +218,14 @@ impl OCITarBuilder { } impl TarBuilder for OCITarBuilder { - fn append(&mut self, inode: &dyn RafsInodeExt, path: &Path) -> Result<()> { + fn append(&mut self, inode: Arc, path: &Path) -> Result<()> { for builder in &mut self.builders { // Useless one, just go !!!!! - if !builder.can_handle(inode, path) { + if !builder.can_handle(inode.clone(), path) { continue; } - for sect in builder.build(inode, path)? { + for sect in builder.build(inode.clone(), path)? { self.writer.append(§.header, sect.data)?; } diff --git a/src/bin/nydus-image/unpack/pax.rs b/src/bin/nydus-image/unpack/pax.rs index 932d2e27f4b..b609ba15ae9 100644 --- a/src/bin/nydus-image/unpack/pax.rs +++ b/src/bin/nydus-image/unpack/pax.rs @@ -1,4 +1,5 @@ use nix::unistd::{Gid, Group, Uid, User}; +use std::ops::Deref; use std::{ collections::HashMap, ffi::OsStr, @@ -35,11 +36,11 @@ impl OCISocketBuilder { } impl SectionBuilder for OCISocketBuilder { - fn can_handle(&mut self, node: &dyn RafsInodeExt, _: &Path) -> bool { + fn can_handle(&mut self, node: Arc, _: &Path) -> bool { InodeWrapper::from_inode_info(node).is_sock() } - fn build(&self, _: &dyn RafsInodeExt, _: &Path) -> Result> { + fn build(&self, _: Arc, _: &Path) -> Result> { Ok(Vec::new()) } } @@ -59,7 +60,7 @@ impl OCILinkBuilder { } impl SectionBuilder for OCILinkBuilder { - fn can_handle(&mut self, node: &dyn RafsInodeExt, path: &Path) -> bool { + fn can_handle(&mut self, node: Arc, path: &Path) -> bool { if !node.is_hardlink() || node.is_dir() { return false; } @@ -72,7 +73,7 @@ impl SectionBuilder for OCILinkBuilder { is_appeared } - fn build(&self, node: &dyn RafsInodeExt, path: &Path) -> Result> { + fn build(&self, node: Arc, path: &Path) -> Result> { let link = self.links.get(&node.ino()).unwrap(); self.pax_link_builder @@ -94,7 +95,7 @@ impl OCIDirBuilder { } } -fn set_header_by_inode(inode: &dyn RafsInodeExt, header: &mut Header) -> Result<()> { +fn set_header_by_inode(inode: Arc, header: &mut Header) -> Result<()> { let inode = InodeWrapper::from_inode_info(inode); header.set_size(inode.size()); header.set_mtime(inode.mtime()); @@ -122,11 +123,11 @@ fn set_header_by_inode(inode: &dyn RafsInodeExt, header: &mut Header) -> Result< } impl SectionBuilder for OCIDirBuilder { - fn can_handle(&mut self, node: &dyn RafsInodeExt, _: &Path) -> bool { + fn can_handle(&mut self, node: Arc, _: &Path) -> bool { node.is_dir() } - fn build(&self, inode: &dyn RafsInodeExt, path: &Path) -> Result> { + fn build(&self, inode: Arc, path: &Path) -> Result> { if self.is_root(path) { return Ok(Vec::new()); } @@ -136,14 +137,14 @@ impl SectionBuilder for OCIDirBuilder { header.set_device_major(0).unwrap(); header.set_device_minor(0).unwrap(); - set_header_by_inode(inode, &mut header)?; + set_header_by_inode(inode.clone(), &mut header)?; header.set_size(0); let mut extensions = Vec::with_capacity(2); if let Some(extension) = PAXUtil::set_path(&mut header, path)? { extensions.push(extension); } - if let Some(extension) = PAXUtil::get_xattr_as_extensions(inode) { + if let Some(extension) = PAXUtil::get_xattr_as_extensions(inode.deref()) { extensions.extend(extension); } @@ -203,22 +204,22 @@ impl OCIRegBuilder { } impl SectionBuilder for OCIRegBuilder { - fn can_handle(&mut self, node: &dyn RafsInodeExt, _: &Path) -> bool { + fn can_handle(&mut self, node: Arc, _: &Path) -> bool { node.is_reg() } - fn build(&self, inode: &dyn RafsInodeExt, path: &Path) -> Result> { + fn build(&self, inode: Arc, path: &Path) -> Result> { let mut header = Header::new_ustar(); header.set_entry_type(EntryType::file()); header.set_device_major(0).unwrap(); header.set_device_minor(0).unwrap(); - set_header_by_inode(inode, &mut header)?; + set_header_by_inode(inode.clone(), &mut header)?; let mut extensions = Vec::with_capacity(2); if let Some(extension) = PAXUtil::set_path(&mut header, path)? { extensions.push(extension); } - if let Some(extension) = PAXUtil::get_xattr_as_extensions(inode) { + if let Some(extension) = PAXUtil::get_xattr_as_extensions(inode.deref()) { extensions.extend(extension); } @@ -231,7 +232,7 @@ impl SectionBuilder for OCIRegBuilder { let main_header = TarSection { header, - data: Box::new(self.build_data(inode)), + data: Box::new(self.build_data(inode.deref())), }; sections.push(main_header); @@ -250,11 +251,11 @@ impl OCISymlinkBuilder { } impl SectionBuilder for OCISymlinkBuilder { - fn can_handle(&mut self, node: &dyn RafsInodeExt, _: &Path) -> bool { + fn can_handle(&mut self, node: Arc, _: &Path) -> bool { node.is_symlink() } - fn build(&self, node: &dyn RafsInodeExt, path: &Path) -> Result> { + fn build(&self, node: Arc, path: &Path) -> Result> { let link = node.get_symlink().unwrap(); self.pax_link_builder @@ -275,11 +276,11 @@ impl OCIFifoBuilder { } impl SectionBuilder for OCIFifoBuilder { - fn can_handle(&mut self, node: &dyn RafsInodeExt, _: &Path) -> bool { + fn can_handle(&mut self, node: Arc, _: &Path) -> bool { InodeWrapper::from_inode_info(node).is_fifo() } - fn build(&self, inode: &dyn RafsInodeExt, path: &Path) -> Result> { + fn build(&self, inode: Arc, path: &Path) -> Result> { self.pax_special_builder .build(EntryType::fifo(), inode, path) } @@ -298,11 +299,11 @@ impl OCICharBuilder { } impl SectionBuilder for OCICharBuilder { - fn can_handle(&mut self, node: &dyn RafsInodeExt, _: &Path) -> bool { + fn can_handle(&mut self, node: Arc, _: &Path) -> bool { InodeWrapper::from_inode_info(node).is_chrdev() } - fn build(&self, inode: &dyn RafsInodeExt, path: &Path) -> Result> { + fn build(&self, inode: Arc, path: &Path) -> Result> { self.pax_special_builder .build(EntryType::character_special(), inode, path) } @@ -321,11 +322,11 @@ impl OCIBlockBuilder { } impl SectionBuilder for OCIBlockBuilder { - fn can_handle(&mut self, node: &dyn RafsInodeExt, _: &Path) -> bool { + fn can_handle(&mut self, node: Arc, _: &Path) -> bool { InodeWrapper::from_inode_info(node).is_blkdev() } - fn build(&self, inode: &dyn RafsInodeExt, path: &Path) -> Result> { + fn build(&self, inode: Arc, path: &Path) -> Result> { self.pax_special_builder .build(EntryType::block_special(), inode, path) } @@ -343,12 +344,12 @@ impl PAXSpecialSectionBuilder { fn build( &self, entry_type: EntryType, - inode: &dyn RafsInodeExt, + inode: Arc, path: &Path, ) -> Result> { let mut header = Header::new_ustar(); header.set_entry_type(entry_type); - set_header_by_inode(inode, &mut header)?; + set_header_by_inode(inode.clone(), &mut header)?; let dev_id = self.cal_dev(inode.rdev() as u64); header.set_device_major(dev_id.0)?; @@ -358,7 +359,7 @@ impl PAXSpecialSectionBuilder { if let Some(extension) = PAXUtil::set_path(&mut header, path)? { extensions.push(extension); } - if let Some(extension) = PAXUtil::get_xattr_as_extensions(inode) { + if let Some(extension) = PAXUtil::get_xattr_as_extensions(inode.deref()) { extensions.extend(extension); } @@ -492,12 +493,12 @@ impl PAXLinkBuilder { fn build( &self, entry_type: EntryType, - inode: &dyn RafsInodeExt, + inode: Arc, path: &Path, link: &Path, ) -> Result> { let mut header = Header::new_ustar(); - set_header_by_inode(inode, &mut header)?; + set_header_by_inode(inode.clone(), &mut header)?; header.set_entry_type(entry_type); header.set_size(0); header.set_device_major(0).unwrap(); @@ -510,7 +511,7 @@ impl PAXLinkBuilder { if let Some(extension) = PAXUtil::set_link(&mut header, link)? { extensions.push(extension); } - if let Some(extension) = PAXUtil::get_xattr_as_extensions(inode) { + if let Some(extension) = PAXUtil::get_xattr_as_extensions(inode.deref()) { extensions.extend(extension); } From 3fc59da93c51e9a210a2f14baed62a832d0fd37d Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Fri, 3 Mar 2023 14:33:25 +0800 Subject: [PATCH 03/10] rafs: move builder from nydus-image into rafs Move builder from nydus-image into rafs, so it can be reused. Signed-off-by: Jiang Liu --- Cargo.lock | 18 ++-- Cargo.toml | 9 +- rafs/Cargo.toml | 9 ++ .../src/builder/compact.rs | 38 ++++--- .../src/builder}/core/blob.rs | 2 +- .../src/builder}/core/bootstrap.rs | 23 ++-- .../src/builder}/core/chunk_dict.rs | 49 +++++---- .../src/builder}/core/context.rs | 20 ++-- .../src/builder}/core/feature.rs | 11 +- .../src/builder}/core/layout.rs | 4 +- .../src/builder}/core/mod.rs | 1 - .../src/builder}/core/node.rs | 102 ++++++++++++------ .../src/builder}/core/prefetch.rs | 8 +- .../src/builder}/core/tree.rs | 48 +++++---- .../src}/builder/directory.rs | 15 +-- .../nydus-image => rafs/src}/builder/mod.rs | 35 +++--- .../src}/builder/stargz.rs | 41 ++++--- .../src}/builder/tarball.rs | 22 ++-- rafs/src/lib.rs | 1 + src/bin/nydus-image/main.rs | 23 ++-- src/bin/nydus-image/merge.rs | 14 +-- src/bin/nydus-image/stat.rs | 10 +- src/bin/nydus-image/validator.rs | 3 +- utils/src/lib.rs | 1 + {src/bin/nydus-image => utils/src}/trace.rs | 1 + 25 files changed, 285 insertions(+), 223 deletions(-) rename src/bin/nydus-image/core/blob_compact.rs => rafs/src/builder/compact.rs (95%) rename {src/bin/nydus-image => rafs/src/builder}/core/blob.rs (99%) rename {src/bin/nydus-image => rafs/src/builder}/core/bootstrap.rs (99%) rename {src/bin/nydus-image => rafs/src/builder}/core/chunk_dict.rs (83%) rename {src/bin/nydus-image => rafs/src/builder}/core/context.rs (98%) rename {src/bin/nydus-image => rafs/src/builder}/core/feature.rs (76%) rename {src/bin/nydus-image => rafs/src/builder}/core/layout.rs (95%) rename {src/bin/nydus-image => rafs/src/builder}/core/mod.rs (91%) rename {src/bin/nydus-image => rafs/src/builder}/core/node.rs (96%) rename {src/bin/nydus-image => rafs/src/builder}/core/prefetch.rs (97%) rename {src/bin/nydus-image => rafs/src/builder}/core/tree.rs (91%) rename {src/bin/nydus-image => rafs/src}/builder/directory.rs (94%) rename {src/bin/nydus-image => rafs/src}/builder/mod.rs (88%) rename {src/bin/nydus-image => rafs/src}/builder/stargz.rs (96%) rename {src/bin/nydus-image => rafs/src}/builder/tarball.rs (97%) rename {src/bin/nydus-image => utils/src}/trace.rs (99%) diff --git a/Cargo.lock b/Cargo.lock index 80709f97e57..bbd8896eedc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -99,9 +99,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" @@ -1189,11 +1189,13 @@ dependencies = [ "anyhow", "arc-swap", "assert_matches", - "base64 0.13.0", + "base64 0.13.1", "bitflags", "blake3", "fuse-backend-rs", "futures", + "hex", + "indexmap", "lazy_static", "libc", "log", @@ -1205,9 +1207,12 @@ dependencies = [ "nydus-utils", "serde", "serde_json", + "sha2", "spmc", + "tar", "vm-memory", "vmm-sys-util", + "xattr", ] [[package]] @@ -1215,13 +1220,11 @@ name = "nydus-rs" version = "0.0.0-git" dependencies = [ "anyhow", - "base64 0.13.0", "clap", "fuse-backend-rs", "hex", "hyper", "hyperlocal", - "indexmap", "lazy_static", "libc", "log", @@ -1238,7 +1241,6 @@ dependencies = [ "rlimit", "serde", "serde_json", - "sha2", "tar", "tokio", "vhost", @@ -1246,8 +1248,6 @@ dependencies = [ "virtio-bindings", "virtio-queue", "vm-memory", - "vmm-sys-util", - "xattr", ] [[package]] @@ -1283,7 +1283,7 @@ name = "nydus-storage" version = "0.6.2" dependencies = [ "arc-swap", - "base64 0.13.0", + "base64 0.13.1", "bitflags", "fuse-backend-rs", "gpt", diff --git a/Cargo.toml b/Cargo.toml index a1351089858..1c6287921a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,13 +32,11 @@ path = "src/lib.rs" [dependencies] anyhow = "1" -base64 = "0.13.0" clap = { version = "4.0.18", features = ["derive", "cargo"] } fuse-backend-rs = "0.10.1" hex = "0.4.3" hyper = "0.14.11" hyperlocal = "0.8.0" -indexmap = "1" lazy_static = "1" libc = "0.2" log = "0.4.8" @@ -47,11 +45,8 @@ nix = "0.24.0" rlimit = "0.9.0" serde = { version = "1.0.110", features = ["serde_derive", "rc"] } serde_json = "1.0.51" -sha2 = "0.10.2" tar = "0.4.38" tokio = { version = "1.24", features = ["macros"] } -vmm-sys-util = "0.10.0" -xattr = "0.2.3" # Build static linked openssl library openssl = { version = "0.10.45", features = ["vendored"] } @@ -61,7 +56,7 @@ openssl = { version = "0.10.45", features = ["vendored"] } nydus-api = { version = "0.2.2", path = "api", features = ["handler"] } nydus-app = { version = "0.3.2", path = "app" } nydus-error = { version = "0.2.3", path = "error" } -nydus-rafs = { version = "0.2.2", path = "rafs" } +nydus-rafs = { version = "0.2.2", path = "rafs", features = ["builder"] } nydus-service = { version = "0.2.0", path = "service" } nydus-storage = { version = "0.6.2", path = "storage" } nydus-utils = { version = "0.4.1", path = "utils" } @@ -71,6 +66,7 @@ vhost-user-backend = { version = "0.7.0", optional = true } virtio-bindings = { version = "0.1", features = ["virtio-v5_0_0"], optional = true } virtio-queue = { version = "0.6.0", optional = true } vm-memory = { version = "0.9.0", features = ["backend-mmap"], optional = true } +vmm-sys-util = { version = "0.10.0", optional = true } [features] default = [ @@ -88,6 +84,7 @@ virtiofs = [ "virtio-bindings", "virtio-queue", "vm-memory", + "vmm-sys-util", ] backend-http-proxy = ["nydus-storage/backend-http-proxy"] diff --git a/rafs/Cargo.toml b/rafs/Cargo.toml index b8b50bbf110..6fd360e4cfc 100644 --- a/rafs/Cargo.toml +++ b/rafs/Cargo.toml @@ -26,6 +26,13 @@ spmc = "0.3.0" vm-memory = "0.9" fuse-backend-rs = "0.10" +hex = { version = "0.4.3", optional = true } +indexmap = { version = "1", optional = true } +sha2 = { version = "0.10.2", optional = true } +tar = { version = "0.4.38", optional = true } +vmm-sys-util = { version = "0.10.0", optional = true } +xattr = { version = "0.2.3", optional = true } + nydus-api = { version = "0.2", path = "../api" } nydus-error = { version = "0.2", path = "../error" } nydus-storage = { version = "0.6", path = "../storage", features = ["backend-localfs"] } @@ -40,6 +47,8 @@ fusedev = ["fuse-backend-rs/fusedev"] virtio-fs = ["fuse-backend-rs/virtiofs", "vm-memory/backend-mmap"] vhost-user-fs = ["fuse-backend-rs/vhost-user-fs"] +builder = ["base64", "hex", "indexmap", "sha2", "tar", "vmm-sys-util", "xattr"] + [package.metadata.docs.rs] all-features = true targets = ["x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu", "aarch64-apple-darwin"] diff --git a/src/bin/nydus-image/core/blob_compact.rs b/rafs/src/builder/compact.rs similarity index 95% rename from src/bin/nydus-image/core/blob_compact.rs rename to rafs/src/builder/compact.rs index 8ba7cda57a3..614a8b0a74b 100644 --- a/src/bin/nydus-image/core/blob_compact.rs +++ b/rafs/src/builder/compact.rs @@ -5,30 +5,31 @@ use std::collections::hash_map::Entry; use std::collections::HashMap; use std::io::Write; +use std::ops::Deref; use std::path::PathBuf; use std::sync::Arc; -use anyhow::Result; +use anyhow::{bail, ensure, Result}; use serde::{Deserialize, Serialize}; use sha2::Digest; -use nydus_rafs::metadata::chunk::ChunkWrapper; -use nydus_rafs::metadata::{RafsSuper, RafsVersion}; use nydus_storage::backend::BlobBackend; use nydus_storage::utils::alloc_buf; use nydus_utils::digest::RafsDigest; use nydus_utils::{digest, try_round_up_4k}; -use crate::core::blob::Blob; -use crate::core::bootstrap::Bootstrap; -use crate::core::chunk_dict::{ChunkDict, HashChunkDict}; -use crate::core::context::{ +use super::core::blob::Blob; +use super::core::bootstrap::Bootstrap; +use super::core::chunk_dict::{ChunkDict, HashChunkDict}; +use super::core::context::{ ArtifactStorage, ArtifactWriter, BlobContext, BlobManager, BootstrapManager, BuildContext, BuildOutput, ConversionType, }; -use crate::core::feature::Features; -use crate::core::node::{Node, WhiteoutSpec}; -use crate::core::tree::Tree; +use super::core::feature::Features; +use super::core::node::{Node, WhiteoutSpec}; +use super::core::tree::Tree; +use crate::metadata::chunk::ChunkWrapper; +use crate::metadata::{RafsSuper, RafsVersion}; const DEFAULT_COMPACT_BLOB_SIZE: usize = 10 * 1024 * 1024; const DEFAULT_MAX_COMPACT_SIZE: usize = 100 * 1024 * 1024; @@ -304,9 +305,13 @@ impl BlobCompactor { if let Some(c) = chunk_dict.get_chunk(chunk.inner.id(), chunk.inner.uncompressed_size()) { - apply_chunk_change(c, &mut chunk.inner)?; + let mut chunk_inner = chunk.inner.deref().clone(); + apply_chunk_change(c, &mut chunk_inner)?; + chunk.inner = Arc::new(chunk_inner); } else if let Some(c) = all_chunks.get_chunk(&chunk_key) { - apply_chunk_change(c, &mut chunk.inner)?; + let mut chunk_inner = chunk.inner.deref().clone(); + apply_chunk_change(c, &mut chunk_inner)?; + chunk.inner = Arc::new(chunk_inner); } else { all_chunks.add_chunk(&chunk.inner); // add to per blob ChunkSet @@ -357,9 +362,7 @@ impl BlobCompactor { self.nodes[*node_idx].chunks[*chunk_idx].inner.blob_index() == from, "unexpected blob_index of chunk" ); - self.nodes[*node_idx].chunks[*chunk_idx] - .inner - .set_blob_index(to); + self.nodes[*node_idx].chunks[*chunk_idx].set_blob_index(to); } } Ok(()) @@ -368,7 +371,10 @@ impl BlobCompactor { fn apply_chunk_change(&mut self, c: &(ChunkWrapper, ChunkWrapper)) -> Result<()> { if let Some(idx_list) = self.c2nodes.get(&ChunkKey::from(&c.0)) { for (node_idx, chunk_idx) in idx_list.iter() { - apply_chunk_change(&c.1, &mut self.nodes[*node_idx].chunks[*chunk_idx].inner)?; + let chunk = &mut self.nodes[*node_idx].chunks[*chunk_idx]; + let mut chunk_inner = chunk.inner.deref().clone(); + apply_chunk_change(&c.1, &mut chunk_inner)?; + chunk.inner = Arc::new(chunk_inner); } } Ok(()) diff --git a/src/bin/nydus-image/core/blob.rs b/rafs/src/builder/core/blob.rs similarity index 99% rename from src/bin/nydus-image/core/blob.rs rename to rafs/src/builder/core/blob.rs index 60229d00ce9..677f79fd840 100644 --- a/src/bin/nydus-image/core/blob.rs +++ b/rafs/src/builder/core/blob.rs @@ -7,7 +7,6 @@ use std::io::Write; use std::slice; use anyhow::{Context, Result}; -use nydus_rafs::metadata::RAFS_MAX_CHUNK_SIZE; use nydus_storage::device::BlobFeatures; use nydus_storage::meta::{toc, BlobMetaChunkArray}; use nydus_utils::compress; @@ -18,6 +17,7 @@ use super::context::{ArtifactWriter, BlobContext, BlobManager, BuildContext, Con use super::feature::Feature; use super::layout::BlobLayout; use super::node::Node; +use crate::metadata::RAFS_MAX_CHUNK_SIZE; pub struct Blob {} diff --git a/src/bin/nydus-image/core/bootstrap.rs b/rafs/src/builder/core/bootstrap.rs similarity index 99% rename from src/bin/nydus-image/core/bootstrap.rs rename to rafs/src/builder/core/bootstrap.rs index 031e9f53869..279d73f7a90 100644 --- a/src/bin/nydus-image/core/bootstrap.rs +++ b/rafs/src/builder/core/bootstrap.rs @@ -8,29 +8,30 @@ use std::ffi::OsString; use std::io::SeekFrom; use std::mem::size_of; -use anyhow::{Context, Error, Result}; -use nydus_rafs::metadata::layout::v5::{ - RafsV5BlobTable, RafsV5ChunkInfo, RafsV5InodeTable, RafsV5SuperBlock, RafsV5XAttrsTable, -}; -use nydus_rafs::metadata::layout::v6::{ - align_offset, calculate_nid, RafsV6BlobTable, RafsV6Device, RafsV6SuperBlock, - RafsV6SuperBlockExt, EROFS_BLOCK_SIZE, EROFS_DEVTABLE_OFFSET, EROFS_INODE_SLOT_SIZE, -}; -use nydus_rafs::metadata::layout::{RafsBlobTable, RAFS_V5_ROOT_INODE}; -use nydus_rafs::metadata::{RafsStore, RafsSuper, RafsSuperConfig}; +use anyhow::{bail, Context, Error, Result}; use nydus_storage::device::BlobFeatures; use nydus_utils::digest::{self, DigestHasher, RafsDigest}; +use nydus_utils::{root_tracer, timing_tracer}; use super::context::{ ArtifactStorage, BlobManager, BootstrapContext, BootstrapManager, BuildContext, ConversionType, }; use super::node::{Node, WhiteoutType, OVERLAYFS_WHITEOUT_OPAQUE}; use super::tree::Tree; +use crate::metadata::layout::v5::{ + RafsV5BlobTable, RafsV5ChunkInfo, RafsV5InodeTable, RafsV5SuperBlock, RafsV5XAttrsTable, +}; +use crate::metadata::layout::v6::{ + align_offset, calculate_nid, RafsV6BlobTable, RafsV6Device, RafsV6SuperBlock, + RafsV6SuperBlockExt, EROFS_BLOCK_SIZE, EROFS_DEVTABLE_OFFSET, EROFS_INODE_SLOT_SIZE, +}; +use crate::metadata::layout::{RafsBlobTable, RAFS_V5_ROOT_INODE}; +use crate::metadata::{RafsStore, RafsSuper, RafsSuperConfig}; pub(crate) const STARGZ_DEFAULT_BLOCK_SIZE: u32 = 4 << 20; const WRITE_PADDING_DATA: [u8; 4096] = [0u8; 4096]; -pub(crate) struct Bootstrap {} +pub struct Bootstrap {} impl Bootstrap { /// Create a new instance of `Bootstrap`. diff --git a/src/bin/nydus-image/core/chunk_dict.rs b/rafs/src/builder/core/chunk_dict.rs similarity index 83% rename from src/bin/nydus-image/core/chunk_dict.rs rename to rafs/src/builder/core/chunk_dict.rs index 223b004f5a6..e38be053f5d 100644 --- a/src/bin/nydus-image/core/chunk_dict.rs +++ b/rafs/src/builder/core/chunk_dict.rs @@ -8,33 +8,37 @@ use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::{Arc, Mutex}; -use anyhow::{Context, Result}; +use anyhow::{bail, Context, Result}; use nydus_api::ConfigV2; -use nydus_rafs::metadata::chunk::ChunkWrapper; -use nydus_rafs::metadata::layout::v5::RafsV5ChunkInfo; -use nydus_rafs::metadata::{RafsSuper, RafsSuperConfig}; use nydus_storage::device::BlobInfo; use nydus_utils::digest::{self, RafsDigest}; -use crate::core::tree::Tree; +use super::tree::Tree; +use crate::metadata::chunk::ChunkWrapper; +use crate::metadata::layout::v5::RafsV5ChunkInfo; +use crate::metadata::{RafsSuper, RafsSuperConfig}; #[derive(Debug, PartialEq, Eq, Hash, Ord, PartialOrd)] pub struct DigestWithBlobIndex(pub RafsDigest, pub u32); pub trait ChunkDict: Sync + Send + 'static { - fn add_chunk(&mut self, chunk: ChunkWrapper, digester: digest::Algorithm); - fn get_chunk(&self, digest: &RafsDigest, uncompressed_size: u32) -> Option<&ChunkWrapper>; + fn add_chunk(&mut self, chunk: Arc, digester: digest::Algorithm); + fn get_chunk(&self, digest: &RafsDigest, uncompressed_size: u32) -> Option<&Arc>; fn get_blobs(&self) -> Vec>; - fn get_blob_by_inner_idx(&self, idx: u32) -> Option<&BlobInfo>; + fn get_blob_by_inner_idx(&self, idx: u32) -> Option<&Arc>; fn set_real_blob_idx(&self, inner_idx: u32, out_idx: u32); fn get_real_blob_idx(&self, inner_idx: u32) -> Option; fn digester(&self) -> digest::Algorithm; } impl ChunkDict for () { - fn add_chunk(&mut self, _chunk: ChunkWrapper, _digester: digest::Algorithm) {} + fn add_chunk(&mut self, _chunk: Arc, _digester: digest::Algorithm) {} - fn get_chunk(&self, _digest: &RafsDigest, _uncompressed_size: u32) -> Option<&ChunkWrapper> { + fn get_chunk( + &self, + _digest: &RafsDigest, + _uncompressed_size: u32, + ) -> Option<&Arc> { None } @@ -42,7 +46,7 @@ impl ChunkDict for () { Vec::new() } - fn get_blob_by_inner_idx(&self, _idx: u32) -> Option<&BlobInfo> { + fn get_blob_by_inner_idx(&self, _idx: u32) -> Option<&Arc> { None } @@ -60,14 +64,14 @@ impl ChunkDict for () { } pub struct HashChunkDict { - pub m: HashMap, + m: HashMap, AtomicU32)>, blobs: Vec>, blob_idx_m: Mutex>, digester: digest::Algorithm, } impl ChunkDict for HashChunkDict { - fn add_chunk(&mut self, chunk: ChunkWrapper, digester: digest::Algorithm) { + fn add_chunk(&mut self, chunk: Arc, digester: digest::Algorithm) { if self.digester == digester { if let Some(e) = self.m.get(chunk.id()) { e.1.fetch_add(1, Ordering::AcqRel); @@ -78,7 +82,7 @@ impl ChunkDict for HashChunkDict { } } - fn get_chunk(&self, digest: &RafsDigest, uncompressed_size: u32) -> Option<&ChunkWrapper> { + fn get_chunk(&self, digest: &RafsDigest, uncompressed_size: u32) -> Option<&Arc> { if let Some((chunk, _)) = self.m.get(digest) { if chunk.uncompressed_size() == 0 || chunk.uncompressed_size() == uncompressed_size { return Some(chunk); @@ -91,8 +95,8 @@ impl ChunkDict for HashChunkDict { self.blobs.clone() } - fn get_blob_by_inner_idx(&self, idx: u32) -> Option<&BlobInfo> { - self.blobs.get(idx as usize).map(|b| b.as_ref()) + fn get_blob_by_inner_idx(&self, idx: u32) -> Option<&Arc> { + self.blobs.get(idx as usize) } fn set_real_blob_idx(&self, inner_idx: u32, out_idx: u32) { @@ -118,6 +122,10 @@ impl HashChunkDict { } } + pub fn hashmap(&self) -> &HashMap, AtomicU32)> { + &self.m + } + fn from_bootstrap_file( path: &Path, config: Arc, @@ -163,7 +171,8 @@ impl HashChunkDict { for idx in 0..(size / unit_size) { let chunk = rs.superblock.get_chunk_info(idx)?; - self.add_chunk(ChunkWrapper::from_chunk_info(chunk), self.digester); + let chunk_info = Arc::new(ChunkWrapper::from_chunk_info(chunk)); + self.add_chunk(chunk_info, self.digester); } Ok(()) @@ -197,7 +206,7 @@ pub fn parse_chunk_dict_arg(arg: &str) -> Result { } /// Load a chunk dictionary from external source. -pub(crate) fn import_chunk_dict( +pub fn import_chunk_dict( arg: &str, config: Arc, rafs_config: &RafsSuperConfig, @@ -210,7 +219,7 @@ pub(crate) fn import_chunk_dict( #[cfg(test)] mod tests { use super::*; - use nydus_rafs::metadata::RafsVersion; + use crate::metadata::RafsVersion; use nydus_utils::{compress, digest}; use std::path::PathBuf; @@ -218,7 +227,7 @@ mod tests { fn test_null_dict() { let mut dict = Box::new(()) as Box; - let chunk = ChunkWrapper::new(RafsVersion::V5); + let chunk = Arc::new(ChunkWrapper::new(RafsVersion::V5)); dict.add_chunk(chunk.clone(), digest::Algorithm::Sha256); assert!(dict.get_chunk(chunk.id(), 0).is_none()); assert_eq!(dict.get_blobs().len(), 0); diff --git a/src/bin/nydus-image/core/context.rs b/rafs/src/builder/core/context.rs similarity index 98% rename from src/bin/nydus-image/core/context.rs rename to rafs/src/builder/core/context.rs index 685cb28bd6e..5c1f73f5526 100644 --- a/src/bin/nydus-image/core/context.rs +++ b/rafs/src/builder/core/context.rs @@ -16,19 +16,12 @@ use std::str::FromStr; use std::sync::{Arc, Mutex}; use std::{fmt, fs}; -use anyhow::{Context, Error, Result}; +use anyhow::{anyhow, Context, Error, Result}; use sha2::{Digest, Sha256}; use tar::{EntryType, Header}; use vmm_sys_util::tempfile::TempFile; use nydus_api::ConfigV2; -use nydus_rafs::metadata::chunk::ChunkWrapper; -use nydus_rafs::metadata::layout::v5::RafsV5BlobTable; -use nydus_rafs::metadata::layout::v6::{RafsV6BlobTable, EROFS_BLOCK_SIZE, EROFS_INODE_SLOT_SIZE}; -use nydus_rafs::metadata::layout::RafsBlobTable; -use nydus_rafs::metadata::{Inode, RAFS_DEFAULT_CHUNK_SIZE}; -use nydus_rafs::metadata::{RafsSuperFlags, RafsVersion}; -use nydus_rafs::RafsIoWrite; use nydus_storage::device::{BlobFeatures, BlobInfo}; use nydus_storage::factory::BlobFactory; use nydus_storage::meta::toc::{TocEntryList, TocLocation}; @@ -43,6 +36,13 @@ use super::chunk_dict::{ChunkDict, HashChunkDict}; use super::feature::{Feature, Features}; use super::node::{ChunkSource, Node, WhiteoutSpec}; use super::prefetch::{Prefetch, PrefetchPolicy}; +use crate::metadata::chunk::ChunkWrapper; +use crate::metadata::layout::v5::RafsV5BlobTable; +use crate::metadata::layout::v6::{RafsV6BlobTable, EROFS_BLOCK_SIZE, EROFS_INODE_SLOT_SIZE}; +use crate::metadata::layout::RafsBlobTable; +use crate::metadata::{Inode, RAFS_DEFAULT_CHUNK_SIZE}; +use crate::metadata::{RafsSuperFlags, RafsVersion}; +use crate::RafsIoWrite; // TODO: select BufWriter capacity by performance testing. pub const BUF_WRITER_CAPACITY: usize = 2 << 17; @@ -794,6 +794,10 @@ impl BlobManager { self.blobs.len() } + pub fn is_empty(&self) -> bool { + self.blobs.is_empty() + } + /// Get all blob contexts (include the blob context that does not have a blob). pub fn get_blobs(&self) -> Vec<&BlobContext> { self.blobs.iter().collect() diff --git a/src/bin/nydus-image/core/feature.rs b/rafs/src/builder/core/feature.rs similarity index 76% rename from src/bin/nydus-image/core/feature.rs rename to rafs/src/builder/core/feature.rs index 10de59c42ac..a193ccf7e0e 100644 --- a/src/bin/nydus-image/core/feature.rs +++ b/rafs/src/builder/core/feature.rs @@ -2,25 +2,30 @@ // // SPDX-License-Identifier: Apache-2.0 +use anyhow::{bail, Result}; use std::collections::HashSet; use std::convert::TryFrom; const ERR_UNSUPPORTED_FEATURE: &str = "unsupported feature"; +/// Feature bits for RAFS filesystem builder. #[derive(Clone, Hash, PartialEq, Eq)] pub enum Feature { - // Enable to append TOC footer to rafs blob. + /// Enable to append TOC footer to rafs blob. BlobToc, } +/// Feature set for RAFS filesystem builder. pub struct Features(HashSet); impl Features { + /// Create a new instance of [Features]. pub fn new() -> Self { Self(HashSet::new()) } - pub fn from(features: &str) -> anyhow::Result { + /// Create a new instance of [Features] from a string. + pub fn from(features: &str) -> Result { let mut list = Features::new(); let features = features.trim(); if features.is_empty() { @@ -42,7 +47,7 @@ impl Features { impl TryFrom<&str> for Feature { type Error = anyhow::Error; - fn try_from(f: &str) -> std::result::Result { + fn try_from(f: &str) -> Result { match f { "blob-toc" => Ok(Self::BlobToc), _ => bail!( diff --git a/src/bin/nydus-image/core/layout.rs b/rafs/src/builder/core/layout.rs similarity index 95% rename from src/bin/nydus-image/core/layout.rs rename to rafs/src/builder/core/layout.rs index c83d48de9ca..b99a1eb2b3f 100644 --- a/src/bin/nydus-image/core/layout.rs +++ b/rafs/src/builder/core/layout.rs @@ -4,8 +4,8 @@ use anyhow::Result; -use crate::core::node::{Node, Overlay}; -use crate::core::prefetch::Prefetch; +use super::node::{Node, Overlay}; +use super::prefetch::Prefetch; #[derive(Clone)] pub struct BlobLayout {} diff --git a/src/bin/nydus-image/core/mod.rs b/rafs/src/builder/core/mod.rs similarity index 91% rename from src/bin/nydus-image/core/mod.rs rename to rafs/src/builder/core/mod.rs index 9da44e56323..b7856639ac2 100644 --- a/src/bin/nydus-image/core/mod.rs +++ b/rafs/src/builder/core/mod.rs @@ -3,7 +3,6 @@ // SPDX-License-Identifier: Apache-2.0 pub(crate) mod blob; -pub(crate) mod blob_compact; pub(crate) mod bootstrap; pub(crate) mod chunk_dict; pub(crate) mod context; diff --git a/src/bin/nydus-image/core/node.rs b/rafs/src/builder/core/node.rs similarity index 96% rename from src/bin/nydus-image/core/node.rs rename to rafs/src/builder/core/node.rs index 47028734760..707f52eb196 100644 --- a/src/bin/nydus-image/core/node.rs +++ b/rafs/src/builder/core/node.rs @@ -9,6 +9,7 @@ use std::fmt::{self, Display, Formatter, Result as FmtResult}; use std::fs::{self, File}; use std::io::{Read, SeekFrom, Write}; use std::mem::size_of; +use std::ops::Deref; #[cfg(target_os = "linux")] use std::os::linux::fs::MetadataExt; #[cfg(target_os = "macos")] @@ -16,30 +17,32 @@ use std::os::macos::fs::MetadataExt; use std::os::unix::ffi::OsStrExt; use std::path::{Component, Path, PathBuf}; use std::str::FromStr; +use std::sync::Arc; -use anyhow::{Context, Error, Result}; -use sha2::digest::Digest; - -use nydus_rafs::metadata::chunk::ChunkWrapper; -use nydus_rafs::metadata::inode::{new_v6_inode, InodeWrapper}; -use nydus_rafs::metadata::layout::v5::RafsV5InodeWrapper; -use nydus_rafs::metadata::layout::v6::{ - align_offset, calculate_nid, RafsV6Dirent, RafsV6InodeChunkAddr, RafsV6InodeChunkHeader, - RafsV6OndiskInode, EROFS_BLOCK_SIZE, EROFS_INODE_CHUNK_BASED, EROFS_INODE_FLAT_INLINE, - EROFS_INODE_FLAT_PLAIN, -}; -use nydus_rafs::metadata::layout::RafsXAttrs; -use nydus_rafs::metadata::{Inode, RafsStore, RafsVersion}; -use nydus_rafs::RafsIoWrite; +use anyhow::{anyhow, bail, ensure, Context, Error, Result}; use nydus_storage::device::BlobFeatures; use nydus_storage::meta::{BlobChunkInfoV2Ondisk, BlobMetaChunkInfo}; use nydus_utils::compress; use nydus_utils::digest::{DigestHasher, RafsDigest}; -use nydus_utils::{div_round_up, round_down_4k, round_up, try_round_up_4k, ByteSize}; +use nydus_utils::{ + div_round_up, event_tracer, root_tracer, round_down_4k, round_up, try_round_up_4k, ByteSize, +}; +use sha2::digest::Digest; use super::chunk_dict::{ChunkDict, DigestWithBlobIndex}; use super::context::{ArtifactWriter, BlobContext, BlobManager, BootstrapContext, BuildContext}; use super::tree::Tree; +use crate::metadata::chunk::ChunkWrapper; +use crate::metadata::inode::{new_v6_inode, InodeWrapper}; +use crate::metadata::layout::v5::RafsV5InodeWrapper; +use crate::metadata::layout::v6::{ + align_offset, calculate_nid, RafsV6Dirent, RafsV6InodeChunkAddr, RafsV6InodeChunkHeader, + RafsV6OndiskInode, EROFS_BLOCK_SIZE, EROFS_INODE_CHUNK_BASED, EROFS_INODE_FLAT_INLINE, + EROFS_INODE_FLAT_PLAIN, +}; +use crate::metadata::layout::RafsXAttrs; +use crate::metadata::{Inode, RafsStore, RafsVersion}; +use crate::RafsIoWrite; // Filesystem may have different algorithms to calculate `i_size` for directory entries, // which may break "repeatable build". To support repeatable build, instead of reuse the value @@ -160,18 +163,6 @@ impl Display for Overlay { } } -#[derive(Clone)] -pub struct NodeChunk { - pub source: ChunkSource, - pub inner: ChunkWrapper, -} - -impl Display for NodeChunk { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.inner,) - } -} - /// Where the chunk data is actually stored. #[derive(Clone, Hash, PartialEq, Eq)] pub enum ChunkSource { @@ -193,6 +184,50 @@ impl Display for ChunkSource { } } +#[derive(Clone)] +pub struct NodeChunk { + pub source: ChunkSource, + pub inner: Arc, +} + +impl Display for NodeChunk { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.inner,) + } +} + +impl NodeChunk { + pub fn set_index(&mut self, index: u32) { + let mut chunk = self.inner.deref().clone(); + chunk.set_index(index); + self.inner = Arc::new(chunk); + } + + pub fn copy_from(&mut self, other: &ChunkWrapper) { + let mut chunk = self.inner.deref().clone(); + chunk.copy_from(other); + self.inner = Arc::new(chunk); + } + + pub fn set_file_offset(&mut self, offset: u64) { + let mut chunk = self.inner.deref().clone(); + chunk.set_file_offset(offset); + self.inner = Arc::new(chunk); + } + + pub fn set_blob_index(&mut self, index: u32) { + let mut chunk = self.inner.deref().clone(); + chunk.set_blob_index(index); + self.inner = Arc::new(chunk); + } + + pub fn set_compressed_size(&mut self, size: u32) { + let mut chunk = self.inner.deref().clone(); + chunk.set_compressed_size(size); + self.inner = Arc::new(chunk); + } +} + /// An in-memory representation of RAFS inode for image building and inspection. #[derive(Clone)] pub struct Node { @@ -414,6 +449,7 @@ impl Node { chunk.set_file_offset(file_offset); self.dump_file_chunk(ctx, blob_ctx, blob_writer, chunk_data, &mut chunk)?; + let chunk = Arc::new(chunk); blob_size += chunk.compressed_size() as u64; blob_ctx.add_chunk_meta_info(&chunk, chunk_info)?; blob_mgr @@ -674,7 +710,7 @@ impl Node { }; self.chunks.push(NodeChunk { source, - inner: chunk, + inner: Arc::new(chunk), }); Ok(None) @@ -899,7 +935,7 @@ impl Node { f_bootstrap: &mut dyn RafsIoWrite, orig_meta_addr: u64, meta_addr: u64, - chunk_cache: &mut BTreeMap, + chunk_cache: &mut BTreeMap>, ) -> Result<()> { let xattr_inline_count = self.xattrs.count_v6(); ensure!( @@ -1273,7 +1309,7 @@ impl Node { &mut self, ctx: &mut BuildContext, f_bootstrap: &mut dyn RafsIoWrite, - chunk_cache: &mut BTreeMap, + chunk_cache: &mut BTreeMap>, inode: &mut Box, ) -> Result<()> { let mut is_continuous = true; @@ -1446,9 +1482,9 @@ impl Node { #[cfg(test)] mod tests { use super::*; - use crate::core::context::{ArtifactStorage, BootstrapContext}; - use nydus_rafs::metadata::layout::v6::{EROFS_INODE_CHUNK_BASED, EROFS_INODE_SLOT_SIZE}; - use nydus_rafs::metadata::RAFS_DEFAULT_CHUNK_SIZE; + use crate::builder::core::context::{ArtifactStorage, BootstrapContext}; + use crate::metadata::layout::v6::{EROFS_INODE_CHUNK_BASED, EROFS_INODE_SLOT_SIZE}; + use crate::metadata::RAFS_DEFAULT_CHUNK_SIZE; use std::fs::File; use vmm_sys_util::{tempdir::TempDir, tempfile::TempFile}; diff --git a/src/bin/nydus-image/core/prefetch.rs b/rafs/src/builder/core/prefetch.rs similarity index 97% rename from src/bin/nydus-image/core/prefetch.rs rename to rafs/src/builder/core/prefetch.rs index a397b1e9d67..afa0b71e3af 100644 --- a/src/bin/nydus-image/core/prefetch.rs +++ b/rafs/src/builder/core/prefetch.rs @@ -6,12 +6,12 @@ use std::collections::BTreeMap; use std::path::PathBuf; use std::str::FromStr; -use anyhow::{Context, Error, Result}; +use anyhow::{anyhow, Context, Error, Result}; use indexmap::IndexMap; -use nydus_rafs::metadata::layout::v5::RafsV5PrefetchTable; -use nydus_rafs::metadata::layout::v6::{calculate_nid, RafsV6PrefetchTable}; -use crate::node::Node; +use super::node::Node; +use crate::metadata::layout::v5::RafsV5PrefetchTable; +use crate::metadata::layout::v6::{calculate_nid, RafsV6PrefetchTable}; #[derive(Clone, Copy, Debug, PartialEq)] pub enum PrefetchPolicy { diff --git a/src/bin/nydus-image/core/tree.rs b/rafs/src/builder/core/tree.rs similarity index 91% rename from src/bin/nydus-image/core/tree.rs rename to rafs/src/builder/core/tree.rs index 537997dd351..d7abd161254 100644 --- a/src/bin/nydus-image/core/tree.rs +++ b/rafs/src/builder/core/tree.rs @@ -16,18 +16,18 @@ //! - Traverse the merged tree (OverlayTree) to dump bootstrap and data blobs. use std::ffi::OsStr; -use std::ffi::OsString; use std::path::{Path, PathBuf}; use std::sync::Arc; use anyhow::Result; -use nydus_rafs::metadata::chunk::ChunkWrapper; -use nydus_rafs::metadata::inode::InodeWrapper; -use nydus_rafs::metadata::layout::{bytes_to_os_str, RafsXAttrs}; -use nydus_rafs::metadata::{Inode, RafsInodeExt, RafsSuper}; +use nydus_utils::{event_tracer, root_tracer, timing_tracer}; use super::chunk_dict::ChunkDict; use super::node::{ChunkSource, Node, NodeChunk, Overlay, WhiteoutSpec, WhiteoutType}; +use crate::metadata::chunk::ChunkWrapper; +use crate::metadata::inode::InodeWrapper; +use crate::metadata::layout::{bytes_to_os_str, RafsXAttrs}; +use crate::metadata::{Inode, RafsInodeExt, RafsSuper}; /// An in-memory tree structure to maintain information and topology of filesystem nodes. #[derive(Clone)] @@ -55,7 +55,14 @@ impl Tree { let mut tree = Tree::new(root_node); tree.children = timing_tracer!( - { tree_builder.load_children(rs.superblock.root_ino(), None, chunk_dict, true) }, + { + tree_builder.load_children( + rs.superblock.root_ino(), + Option::::None, + chunk_dict, + true, + ) + }, "load_tree_from_bootstrap" )?; @@ -88,7 +95,7 @@ impl Tree { handle_whiteout: bool, whiteout_spec: WhiteoutSpec, ) -> Result { - // Handle whiteout file + // Handle whiteout objects if handle_whiteout { if let Some(whiteout_type) = target.whiteout_type(whiteout_spec) { let origin_name = target.origin_name(whiteout_type); @@ -121,7 +128,6 @@ impl Tree { // Don't search if path recursive depth out of target path if depth < target_paths_len { - // TODO: Search child by binary search for child in self.children.iter_mut() { // Skip if path component name not match if target_paths[depth] != child.node.name() { @@ -187,7 +193,6 @@ impl Tree { return Ok(true); } - // TODO: Search child by binary search for idx in 0..self.children.len() { let child = &mut self.children[idx]; @@ -247,10 +252,10 @@ impl<'a> MetadataTreeBuilder<'a> { } /// Build node tree by loading bootstrap file - fn load_children( + fn load_children>( &self, ino: Inode, - parent: Option<&PathBuf>, + parent: Option

, chunk_dict: &mut T, validate_digest: bool, ) -> Result> { @@ -260,7 +265,7 @@ impl<'a> MetadataTreeBuilder<'a> { } let parent_path = if let Some(parent) = parent { - parent.join(inode.name()) + parent.as_ref().join(inode.name()) } else { PathBuf::from("/") }; @@ -268,9 +273,6 @@ impl<'a> MetadataTreeBuilder<'a> { let blobs = self.rs.superblock.get_blob_infos(); let child_count = inode.get_child_count(); let mut children = Vec::with_capacity(child_count as usize); - event_tracer!("load_from_parent_bootstrap", +child_count); - // TODO(chge): Implement `Iterator` for both V5 and V6 Inodes. Then we don't need - // `get_child_count` and `get_child_by_index` thus to get rid of concept `index`. for idx in 0..child_count { let child = inode.get_child_by_index(idx)?; let child_ino = child.ino(); @@ -306,7 +308,7 @@ impl<'a> MetadataTreeBuilder<'a> { let cki = inode.get_chunk_info(i)?; chunks.push(NodeChunk { source: ChunkSource::Parent, - inner: ChunkWrapper::from_chunk_info(cki), + inner: Arc::new(ChunkWrapper::from_chunk_info(cki)), }); } chunks @@ -330,31 +332,31 @@ impl<'a> MetadataTreeBuilder<'a> { // Nodes loaded from bootstrap will only be used as `Overlay::Lower`, so make `dev` invalid // to avoid breaking hardlink detecting logic. let src_dev = u64::MAX; - - let inode_wrapper = InodeWrapper::from_inode_info(inode.clone()); + let rdev = inode.rdev() as u64; + let inode = InodeWrapper::from_inode_info(inode.clone()); let source = PathBuf::from("/"); let target = Node::generate_target(&path, &source); let target_vec = Node::generate_target_vec(&target); Ok(Node { index: 0, - src_ino: inode_wrapper.ino(), + src_ino: inode.ino(), src_dev, - rdev: inode.rdev() as u64, + rdev, overlay: Overlay::Lower, explicit_uidgid: rs.meta.explicit_uidgid(), + path, source, target, - path, target_vec, - inode: inode_wrapper, + inode, chunks, symlink, xattrs, layer_idx: 0, ctime: 0, v6_offset: 0, - v6_dirents: Vec::<(u64, OsString, u32)>::new(), + v6_dirents: Vec::new(), v6_datalayout: 0, v6_compact_inode: false, v6_force_extended_inode: false, diff --git a/src/bin/nydus-image/builder/directory.rs b/rafs/src/builder/directory.rs similarity index 94% rename from src/bin/nydus-image/builder/directory.rs rename to rafs/src/builder/directory.rs index 1f1ba72529b..81309b19d1e 100644 --- a/src/bin/nydus-image/builder/directory.rs +++ b/rafs/src/builder/directory.rs @@ -5,15 +5,16 @@ use std::fs; use std::fs::DirEntry; -use anyhow::{Context, Result}; +use anyhow::{anyhow, Context, Result}; +use nydus_utils::{event_tracer, root_tracer, timing_tracer}; -use crate::builder::{build_bootstrap, dump_bootstrap, finalize_blob, Builder}; -use crate::core::blob::Blob; -use crate::core::context::{ +use super::core::blob::Blob; +use super::core::context::{ ArtifactWriter, BlobManager, BootstrapContext, BootstrapManager, BuildContext, BuildOutput, }; -use crate::core::node::{Node, Overlay}; -use crate::core::tree::Tree; +use super::core::node::{Node, Overlay}; +use super::core::tree::Tree; +use super::{build_bootstrap, dump_bootstrap, finalize_blob, Builder}; struct FilesystemTreeBuilder {} @@ -74,7 +75,7 @@ impl FilesystemTreeBuilder { } } -pub(crate) struct DirectoryBuilder {} +pub struct DirectoryBuilder {} impl DirectoryBuilder { pub fn new() -> Self { diff --git a/src/bin/nydus-image/builder/mod.rs b/rafs/src/builder/mod.rs similarity index 88% rename from src/bin/nydus-image/builder/mod.rs rename to rafs/src/builder/mod.rs index 3e0e1628c9a..4735252b56a 100644 --- a/src/bin/nydus-image/builder/mod.rs +++ b/rafs/src/builder/mod.rs @@ -2,30 +2,37 @@ // // SPDX-License-Identifier: Apache-2.0 -use anyhow::{Context, Result}; +//! Builder to create RAFS filesystems from directories and tarballs. +use anyhow::{anyhow, Context, Result}; use nydus_storage::meta::toc; use nydus_utils::digest::{DigestHasher, RafsDigest}; -use nydus_utils::{compress, digest}; +use nydus_utils::{compress, digest, root_tracer, timing_tracer}; use sha2::Digest; -use crate::core::bootstrap::Bootstrap; -use crate::core::context::{ - ArtifactWriter, BlobContext, BlobManager, BootstrapContext, BootstrapManager, BuildContext, - BuildOutput, +pub use self::compact::BlobCompactor; +pub use self::core::bootstrap::Bootstrap; +pub use self::core::chunk_dict::{import_chunk_dict, parse_chunk_dict_arg}; +pub use self::core::chunk_dict::{ChunkDict, HashChunkDict}; +pub use self::core::context::{ + ArtifactStorage, ArtifactWriter, BlobContext, BlobManager, BootstrapContext, BootstrapManager, + BuildContext, BuildOutput, ConversionType, }; -use crate::core::feature::Feature; -use crate::core::tree::Tree; - -pub(crate) use self::directory::DirectoryBuilder; -pub(crate) use self::stargz::StargzBuilder; -pub(crate) use self::tarball::TarballBuilder; - +pub use self::core::feature::{Feature, Features}; +pub use self::core::node::{ChunkSource, Overlay, WhiteoutSpec}; +pub use self::core::prefetch::{Prefetch, PrefetchPolicy}; +pub use self::core::tree::{MetadataTreeBuilder, Tree}; +pub use self::directory::DirectoryBuilder; +pub use self::stargz::StargzBuilder; +pub use self::tarball::TarballBuilder; + +pub mod compact; +mod core; mod directory; mod stargz; mod tarball; /// Trait to generate a RAFS filesystem from the source. -pub(crate) trait Builder { +pub trait Builder { fn build( &mut self, build_ctx: &mut BuildContext, diff --git a/src/bin/nydus-image/builder/stargz.rs b/rafs/src/builder/stargz.rs similarity index 96% rename from src/bin/nydus-image/builder/stargz.rs rename to rafs/src/builder/stargz.rs index 786f1268587..f476134f32e 100644 --- a/src/bin/nydus-image/builder/stargz.rs +++ b/rafs/src/builder/stargz.rs @@ -11,30 +11,31 @@ use std::fs::File; use std::os::unix::ffi::OsStrExt; use std::path::{Path, PathBuf}; use std::rc::Rc; +use std::sync::Arc; use anyhow::{anyhow, bail, Context, Error, Result}; use serde::{Deserialize, Serialize}; -use nydus_rafs::metadata::chunk::ChunkWrapper; -use nydus_rafs::metadata::inode::InodeWrapper; -use nydus_rafs::metadata::layout::v5::{RafsV5ChunkInfo, RafsV5Inode, RafsV5InodeFlags}; -use nydus_rafs::metadata::layout::RafsXAttrs; -use nydus_rafs::metadata::{Inode, RafsVersion}; use nydus_storage::device::BlobChunkFlags; use nydus_storage::{RAFS_MAX_CHUNKS_PER_BLOB, RAFS_MAX_CHUNK_SIZE}; use nydus_utils::compact::makedev; use nydus_utils::compress::compute_compressed_gzip_size; use nydus_utils::digest::{self, Algorithm, DigestHasher, RafsDigest}; -use nydus_utils::{compress, try_round_up_4k, ByteSize}; +use nydus_utils::{compress, root_tracer, timing_tracer, try_round_up_4k, ByteSize}; -use crate::builder::{build_bootstrap, Builder}; -use crate::core::blob::Blob; -use crate::core::context::{ +use super::core::blob::Blob; +use super::core::context::{ ArtifactWriter, BlobContext, BlobManager, BootstrapContext, BootstrapManager, BuildContext, BuildOutput, }; -use crate::core::node::{ChunkSource, Node, NodeChunk, Overlay}; -use crate::core::tree::Tree; +use super::core::node::{ChunkSource, Node, NodeChunk, Overlay}; +use super::core::tree::Tree; +use super::{build_bootstrap, Builder}; +use crate::metadata::chunk::ChunkWrapper; +use crate::metadata::inode::InodeWrapper; +use crate::metadata::layout::v5::{RafsV5ChunkInfo, RafsV5Inode, RafsV5InodeFlags}; +use crate::metadata::layout::RafsXAttrs; +use crate::metadata::{Inode, RafsVersion}; type RcTocEntry = Rc>; @@ -387,8 +388,8 @@ impl StargzTreeBuilder { let chunk = NodeChunk { source: ChunkSource::Build, inner: match ctx.fs_version { - RafsVersion::V5 => v5_chunk_info, - RafsVersion::V6 => v5_chunk_info, + RafsVersion::V5 => Arc::new(v5_chunk_info), + RafsVersion::V6 => Arc::new(v5_chunk_info), }, }; @@ -649,7 +650,7 @@ impl StargzTreeBuilder { } } -pub(crate) struct StargzBuilder { +pub struct StargzBuilder { blob_size: u64, } @@ -722,16 +723,14 @@ impl StargzBuilder { max_gzip_size ); } - blob_chunks[idx] - .inner - .set_compressed_size(max_gzip_size as u32); + blob_chunks[idx].set_compressed_size(max_gzip_size as u32); } let mut chunk_map = HashMap::new(); for chunk in &mut blob_chunks { if !chunk_map.contains_key(chunk.inner.id()) { let chunk_index = blob_ctx.alloc_chunk_index()?; - chunk.inner.set_index(chunk_index); + chunk.set_index(chunk_index); blob_ctx.add_chunk_meta_info(&chunk.inner, None)?; chunk_map.insert(*chunk.inner.id(), chunk_index); } else { @@ -758,9 +757,9 @@ impl StargzBuilder { let chunk_index = *chunk_map.get(chunk.inner.id()).unwrap(); let prepared = &blob_chunks[chunk_index as usize]; let file_offset = chunk.inner.file_offset(); - chunk.inner.copy_from(&prepared.inner); - chunk.inner.set_file_offset(file_offset); - chunk.inner.set_blob_index(blob_index); + chunk.copy_from(&prepared.inner); + chunk.set_file_offset(file_offset); + chunk.set_blob_index(blob_index); // This method is used here to calculate uncompressed_blob_size to // be compatible with the possible 4k alignment requirement of diff --git a/src/bin/nydus-image/builder/tarball.rs b/rafs/src/builder/tarball.rs similarity index 97% rename from src/bin/nydus-image/builder/tarball.rs rename to rafs/src/builder/tarball.rs index 17153d9d502..3d2cbfa0748 100644 --- a/src/bin/nydus-image/builder/tarball.rs +++ b/rafs/src/builder/tarball.rs @@ -23,13 +23,9 @@ use std::os::unix::ffi::OsStrExt; use std::path::{Path, PathBuf}; use std::sync::Mutex; -use anyhow::{Context, Result}; +use anyhow::{anyhow, bail, Context, Result}; use tar::{Archive, Entry, EntryType, Header}; -use nydus_rafs::metadata::inode::InodeWrapper; -use nydus_rafs::metadata::layout::v5::{RafsV5Inode, RafsV5InodeFlags}; -use nydus_rafs::metadata::layout::RafsXAttrs; -use nydus_rafs::metadata::{Inode, RafsVersion}; use nydus_storage::device::BlobFeatures; use nydus_storage::meta::ZranContextGenerator; use nydus_storage::RAFS_MAX_CHUNKS_PER_BLOB; @@ -37,15 +33,19 @@ use nydus_utils::compact::makedev; use nydus_utils::compress::zlib_random::{ZranReader, ZRAN_READER_BUF_SIZE}; use nydus_utils::compress::ZlibDecoder; use nydus_utils::digest::RafsDigest; -use nydus_utils::{div_round_up, BufReaderInfo, ByteSize}; +use nydus_utils::{div_round_up, root_tracer, timing_tracer, BufReaderInfo, ByteSize}; -use crate::builder::{build_bootstrap, dump_bootstrap, finalize_blob, Builder}; -use crate::core::blob::Blob; -use crate::core::context::{ +use super::core::blob::Blob; +use super::core::context::{ ArtifactWriter, BlobManager, BootstrapManager, BuildContext, BuildOutput, ConversionType, }; -use crate::core::node::{Node, Overlay}; -use crate::core::tree::Tree; +use super::core::node::{Node, Overlay}; +use super::core::tree::Tree; +use super::{build_bootstrap, dump_bootstrap, finalize_blob, Builder}; +use crate::metadata::inode::InodeWrapper; +use crate::metadata::layout::v5::{RafsV5Inode, RafsV5InodeFlags}; +use crate::metadata::layout::RafsXAttrs; +use crate::metadata::{Inode, RafsVersion}; enum TarReader { File(File), diff --git a/rafs/src/lib.rs b/rafs/src/lib.rs index 02ccf465619..8c5879fec87 100644 --- a/rafs/src/lib.rs +++ b/rafs/src/lib.rs @@ -48,6 +48,7 @@ use std::sync::Arc; use crate::metadata::{RafsInodeExt, RafsSuper}; +pub mod builder; pub mod fs; pub mod metadata; #[cfg(test)] diff --git a/src/bin/nydus-image/main.rs b/src/bin/nydus-image/main.rs index 05463b6e80d..915e73907c1 100644 --- a/src/bin/nydus-image/main.rs +++ b/src/bin/nydus-image/main.rs @@ -26,6 +26,11 @@ use nix::unistd::{getegid, geteuid}; use nydus::get_build_time_info; use nydus_api::{BuildTimeInfo, ConfigV2, LocalFsConfig}; use nydus_app::setup_logging; +use nydus_rafs::builder::{ + import_chunk_dict, parse_chunk_dict_arg, ArtifactStorage, BlobCompactor, BlobManager, + BootstrapManager, BuildContext, BuildOutput, Builder, ConversionType, DirectoryBuilder, + Feature, Features, Prefetch, PrefetchPolicy, StargzBuilder, TarballBuilder, WhiteoutSpec, +}; use nydus_rafs::metadata::{RafsSuper, RafsSuperConfig, RafsVersion}; use nydus_storage::backend::localfs::LocalFs; use nydus_storage::backend::BlobBackend; @@ -33,28 +38,14 @@ use nydus_storage::device::BlobFeatures; use nydus_storage::factory::BlobFactory; use nydus_storage::meta::format_blob_features; use nydus_storage::{RAFS_DEFAULT_CHUNK_SIZE, RAFS_MAX_CHUNK_SIZE}; -use nydus_utils::{compress, digest}; +use nydus_utils::trace::{EventTracerClass, TimingTracerClass, TraceClass}; +use nydus_utils::{compress, digest, event_tracer, register_tracer, root_tracer, timing_tracer}; use serde::{Deserialize, Serialize}; -use crate::builder::{Builder, DirectoryBuilder, StargzBuilder, TarballBuilder}; -use crate::core::blob_compact::BlobCompactor; -use crate::core::chunk_dict::{import_chunk_dict, parse_chunk_dict_arg}; -use crate::core::context::{ - ArtifactStorage, BlobManager, BootstrapManager, BuildContext, BuildOutput, ConversionType, -}; -use crate::core::feature::{Feature, Features}; -use crate::core::node::{self, WhiteoutSpec}; -use crate::core::prefetch::{Prefetch, PrefetchPolicy}; -use crate::core::tree; use crate::merge::Merger; -use crate::trace::{EventTracerClass, TimingTracerClass, TraceClass}; use crate::unpack::{OCIUnpacker, Unpacker}; use crate::validator::Validator; -#[macro_use] -mod trace; -mod builder; -mod core; mod inspect; mod merge; mod stat; diff --git a/src/bin/nydus-image/merge.rs b/src/bin/nydus-image/merge.rs index 802b2cad8a5..5764d539b14 100644 --- a/src/bin/nydus-image/merge.rs +++ b/src/bin/nydus-image/merge.rs @@ -10,17 +10,13 @@ use std::sync::Arc; use anyhow::{Context, Result}; use hex::FromHex; use nydus_api::ConfigV2; +use nydus_rafs::builder::{ + ArtifactStorage, BlobContext, BlobManager, Bootstrap, BootstrapContext, BuildContext, + BuildOutput, ChunkSource, HashChunkDict, MetadataTreeBuilder, Overlay, Tree, WhiteoutSpec, +}; use nydus_rafs::metadata::{RafsInodeExt, RafsSuper, RafsVersion}; use nydus_storage::device::{BlobFeatures, BlobInfo}; -use crate::core::bootstrap::Bootstrap; -use crate::core::chunk_dict::HashChunkDict; -use crate::core::context::{ - ArtifactStorage, BlobContext, BlobManager, BootstrapContext, BuildContext, BuildOutput, -}; -use crate::core::node::{ChunkSource, Overlay, WhiteoutSpec}; -use crate::core::tree::{MetadataTreeBuilder, Tree}; - /// Struct to generate the merged RAFS bootstrap for an image from per layer RAFS bootstraps. /// /// A container image contains one or more layers, a RAFS bootstrap is built for each layer. @@ -218,7 +214,7 @@ impl Merger { for chunk in &mut node.chunks { let origin_blob_index = chunk.inner.blob_index() as usize; // Set the blob index of chunk to real index in blob table of final bootstrap. - chunk.inner.set_blob_index(blob_idx_map[origin_blob_index]); + chunk.set_blob_index(blob_idx_map[origin_blob_index]); } // Set node's layer index to distinguish same inode number (from bootstrap) // between different layers. diff --git a/src/bin/nydus-image/stat.rs b/src/bin/nydus-image/stat.rs index fbcfeab97d5..09b20a4a011 100644 --- a/src/bin/nydus-image/stat.rs +++ b/src/bin/nydus-image/stat.rs @@ -10,13 +10,11 @@ use std::sync::Arc; use anyhow::{Context, Result}; use nydus_api::ConfigV2; +use nydus_rafs::builder::{ChunkDict, HashChunkDict, Tree}; use nydus_rafs::metadata::RafsSuper; use nydus_utils::digest; use serde::Serialize; -use crate::core::chunk_dict::{ChunkDict, HashChunkDict}; -use crate::core::tree::Tree; - #[derive(Copy, Clone, Default, Serialize)] struct DedupInfo { raw_chunks: u64, @@ -205,7 +203,7 @@ impl ImageStat { })?; if is_base { - for entry in dict.m.values() { + for entry in dict.hashmap().values() { image.own_chunks += 1; image.own_comp_size += entry.0.compressed_size() as u64; image.own_uncomp_size += entry.0.uncompressed_size() as u64; @@ -213,7 +211,7 @@ impl ImageStat { .add_chunk(entry.0.clone(), rs.meta.get_digester()); } } else { - for entry in dict.m.values() { + for entry in dict.hashmap().values() { if self .dedup_dict .get_chunk(entry.0.id(), entry.0.uncompressed_size()) @@ -241,7 +239,7 @@ impl ImageStat { } if self.dedup_enabled { - for entry in self.dedup_dict.m.values() { + for entry in self.dedup_dict.hashmap().values() { let count = entry.1.load(Ordering::Relaxed); let thresh = std::cmp::min(self.dedup_info.len(), count as usize); for idx in 0..thresh { diff --git a/src/bin/nydus-image/validator.rs b/src/bin/nydus-image/validator.rs index ef6a801a4e4..f97420e219b 100644 --- a/src/bin/nydus-image/validator.rs +++ b/src/bin/nydus-image/validator.rs @@ -9,11 +9,10 @@ use std::sync::Arc; use anyhow::{Context, Result}; use nydus_api::ConfigV2; +use nydus_rafs::builder::Tree; use nydus_rafs::metadata::RafsSuper; use nydus_storage::device::BlobInfo; -use crate::tree::Tree; - pub struct Validator { sb: RafsSuper, } diff --git a/utils/src/lib.rs b/utils/src/lib.rs index b83b603a1ac..2eb555e27bb 100644 --- a/utils/src/lib.rs +++ b/utils/src/lib.rs @@ -29,6 +29,7 @@ pub mod inode_bitmap; pub mod metrics; pub mod mpmc; pub mod reader; +pub mod trace; pub mod types; /// Round up and divide the value `n` by `d`. diff --git a/src/bin/nydus-image/trace.rs b/utils/src/trace.rs similarity index 99% rename from src/bin/nydus-image/trace.rs rename to utils/src/trace.rs index dde82b0dce2..f43d8f00605 100644 --- a/src/bin/nydus-image/trace.rs +++ b/utils/src/trace.rs @@ -164,6 +164,7 @@ lazy_static! { }; } +#[macro_export] macro_rules! root_tracer { () => { &$crate::trace::BUILDING_RECORDER as &$crate::trace::BuildRootTracer From 4b90c87c582e092b55a4de3cda0b55973262bad1 Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Sun, 5 Mar 2023 10:28:44 +0800 Subject: [PATCH 04/10] rafs: refine Node structure to reduce memory consumption and copy Organize immutable fields of Node into a new struct NodeInfo, to reduce memory consumption and copy operations. Signed-off-by: Jiang Liu --- Cargo.toml | 3 + rafs/src/builder/compact.rs | 2 +- rafs/src/builder/core/blob.rs | 5 +- rafs/src/builder/core/bootstrap.rs | 27 ++-- rafs/src/builder/core/chunk_dict.rs | 2 +- rafs/src/builder/core/context.rs | 4 +- rafs/src/builder/core/feature.rs | 6 + rafs/src/builder/core/layout.rs | 2 +- rafs/src/builder/core/node.rs | 213 ++++++++++++++++------------ rafs/src/builder/core/prefetch.rs | 2 +- rafs/src/builder/core/tree.rs | 25 ++-- rafs/src/builder/directory.rs | 3 +- rafs/src/builder/stargz.rs | 25 ++-- rafs/src/builder/tarball.rs | 44 +++--- rafs/src/metadata/layout/mod.rs | 7 + 15 files changed, 219 insertions(+), 151 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 1c6287921a1..f502d8946a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,6 +68,9 @@ virtio-queue = { version = "0.6.0", optional = true } vm-memory = { version = "0.9.0", features = ["backend-mmap"], optional = true } vmm-sys-util = { version = "0.10.0", optional = true } +[dev-dependencies] +xattr = "0.2.3" + [features] default = [ "fuse-backend-rs/fusedev", diff --git a/rafs/src/builder/compact.rs b/rafs/src/builder/compact.rs index 614a8b0a74b..e0057992089 100644 --- a/rafs/src/builder/compact.rs +++ b/rafs/src/builder/compact.rs @@ -611,7 +611,7 @@ impl BlobCompactor { )?; compactor.compact(cfg)?; compactor.dump_new_blobs(&build_ctx, &cfg.blobs_dir, build_ctx.aligned_chunk)?; - if compactor.new_blob_mgr.len() == 0 { + if compactor.new_blob_mgr.is_empty() { info!("blobs of source bootstrap have already been optimized"); return Ok(None); } diff --git a/rafs/src/builder/core/blob.rs b/rafs/src/builder/core/blob.rs index 677f79fd840..7ae92071a2c 100644 --- a/rafs/src/builder/core/blob.rs +++ b/rafs/src/builder/core/blob.rs @@ -13,10 +13,11 @@ use nydus_utils::compress; use nydus_utils::digest::{self, DigestHasher, RafsDigest}; use sha2::digest::Digest; -use super::context::{ArtifactWriter, BlobContext, BlobManager, BuildContext, ConversionType}; -use super::feature::Feature; use super::layout::BlobLayout; use super::node::Node; +use crate::builder::{ + ArtifactWriter, BlobContext, BlobManager, BuildContext, ConversionType, Feature, +}; use crate::metadata::RAFS_MAX_CHUNK_SIZE; pub struct Blob {} diff --git a/rafs/src/builder/core/bootstrap.rs b/rafs/src/builder/core/bootstrap.rs index 279d73f7a90..94ad67d6ffa 100644 --- a/rafs/src/builder/core/bootstrap.rs +++ b/rafs/src/builder/core/bootstrap.rs @@ -13,11 +13,11 @@ use nydus_storage::device::BlobFeatures; use nydus_utils::digest::{self, DigestHasher, RafsDigest}; use nydus_utils::{root_tracer, timing_tracer}; -use super::context::{ +use super::node::{Node, WhiteoutType, OVERLAYFS_WHITEOUT_OPAQUE}; +use crate::builder::{ ArtifactStorage, BlobManager, BootstrapContext, BootstrapManager, BuildContext, ConversionType, + Tree, }; -use super::node::{Node, WhiteoutType, OVERLAYFS_WHITEOUT_OPAQUE}; -use super::tree::Tree; use crate::metadata::layout::v5::{ RafsV5BlobTable, RafsV5ChunkInfo, RafsV5InodeTable, RafsV5SuperBlock, RafsV5XAttrsTable, }; @@ -174,8 +174,8 @@ impl Bootstrap { let mut v6_hardlink_offset: Option = None; if let Some(indexes) = bootstrap_ctx.inode_map.get_mut(&( child.node.layer_idx, - child.node.src_ino, - child.node.src_dev, + child.node.info.src_ino, + child.node.info.src_dev, )) { let nlink = indexes.len() as u32 + 1; let first_index = indexes[0]; @@ -193,7 +193,11 @@ impl Bootstrap { child.node.inode.set_nlink(1); // Store inode real ino bootstrap_ctx.inode_map.insert( - (child.node.layer_idx, child.node.src_ino, child.node.src_dev), + ( + child.node.layer_idx, + child.node.info.src_ino, + child.node.info.src_dev, + ), vec![child.node.index], ); } @@ -462,9 +466,10 @@ impl Bootstrap { inode_offset += node.inode.inode_size() as u32; if node.inode.has_xattr() { has_xattr = true; - if !node.xattrs.is_empty() { - inode_offset += - (size_of::() + node.xattrs.aligned_size_v5()) as u32; + if !node.info.xattrs.is_empty() { + inode_offset += (size_of::() + + node.info.xattrs.aligned_size_v5()) + as u32; } } // Add chunks size @@ -558,11 +563,11 @@ impl Bootstrap { let (prefetch_table_offset, prefetch_table_size) = // If blob_table_size equal to 0, there is no prefetch. - if ctx.prefetch.len() > 0 && blob_table_size > 0 { + if ctx.prefetch.fs_prefetch_rule_count() > 0 && blob_table_size > 0 { // Prefetch table is very close to blob devices table let offset = blob_table_offset + blob_table_size; // Each prefetched file has is nid of `u32` filled into prefetch table. - let size = ctx.prefetch.len() * size_of::() as u32; + let size = ctx.prefetch.fs_prefetch_rule_count() * size_of::() as u32; trace!("prefetch table locates at offset {} size {}", offset, size); (offset, size) } else { diff --git a/rafs/src/builder/core/chunk_dict.rs b/rafs/src/builder/core/chunk_dict.rs index e38be053f5d..181067d092f 100644 --- a/rafs/src/builder/core/chunk_dict.rs +++ b/rafs/src/builder/core/chunk_dict.rs @@ -238,7 +238,7 @@ mod tests { fn test_chunk_dict() { let root_dir = &std::env::var("CARGO_MANIFEST_DIR").expect("$CARGO_MANIFEST_DIR"); let mut source_path = PathBuf::from(root_dir); - source_path.push("tests/texture/bootstrap/rafs-v5.boot"); + source_path.push("../tests/texture/bootstrap/rafs-v5.boot"); let path = source_path.to_str().unwrap(); let rafs_config = RafsSuperConfig { version: RafsVersion::V5, diff --git a/rafs/src/builder/core/context.rs b/rafs/src/builder/core/context.rs index 5c1f73f5526..d670364ccdd 100644 --- a/rafs/src/builder/core/context.rs +++ b/rafs/src/builder/core/context.rs @@ -32,10 +32,8 @@ use nydus_storage::meta::{ use nydus_utils::digest::DigestData; use nydus_utils::{compress, digest, div_round_up, round_down_4k, BufReaderInfo}; -use super::chunk_dict::{ChunkDict, HashChunkDict}; -use super::feature::{Feature, Features}; use super::node::{ChunkSource, Node, WhiteoutSpec}; -use super::prefetch::{Prefetch, PrefetchPolicy}; +use crate::builder::{ChunkDict, Feature, Features, HashChunkDict, Prefetch, PrefetchPolicy}; use crate::metadata::chunk::ChunkWrapper; use crate::metadata::layout::v5::RafsV5BlobTable; use crate::metadata::layout::v6::{RafsV6BlobTable, EROFS_BLOCK_SIZE, EROFS_INODE_SLOT_SIZE}; diff --git a/rafs/src/builder/core/feature.rs b/rafs/src/builder/core/feature.rs index a193ccf7e0e..41b4e690ff9 100644 --- a/rafs/src/builder/core/feature.rs +++ b/rafs/src/builder/core/feature.rs @@ -18,6 +18,12 @@ pub enum Feature { /// Feature set for RAFS filesystem builder. pub struct Features(HashSet); +impl Default for Features { + fn default() -> Self { + Self::new() + } +} + impl Features { /// Create a new instance of [Features]. pub fn new() -> Self { diff --git a/rafs/src/builder/core/layout.rs b/rafs/src/builder/core/layout.rs index b99a1eb2b3f..af737ea4463 100644 --- a/rafs/src/builder/core/layout.rs +++ b/rafs/src/builder/core/layout.rs @@ -5,7 +5,7 @@ use anyhow::Result; use super::node::{Node, Overlay}; -use super::prefetch::Prefetch; +use crate::builder::Prefetch; #[derive(Clone)] pub struct BlobLayout {} diff --git a/rafs/src/builder/core/node.rs b/rafs/src/builder/core/node.rs index 707f52eb196..65d4325ad8e 100644 --- a/rafs/src/builder/core/node.rs +++ b/rafs/src/builder/core/node.rs @@ -197,21 +197,15 @@ impl Display for NodeChunk { } impl NodeChunk { - pub fn set_index(&mut self, index: u32) { - let mut chunk = self.inner.deref().clone(); - chunk.set_index(index); - self.inner = Arc::new(chunk); - } - pub fn copy_from(&mut self, other: &ChunkWrapper) { let mut chunk = self.inner.deref().clone(); chunk.copy_from(other); self.inner = Arc::new(chunk); } - pub fn set_file_offset(&mut self, offset: u64) { + pub fn set_index(&mut self, index: u32) { let mut chunk = self.inner.deref().clone(); - chunk.set_file_offset(offset); + chunk.set_index(index); self.inner = Arc::new(chunk); } @@ -226,13 +220,22 @@ impl NodeChunk { chunk.set_compressed_size(size); self.inner = Arc::new(chunk); } + + pub fn set_file_offset(&mut self, offset: u64) { + let mut chunk = self.inner.deref().clone(); + chunk.set_file_offset(offset); + self.inner = Arc::new(chunk); + } } -/// An in-memory representation of RAFS inode for image building and inspection. -#[derive(Clone)] -pub struct Node { - /// Assigned RAFS inode number. - pub index: u64, +/// Immutable information for a [Node] object. +#[derive(Clone, Default, Debug)] +pub struct NodeInfo { + /// Last status change time of the file, in nanoseconds. + pub ctime: i64, + /// Whether the explicit UID/GID feature is enabled or not. + pub explicit_uidgid: bool, + /// Device id associated with the source inode. /// /// A source directory may contain multiple partitions from different hard disk, so @@ -242,18 +245,6 @@ pub struct Node { pub src_ino: Inode, /// Device ID for special files, describing the device that this inode represents. pub rdev: u64, - /// Define a disk inode structure to persist to disk. - pub inode: InodeWrapper, - /// Chunks info list of regular file - pub chunks: Vec, - /// Extended attributes. - pub xattrs: RafsXAttrs, - /// Symlink info of symlink file - pub symlink: Option, - /// Overlay type for layered build - pub overlay: Overlay, - /// Whether the explicit UID/GID feature is enabled or not. - pub explicit_uidgid: bool, /// Absolute path of the source root directory. pub source: PathBuf, /// Absolute path of the source file/directory. @@ -262,15 +253,33 @@ pub struct Node { pub target: PathBuf, /// Parsed version of `target`. pub target_vec: Vec, + /// Symlink info of symlink file + pub symlink: Option, + /// Extended attributes. + pub xattrs: RafsXAttrs, + + /// V6: whether it's forced to use an extended inode. + pub v6_force_extended_inode: bool, +} + +/// An in-memory representation of RAFS inode for image building and inspection. +#[derive(Clone)] +pub struct Node { + /// Immutable fields of a Node object. + pub info: Arc, + /// Assigned RAFS inode number. + pub index: u64, + /// Define a disk inode structure to persist to disk. + pub inode: InodeWrapper, + /// Chunks info list of regular file + pub chunks: Vec, /// Layer index where node is located. pub layer_idx: u16, - /// Last status change time of the file, in nanoseconds. - pub ctime: i64, + /// Overlay type for layered build + pub overlay: Overlay, /// V6: whether it's a compact inode or an extended inode. pub v6_compact_inode: bool, - /// V6: whether it's forced to use an extended inode. - pub v6_force_extended_inode: bool, /// V6: inode data layout. pub v6_datalayout: u16, /// V6: offset to calculate nid. @@ -290,7 +299,7 @@ impl Display for Node { self.target(), self.index, self.inode.ino(), - self.src_ino, + self.info.src_ino, self.inode.parent(), self.inode.child_index(), self.inode.child_count(), @@ -300,7 +309,7 @@ impl Display for Node { self.inode.name_size(), self.inode.symlink_size(), self.inode.has_xattr(), - self.symlink, + self.info.symlink, self.inode.mtime(), self.inode.mtime_nsec(), ) @@ -319,8 +328,9 @@ impl Node { ) -> Result { let target = Self::generate_target(&path, &source); let target_vec = Self::generate_target_vec(&target); - let mut node = Node { - index: 0, + let info = NodeInfo { + ctime: 0, + explicit_uidgid, src_ino: 0, src_dev: u64::MAX, rdev: u64::MAX, @@ -328,19 +338,21 @@ impl Node { target, path, target_vec, - overlay, - inode: InodeWrapper::new(version), - chunks: Vec::new(), symlink: None, xattrs: RafsXAttrs::default(), - explicit_uidgid, + v6_force_extended_inode, + }; + let mut node = Node { + info: Arc::new(info), + index: 0, layer_idx: 0, - ctime: 0, - v6_offset: 0, - v6_dirents: Vec::new(), + overlay, + inode: InodeWrapper::new(version), + chunks: Vec::new(), v6_datalayout: EROFS_INODE_FLAT_PLAIN, - v6_force_extended_inode, v6_compact_inode: false, + v6_offset: 0, + v6_dirents: Vec::new(), v6_dirents_offset: 0, }; @@ -367,8 +379,8 @@ impl Node { chunk_data_buf: &mut [u8], ) -> Result { let mut reader = if self.is_reg() { - let file = File::open(&self.path) - .with_context(|| format!("failed to open node file {:?}", self.path))?; + let file = File::open(&self.path()) + .with_context(|| format!("failed to open node file {:?}", self.path()))?; Some(file) } else { None @@ -394,7 +406,7 @@ impl Node { if self.is_dir() { return Ok(0); } else if self.is_symlink() { - if let Some(symlink) = self.symlink.as_ref() { + if let Some(symlink) = self.info.symlink.as_ref() { self.inode .set_digest(RafsDigest::from_buf(symlink.as_bytes(), ctx.digester)); return Ok(0); @@ -470,40 +482,45 @@ impl Node { } fn build_inode_xattr(&mut self) -> Result<()> { - let file_xattrs = match xattr::list(&self.path) { + let file_xattrs = match xattr::list(self.path()) { Ok(x) => x, Err(e) => { if e.raw_os_error() == Some(libc::EOPNOTSUPP) { return Ok(()); } else { - return Err(anyhow!("failed to list xattr of {:?}", self.path)); + return Err(anyhow!("failed to list xattr of {:?}", self.path())); } } }; + let mut info = self.info.deref().clone(); for key in file_xattrs { - let value = xattr::get(&self.path, &key) - .context(format!("failed to get xattr {:?} of {:?}", key, self.path))?; - self.xattrs.add(key, value.unwrap_or_default())?; + let value = xattr::get(self.path(), &key).context(format!( + "failed to get xattr {:?} of {:?}", + key, + self.path() + ))?; + info.xattrs.add(key, value.unwrap_or_default())?; } - - if !self.xattrs.is_empty() { + if !info.xattrs.is_empty() { self.inode.set_has_xattr(true); } + self.info = Arc::new(info); Ok(()) } fn build_inode_stat(&mut self) -> Result<()> { let meta = self.meta()?; + let mut info = self.info.deref().clone(); - self.src_ino = meta.st_ino(); - self.src_dev = meta.st_dev(); - self.rdev = meta.st_rdev(); - self.ctime = meta.st_ctime(); + info.src_ino = meta.st_ino(); + info.src_dev = meta.st_dev(); + info.rdev = meta.st_rdev(); + info.ctime = meta.st_ctime(); self.inode.set_mode(meta.st_mode()); - if self.explicit_uidgid { + if info.explicit_uidgid { self.inode.set_uid(meta.st_uid()); self.inode.set_gid(meta.st_gid()); } @@ -530,6 +547,7 @@ impl Node { self.inode.set_size(meta.st_size()); self.set_inode_blocks(); } + self.info = Arc::new(info); Ok(()) } @@ -540,27 +558,29 @@ impl Node { // NOTE: Always retrieve xattr before attr so that we can know the size of xattr pairs. self.build_inode_xattr()?; self.build_inode_stat() - .with_context(|| format!("failed to build inode {:?}", self.path))?; + .with_context(|| format!("failed to build inode {:?}", self.path()))?; if self.is_reg() { // Reuse `child_count` to store `chunk_count` for normal files. self.inode .set_child_count(self.chunk_count(chunk_size as u64)); } else if self.is_symlink() { - let target_path = fs::read_link(&self.path)?; + let target_path = fs::read_link(self.path())?; let symlink: OsString = target_path.into(); let size = symlink.byte_size(); self.inode.set_symlink_size(size); - self.symlink = Some(symlink); + let mut info = self.info.deref().clone(); + info.symlink = Some(symlink); + self.info = Arc::new(info); } Ok(()) } fn meta(&self) -> Result { - self.path + self.path() .symlink_metadata() - .with_context(|| format!("failed to get metadata from {:?}", self.path)) + .with_context(|| format!("failed to get metadata from {:?}", self.path())) } fn read_file_chunk( @@ -576,7 +596,7 @@ impl Node { zran.start_chunk(ctx.chunk_size as u64)?; reader .read_exact(buf) - .with_context(|| format!("failed to read node file {:?}", self.path))?; + .with_context(|| format!("failed to read node file {:?}", self.path()))?; let info = zran.finish_chunk()?; chunk.set_compressed_offset(info.compressed_offset()); chunk.set_compressed_size(info.compressed_size()); @@ -590,11 +610,11 @@ impl Node { chunk.set_compressed(false); reader .read_exact(buf) - .with_context(|| format!("failed to read node file {:?}", self.path))?; + .with_context(|| format!("failed to read node file {:?}", self.path()))?; } else { reader .read_exact(buf) - .with_context(|| format!("failed to read node file {:?}", self.path))?; + .with_context(|| format!("failed to read node file {:?}", self.path()))?; } let chunk_id = RafsDigest::from_buf(buf, ctx.digester); @@ -629,7 +649,7 @@ impl Node { } else { // For other case which needs to write chunk data to data blobs. let (compressed, is_compressed) = compress::compress(chunk_data, ctx.compressor) - .with_context(|| format!("failed to compress node file {:?}", self.path))?; + .with_context(|| format!("failed to compress node file {:?}", self.path()))?; let compressed_size = compressed.len() as u32; let pre_compressed_offset = blob_ctx.current_compressed_offset; blob_writer @@ -774,17 +794,17 @@ impl Node { /// Get filename of the inode. pub fn name(&self) -> &OsStr { - if self.path == self.source { + if self.path() == &self.info.source { OsStr::from_bytes(ROOT_PATH_NAME) } else { // Safe to unwrap because `path` is returned from `path()` which is canonicalized - self.path.file_name().unwrap() + self.path().file_name().unwrap() } } /// Get path of the inode pub fn path(&self) -> &PathBuf { - &self.path + &self.info.path } /// Generate cached components of the target file path. @@ -801,7 +821,7 @@ impl Node { /// Get cached components of the target file path. pub fn target_vec(&self) -> &[OsString] { - &self.target_vec + &self.info.target_vec } /// Generate target path by stripping the `root` prefix. @@ -822,7 +842,7 @@ impl Node { /// Get the absolute path of the inode within the RAFS filesystem. pub fn target(&self) -> &PathBuf { - &self.target + &self.info.target } /// Calculate and set `i_blocks` for inode. @@ -836,18 +856,27 @@ impl Node { // Set inode blocks for RAFS v5 inode, v6 will calculate it at runtime. if let InodeWrapper::V5(_) = self.inode { self.inode.set_blocks(div_round_up( - self.inode.size() + self.xattrs.aligned_size_v5() as u64, + self.inode.size() + self.info.xattrs.aligned_size_v5() as u64, 512, )); } } + /// Set extended attributes for the node. + pub fn set_xattr(&mut self, xattr: RafsXAttrs) { + let mut info = self.info.deref().clone(); + info.xattrs = xattr; + self.info = Arc::new(info); + } + /// Delete an extend attribute with id `key`. pub fn remove_xattr(&mut self, key: &OsStr) { - self.xattrs.remove(key); - if self.xattrs.is_empty() { + let mut info = self.info.deref().clone(); + info.xattrs.remove(key); + if info.xattrs.is_empty() { self.inode.set_has_xattr(false); } + self.info = Arc::new(info); } } @@ -866,7 +895,7 @@ impl Node { let name = self.name(); let inode = RafsV5InodeWrapper { name, - symlink: self.symlink.as_deref(), + symlink: self.info.symlink.as_deref(), inode: raw_inode, }; inode @@ -874,8 +903,9 @@ impl Node { .context("failed to dump inode to bootstrap")?; // Dump inode xattr - if !self.xattrs.is_empty() { - self.xattrs + if !self.info.xattrs.is_empty() { + self.info + .xattrs .store_v5(f_bootstrap) .context("failed to dump xattr to bootstrap")?; ctx.has_xattr = true; @@ -937,7 +967,7 @@ impl Node { meta_addr: u64, chunk_cache: &mut BTreeMap>, ) -> Result<()> { - let xattr_inline_count = self.xattrs.count_v6(); + let xattr_inline_count = self.info.xattrs.count_v6(); ensure!( xattr_inline_count <= u16::MAX as usize, "size of extended attributes is too big" @@ -1044,7 +1074,7 @@ impl Node { fn v6_size_with_xattr(&self) -> usize { self.inode - .get_inode_size_with_xattr(&self.xattrs, self.v6_compact_inode) + .get_inode_size_with_xattr(&self.info.xattrs, self.v6_compact_inode) } // For DIR inode, size is the total bytes of 'dirents + names'. @@ -1160,12 +1190,12 @@ impl Node { } fn v6_set_inode_compact(&mut self) { - if self.v6_force_extended_inode + if self.info.v6_force_extended_inode || self.inode.uid() > u16::MAX as u32 || self.inode.gid() > u16::MAX as u32 || self.inode.nlink() > u16::MAX as u32 || self.inode.size() > u32::MAX as u64 - || self.path.extension() == Some(OsStr::new("pyc")) + || self.path().extension() == Some(OsStr::new("pyc")) { self.v6_compact_inode = false; } else { @@ -1178,8 +1208,9 @@ impl Node { ctx: &mut BuildContext, f_bootstrap: &mut dyn RafsIoWrite, ) -> Result<()> { - if !self.xattrs.is_empty() { - self.xattrs + if !self.info.xattrs.is_empty() { + self.info + .xattrs .store_v6(f_bootstrap) .context("failed to dump xattr to bootstrap")?; ctx.has_xattr = true; @@ -1200,7 +1231,7 @@ impl Node { inode.set_u((dirent_off / EROFS_BLOCK_SIZE) as u32); // Dump inode - trace!("{:?} dir inode: offset {}", self.target, self.v6_offset); + trace!("{:?} dir inode: offset {}", self.target(), self.v6_offset); f_bootstrap .seek(SeekFrom::Start(self.v6_offset)) .context("failed seek for dir inode")?; @@ -1216,7 +1247,7 @@ impl Node { trace!( "{:?} self.dirents.len {}", - self.target, + self.target(), self.v6_dirents.len() ); // fill dir blocks one by one @@ -1272,7 +1303,7 @@ impl Node { trace!( "{:?} used {} dir size {}", - self.target, + self.target(), used, self.inode.size() ); @@ -1383,7 +1414,7 @@ impl Node { self.v6_store_xattrs(ctx, f_bootstrap)?; // write symlink. - if let Some(symlink) = &self.symlink { + if let Some(symlink) = &self.info.symlink { let tail_off = match self.v6_datalayout { EROFS_INODE_FLAT_INLINE => self.v6_offset + self.v6_size_with_xattr() as u64, EROFS_INODE_FLAT_PLAIN => data_off, @@ -1410,8 +1441,8 @@ impl Node { return false; } self.inode.is_chrdev() - && nydus_utils::compact::major_dev(self.rdev) == 0 - && nydus_utils::compact::minor_dev(self.rdev) == 0 + && nydus_utils::compact::major_dev(self.info.rdev) == 0 + && nydus_utils::compact::minor_dev(self.info.rdev) == 0 } /// Check whether the inode (directory) is a overlayfs whiteout opaque. @@ -1421,7 +1452,11 @@ impl Node { } // A directory is made opaque by setting the xattr "trusted.overlay.opaque" to "y". - if let Some(v) = self.xattrs.get(&OsString::from(OVERLAYFS_WHITEOUT_OPAQUE)) { + if let Some(v) = self + .info + .xattrs + .get(&OsString::from(OVERLAYFS_WHITEOUT_OPAQUE)) + { if let Ok(v) = std::str::from_utf8(v.as_slice()) { return v == "y"; } diff --git a/rafs/src/builder/core/prefetch.rs b/rafs/src/builder/core/prefetch.rs index afa0b71e3af..ccdc760cd5d 100644 --- a/rafs/src/builder/core/prefetch.rs +++ b/rafs/src/builder/core/prefetch.rs @@ -165,7 +165,7 @@ impl Prefetch { self.files.values().copied().collect() } - pub fn len(&self) -> u32 { + pub fn fs_prefetch_rule_count(&self) -> u32 { if self.policy == PrefetchPolicy::Fs { self.patterns.values().len() as u32 } else { diff --git a/rafs/src/builder/core/tree.rs b/rafs/src/builder/core/tree.rs index d7abd161254..fc1d651e345 100644 --- a/rafs/src/builder/core/tree.rs +++ b/rafs/src/builder/core/tree.rs @@ -22,8 +22,8 @@ use std::sync::Arc; use anyhow::Result; use nydus_utils::{event_tracer, root_tracer, timing_tracer}; -use super::chunk_dict::ChunkDict; -use super::node::{ChunkSource, Node, NodeChunk, Overlay, WhiteoutSpec, WhiteoutType}; +use super::node::{ChunkSource, Node, NodeChunk, NodeInfo, Overlay, WhiteoutSpec, WhiteoutType}; +use crate::builder::ChunkDict; use crate::metadata::chunk::ChunkWrapper; use crate::metadata::inode::InodeWrapper; use crate::metadata::layout::{bytes_to_os_str, RafsXAttrs}; @@ -337,29 +337,32 @@ impl<'a> MetadataTreeBuilder<'a> { let source = PathBuf::from("/"); let target = Node::generate_target(&path, &source); let target_vec = Node::generate_target_vec(&target); - - Ok(Node { - index: 0, + let info = NodeInfo { + ctime: 0, + explicit_uidgid: rs.meta.explicit_uidgid(), src_ino: inode.ino(), src_dev, rdev, - overlay: Overlay::Lower, - explicit_uidgid: rs.meta.explicit_uidgid(), path, source, target, target_vec, - inode, - chunks, symlink, xattrs, + v6_force_extended_inode: false, + }; + + Ok(Node { + info: Arc::new(info), + index: 0, layer_idx: 0, - ctime: 0, + overlay: Overlay::Lower, + inode, + chunks, v6_offset: 0, v6_dirents: Vec::new(), v6_datalayout: 0, v6_compact_inode: false, - v6_force_extended_inode: false, v6_dirents_offset: 0, }) } diff --git a/rafs/src/builder/directory.rs b/rafs/src/builder/directory.rs index 81309b19d1e..42ce6ca1142 100644 --- a/rafs/src/builder/directory.rs +++ b/rafs/src/builder/directory.rs @@ -50,7 +50,7 @@ impl FilesystemTreeBuilder { path.clone(), Overlay::UpperAddition, ctx.chunk_size, - parent.explicit_uidgid, + parent.info.explicit_uidgid, true, ) .with_context(|| format!("failed to create node {:?}", path))?; @@ -75,6 +75,7 @@ impl FilesystemTreeBuilder { } } +#[derive(Default)] pub struct DirectoryBuilder {} impl DirectoryBuilder { diff --git a/rafs/src/builder/stargz.rs b/rafs/src/builder/stargz.rs index f476134f32e..ec61cbe37e8 100644 --- a/rafs/src/builder/stargz.rs +++ b/rafs/src/builder/stargz.rs @@ -28,7 +28,7 @@ use super::core::context::{ ArtifactWriter, BlobContext, BlobManager, BootstrapContext, BootstrapManager, BuildContext, BuildOutput, }; -use super::core::node::{ChunkSource, Node, NodeChunk, Overlay}; +use super::core::node::{ChunkSource, Node, NodeChunk, NodeInfo, Overlay}; use super::core::tree::Tree; use super::{build_bootstrap, Builder}; use crate::metadata::chunk::ChunkWrapper; @@ -622,29 +622,32 @@ impl StargzTreeBuilder { let source = PathBuf::from("/"); let target = Node::generate_target(&path, &source); let target_vec = Node::generate_target_vec(&target); - - Ok(Node { - index: 0, + let info = NodeInfo { + ctime: 0, + explicit_uidgid, src_ino: ino, src_dev: u64::MAX, rdev: entry.rdev() as u64, - overlay: Overlay::UpperAddition, - explicit_uidgid, source, target, path, target_vec, - inode, - chunks: Vec::new(), symlink, xattrs, + v6_force_extended_inode: false, + }; + + Ok(Node { + info: Arc::new(info), + index: 0, + overlay: Overlay::UpperAddition, + inode, + chunks: Vec::new(), layer_idx, - ctime: 0, v6_offset: 0, v6_dirents: Vec::<(u64, OsString, u32)>::new(), v6_datalayout: 0, v6_compact_inode: false, - v6_force_extended_inode: false, v6_dirents_offset: 0, }) } @@ -780,7 +783,7 @@ impl StargzBuilder { if let Some(h) = inode_hasher { let digest = if node.is_symlink() { RafsDigest::from_buf( - node.symlink.as_ref().unwrap().as_bytes(), + node.info.symlink.as_ref().unwrap().as_bytes(), digest::Algorithm::Sha256, ) } else { diff --git a/rafs/src/builder/tarball.rs b/rafs/src/builder/tarball.rs index 3d2cbfa0748..fe528f0eb36 100644 --- a/rafs/src/builder/tarball.rs +++ b/rafs/src/builder/tarball.rs @@ -21,7 +21,7 @@ use std::fs::{File, OpenOptions}; use std::io::{BufReader, Read}; use std::os::unix::ffi::OsStrExt; use std::path::{Path, PathBuf}; -use std::sync::Mutex; +use std::sync::{Arc, Mutex}; use anyhow::{anyhow, bail, Context, Result}; use tar::{Archive, Entry, EntryType, Header}; @@ -39,7 +39,7 @@ use super::core::blob::Blob; use super::core::context::{ ArtifactWriter, BlobManager, BootstrapManager, BuildContext, BuildOutput, ConversionType, }; -use super::core::node::{Node, Overlay}; +use super::core::node::{Node, NodeInfo, Overlay}; use super::core::tree::Tree; use super::{build_bootstrap, dump_bootstrap, finalize_blob, Builder}; use crate::metadata::inode::InodeWrapper; @@ -343,28 +343,31 @@ impl<'a> TarballTreeBuilder<'a> { let source = PathBuf::from("/"); let target = Node::generate_target(path.as_ref(), &source); let target_vec = Node::generate_target_vec(&target); - let mut node = Node { - index: 0, + let info = NodeInfo { + ctime: 0, + explicit_uidgid: self.ctx.explicit_uidgid, src_ino: ino, src_dev: u64::MAX, rdev: rdev as u64, - overlay: Overlay::UpperAddition, - explicit_uidgid: self.ctx.explicit_uidgid, path: path.as_ref().to_path_buf(), source, target, target_vec, - inode, - chunks: Vec::new(), symlink, xattrs, + v6_force_extended_inode: false, + }; + let mut node = Node { + info: Arc::new(info), + index: 0, + overlay: Overlay::UpperAddition, + inode, + chunks: Vec::new(), layer_idx: self.layer_idx, - ctime: 0, v6_offset: 0, v6_dirents: Vec::<(u64, OsString, u32)>::new(), v6_datalayout: 0, v6_compact_inode: false, - v6_force_extended_inode: false, v6_dirents_offset: 0, }; @@ -377,7 +380,7 @@ impl<'a> TarballTreeBuilder<'a> { node.inode.set_size(n.inode.size()); node.inode.set_child_count(n.inode.child_count()); node.chunks = n.chunks.clone(); - node.xattrs = n.xattrs.clone(); + node.set_xattr(n.info.xattrs.clone()); } else { node.dump_node_data_with_reader( self.ctx, @@ -479,28 +482,31 @@ impl<'a> TarballTreeBuilder<'a> { let source = PathBuf::from("/"); let target = Node::generate_target(path, &source); let target_vec = Node::generate_target_vec(&target); - let node = Node { - index: 0, + let info = NodeInfo { + ctime: 0, + explicit_uidgid: self.ctx.explicit_uidgid, src_ino: ino, src_dev: u64::MAX, rdev: u64::MAX, - overlay: Overlay::UpperAddition, - explicit_uidgid: self.ctx.explicit_uidgid, path: path.to_path_buf(), source, target, target_vec, - inode, - chunks: Vec::new(), symlink: None, xattrs: RafsXAttrs::new(), + v6_force_extended_inode: false, + }; + let node = Node { + info: Arc::new(info), + index: 0, + overlay: Overlay::UpperAddition, + inode, + chunks: Vec::new(), layer_idx: self.layer_idx, - ctime: 0, v6_offset: 0, v6_dirents: Vec::<(u64, OsString, u32)>::new(), v6_datalayout: 0, v6_compact_inode: false, - v6_force_extended_inode: false, v6_dirents_offset: 0, }; diff --git a/rafs/src/metadata/layout/mod.rs b/rafs/src/metadata/layout/mod.rs index a2dc7ae6b3c..bc702259e25 100644 --- a/rafs/src/metadata/layout/mod.rs +++ b/rafs/src/metadata/layout/mod.rs @@ -8,6 +8,7 @@ use std::collections::HashMap; use std::convert::TryInto; use std::ffi::{OsStr, OsString}; +use std::fmt::{Debug, Formatter}; use std::io::Result; use std::mem::size_of; use std::os::unix::ffi::OsStrExt; @@ -228,6 +229,12 @@ pub struct RafsXAttrs { pairs: HashMap, } +impl Debug for RafsXAttrs { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "extended attributes[...]") + } +} + impl RafsXAttrs { /// Create a new instance of `RafsXattrs`. pub fn new() -> Self { From fa574fb0c38baae81d45f7538b474f0a9c57e9fa Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Sun, 5 Mar 2023 14:34:18 +0800 Subject: [PATCH 05/10] rafs: move overlay related code into builder/core/overlay.rs Move overlay related code into builder/core/overlay.rs, for better maintenance. Signed-off-by: Jiang Liu --- rafs/src/builder/compact.rs | 9 +- rafs/src/builder/core/bootstrap.rs | 3 +- rafs/src/builder/core/chunk_dict.rs | 2 +- rafs/src/builder/core/context.rs | 6 +- rafs/src/builder/core/layout.rs | 3 +- rafs/src/builder/core/mod.rs | 1 + rafs/src/builder/core/node.rs | 189 +----------------------- rafs/src/builder/core/overlay.rs | 215 ++++++++++++++++++++++++++++ rafs/src/builder/core/tree.rs | 5 +- rafs/src/builder/directory.rs | 3 +- rafs/src/builder/mod.rs | 3 +- rafs/src/builder/stargz.rs | 3 +- rafs/src/builder/tarball.rs | 3 +- 13 files changed, 240 insertions(+), 205 deletions(-) create mode 100644 rafs/src/builder/core/overlay.rs diff --git a/rafs/src/builder/compact.rs b/rafs/src/builder/compact.rs index e0057992089..4f2ca8d953b 100644 --- a/rafs/src/builder/compact.rs +++ b/rafs/src/builder/compact.rs @@ -20,14 +20,11 @@ use nydus_utils::{digest, try_round_up_4k}; use super::core::blob::Blob; use super::core::bootstrap::Bootstrap; -use super::core::chunk_dict::{ChunkDict, HashChunkDict}; -use super::core::context::{ +use super::core::node::Node; +use crate::builder::{ ArtifactStorage, ArtifactWriter, BlobContext, BlobManager, BootstrapManager, BuildContext, - BuildOutput, ConversionType, + BuildOutput, ChunkDict, ConversionType, Features, HashChunkDict, Tree, WhiteoutSpec, }; -use super::core::feature::Features; -use super::core::node::{Node, WhiteoutSpec}; -use super::core::tree::Tree; use crate::metadata::chunk::ChunkWrapper; use crate::metadata::{RafsSuper, RafsVersion}; diff --git a/rafs/src/builder/core/bootstrap.rs b/rafs/src/builder/core/bootstrap.rs index 94ad67d6ffa..cad348889a8 100644 --- a/rafs/src/builder/core/bootstrap.rs +++ b/rafs/src/builder/core/bootstrap.rs @@ -13,7 +13,8 @@ use nydus_storage::device::BlobFeatures; use nydus_utils::digest::{self, DigestHasher, RafsDigest}; use nydus_utils::{root_tracer, timing_tracer}; -use super::node::{Node, WhiteoutType, OVERLAYFS_WHITEOUT_OPAQUE}; +use super::node::Node; +use super::overlay::{WhiteoutType, OVERLAYFS_WHITEOUT_OPAQUE}; use crate::builder::{ ArtifactStorage, BlobManager, BootstrapContext, BootstrapManager, BuildContext, ConversionType, Tree, diff --git a/rafs/src/builder/core/chunk_dict.rs b/rafs/src/builder/core/chunk_dict.rs index 181067d092f..4fb47ce3dfe 100644 --- a/rafs/src/builder/core/chunk_dict.rs +++ b/rafs/src/builder/core/chunk_dict.rs @@ -13,7 +13,7 @@ use nydus_api::ConfigV2; use nydus_storage::device::BlobInfo; use nydus_utils::digest::{self, RafsDigest}; -use super::tree::Tree; +use crate::builder::Tree; use crate::metadata::chunk::ChunkWrapper; use crate::metadata::layout::v5::RafsV5ChunkInfo; use crate::metadata::{RafsSuper, RafsSuperConfig}; diff --git a/rafs/src/builder/core/context.rs b/rafs/src/builder/core/context.rs index d670364ccdd..e25ef161da0 100644 --- a/rafs/src/builder/core/context.rs +++ b/rafs/src/builder/core/context.rs @@ -32,8 +32,10 @@ use nydus_storage::meta::{ use nydus_utils::digest::DigestData; use nydus_utils::{compress, digest, div_round_up, round_down_4k, BufReaderInfo}; -use super::node::{ChunkSource, Node, WhiteoutSpec}; -use crate::builder::{ChunkDict, Feature, Features, HashChunkDict, Prefetch, PrefetchPolicy}; +use super::node::{ChunkSource, Node}; +use crate::builder::{ + ChunkDict, Feature, Features, HashChunkDict, Prefetch, PrefetchPolicy, WhiteoutSpec, +}; use crate::metadata::chunk::ChunkWrapper; use crate::metadata::layout::v5::RafsV5BlobTable; use crate::metadata::layout::v6::{RafsV6BlobTable, EROFS_BLOCK_SIZE, EROFS_INODE_SLOT_SIZE}; diff --git a/rafs/src/builder/core/layout.rs b/rafs/src/builder/core/layout.rs index af737ea4463..ad48423b3ee 100644 --- a/rafs/src/builder/core/layout.rs +++ b/rafs/src/builder/core/layout.rs @@ -4,7 +4,8 @@ use anyhow::Result; -use super::node::{Node, Overlay}; +use super::node::Node; +use super::overlay::Overlay; use crate::builder::Prefetch; #[derive(Clone)] diff --git a/rafs/src/builder/core/mod.rs b/rafs/src/builder/core/mod.rs index b7856639ac2..b4016a6721b 100644 --- a/rafs/src/builder/core/mod.rs +++ b/rafs/src/builder/core/mod.rs @@ -9,5 +9,6 @@ pub(crate) mod context; pub(crate) mod feature; pub(crate) mod layout; pub(crate) mod node; +pub(crate) mod overlay; pub(crate) mod prefetch; pub(crate) mod tree; diff --git a/rafs/src/builder/core/node.rs b/rafs/src/builder/core/node.rs index 65d4325ad8e..aacb065efdb 100644 --- a/rafs/src/builder/core/node.rs +++ b/rafs/src/builder/core/node.rs @@ -16,7 +16,6 @@ use std::os::linux::fs::MetadataExt; use std::os::macos::fs::MetadataExt; use std::os::unix::ffi::OsStrExt; use std::path::{Component, Path, PathBuf}; -use std::str::FromStr; use std::sync::Arc; use anyhow::{anyhow, bail, ensure, Context, Error, Result}; @@ -31,6 +30,7 @@ use sha2::digest::Digest; use super::chunk_dict::{ChunkDict, DigestWithBlobIndex}; use super::context::{ArtifactWriter, BlobContext, BlobManager, BootstrapContext, BuildContext}; +use super::overlay::Overlay; use super::tree::Tree; use crate::metadata::chunk::ChunkWrapper; use crate::metadata::inode::{new_v6_inode, InodeWrapper}; @@ -57,112 +57,6 @@ const RAFS_V5_VIRTUAL_ENTRY_SIZE: u64 = 8; /// Filesystem root path for Unix OSs. pub const ROOT_PATH_NAME: &[u8] = &[b'/']; -/// Prefix for OCI whiteout file. -pub const OCISPEC_WHITEOUT_PREFIX: &str = ".wh."; -/// Prefix for OCI whiteout opaque. -pub const OCISPEC_WHITEOUT_OPAQUE: &str = ".wh..wh..opq"; -/// Extended attribute key for Overlayfs whiteout opaque. -pub const OVERLAYFS_WHITEOUT_OPAQUE: &str = "trusted.overlay.opaque"; - -// # Overlayfs Whiteout -// -// In order to support rm and rmdir without changing the lower filesystem, an overlay filesystem -// needs to record in the upper filesystem that files have been removed. This is done using -// whiteouts and opaque directories (non-directories are always opaque). -// -// A whiteout is created as a character device with 0/0 device number. When a whiteout is found -// in the upper level of a merged directory, any matching name in the lower level is ignored, -// and the whiteout itself is also hidden. -// -// A directory is made opaque by setting the xattr “trusted.overlay.opaque” to “y”. Where the upper -// filesystem contains an opaque directory, any directory in the lower filesystem with the same -// name is ignored. -// -// # OCI Image Whiteout -// - A whiteout file is an empty file with a special filename that signifies a path should be -// deleted. -// - A whiteout filename consists of the prefix .wh. plus the basename of the path to be deleted. -// - As files prefixed with .wh. are special whiteout markers, it is not possible to create a -// filesystem which has a file or directory with a name beginning with .wh.. -// - Once a whiteout is applied, the whiteout itself MUST also be hidden. -// - Whiteout files MUST only apply to resources in lower/parent layers. -// - Files that are present in the same layer as a whiteout file can only be hidden by whiteout -// files in subsequent layers. -// - In addition to expressing that a single entry should be removed from a lower layer, layers -// may remove all of the children using an opaque whiteout entry. -// - An opaque whiteout entry is a file with the name .wh..wh..opq indicating that all siblings -// are hidden in the lower layer. -#[derive(Clone, Copy, Debug, PartialEq)] -pub enum WhiteoutType { - OciOpaque, - OciRemoval, - OverlayFsOpaque, - OverlayFsRemoval, -} - -impl WhiteoutType { - pub fn is_removal(&self) -> bool { - *self == WhiteoutType::OciRemoval || *self == WhiteoutType::OverlayFsRemoval - } -} - -#[derive(Clone, Copy, PartialEq)] -pub enum WhiteoutSpec { - /// https://github.com/opencontainers/image-spec/blob/master/layer.md#whiteouts - Oci, - /// "whiteouts and opaque directories" in https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt - Overlayfs, - /// No whiteout spec, which will build all `.wh.*` and `.wh..wh..opq` files into bootstrap. - None, -} - -impl Default for WhiteoutSpec { - fn default() -> Self { - Self::Oci - } -} - -impl FromStr for WhiteoutSpec { - type Err = Error; - - fn from_str(s: &str) -> Result { - match s { - "oci" => Ok(Self::Oci), - "overlayfs" => Ok(Self::Overlayfs), - "none" => Ok(Self::None), - _ => Err(anyhow!("invalid whiteout spec")), - } - } -} - -#[allow(dead_code)] -#[derive(Clone, Debug, PartialEq)] -pub enum Overlay { - Lower, - UpperAddition, - UpperOpaque, - UpperRemoval, - UpperModification, -} - -impl Overlay { - pub fn is_lower_layer(&self) -> bool { - self == &Overlay::Lower - } -} - -impl Display for Overlay { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - match self { - Overlay::Lower => write!(f, "LOWER"), - Overlay::UpperAddition => write!(f, "ADDED"), - Overlay::UpperOpaque => write!(f, "OPAQUED"), - Overlay::UpperRemoval => write!(f, "REMOVED"), - Overlay::UpperModification => write!(f, "MODIFIED"), - } - } -} - /// Where the chunk data is actually stored. #[derive(Clone, Hash, PartialEq, Eq)] pub enum ChunkSource { @@ -1433,87 +1327,6 @@ impl Node { } } -// OCI and Overlayfs whiteout handling. -impl Node { - /// Check whether the inode is a special overlayfs whiteout file. - pub fn is_overlayfs_whiteout(&self, spec: WhiteoutSpec) -> bool { - if spec != WhiteoutSpec::Overlayfs { - return false; - } - self.inode.is_chrdev() - && nydus_utils::compact::major_dev(self.info.rdev) == 0 - && nydus_utils::compact::minor_dev(self.info.rdev) == 0 - } - - /// Check whether the inode (directory) is a overlayfs whiteout opaque. - pub fn is_overlayfs_opaque(&self, spec: WhiteoutSpec) -> bool { - if spec != WhiteoutSpec::Overlayfs || !self.is_dir() { - return false; - } - - // A directory is made opaque by setting the xattr "trusted.overlay.opaque" to "y". - if let Some(v) = self - .info - .xattrs - .get(&OsString::from(OVERLAYFS_WHITEOUT_OPAQUE)) - { - if let Ok(v) = std::str::from_utf8(v.as_slice()) { - return v == "y"; - } - } - - false - } - - /// Get whiteout type to process the inode. - pub fn whiteout_type(&self, spec: WhiteoutSpec) -> Option { - if self.overlay == Overlay::Lower { - return None; - } - - match spec { - WhiteoutSpec::Oci => { - if let Some(name) = self.name().to_str() { - if name == OCISPEC_WHITEOUT_OPAQUE { - return Some(WhiteoutType::OciOpaque); - } else if name.starts_with(OCISPEC_WHITEOUT_PREFIX) { - return Some(WhiteoutType::OciRemoval); - } - } - } - WhiteoutSpec::Overlayfs => { - if self.is_overlayfs_whiteout(spec) { - return Some(WhiteoutType::OverlayFsRemoval); - } else if self.is_overlayfs_opaque(spec) { - return Some(WhiteoutType::OverlayFsOpaque); - } - } - WhiteoutSpec::None => { - return None; - } - } - - None - } - - /// Get original filename from a whiteout filename. - pub fn origin_name(&self, t: WhiteoutType) -> Option<&OsStr> { - if let Some(name) = self.name().to_str() { - if t == WhiteoutType::OciRemoval { - // the whiteout filename prefixes the basename of the path to be deleted with ".wh.". - return Some(OsStr::from_bytes( - name[OCISPEC_WHITEOUT_PREFIX.len()..].as_bytes(), - )); - } else if t == WhiteoutType::OverlayFsRemoval { - // the whiteout file has the same name as the file to be deleted. - return Some(name.as_ref()); - } - } - - None - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/rafs/src/builder/core/overlay.rs b/rafs/src/builder/core/overlay.rs new file mode 100644 index 00000000000..60656c4c37a --- /dev/null +++ b/rafs/src/builder/core/overlay.rs @@ -0,0 +1,215 @@ +// Copyright 2020 Ant Group. All rights reserved. +// Copyright (C) 2021-2023 Alibaba Cloud. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +//! Execute file/directory whiteout rules when merging multiple RAFS filesystems +//! according to the OCI or Overlayfs specifications. + +use std::ffi::{OsStr, OsString}; +use std::fmt::{self, Display, Formatter}; +use std::os::unix::ffi::OsStrExt; +use std::str::FromStr; + +use anyhow::{anyhow, Error, Result}; + +use super::node::Node; + +/// Prefix for OCI whiteout file. +pub const OCISPEC_WHITEOUT_PREFIX: &str = ".wh."; +/// Prefix for OCI whiteout opaque. +pub const OCISPEC_WHITEOUT_OPAQUE: &str = ".wh..wh..opq"; +/// Extended attribute key for Overlayfs whiteout opaque. +pub const OVERLAYFS_WHITEOUT_OPAQUE: &str = "trusted.overlay.opaque"; + +/// RAFS filesystem overlay specifications. +/// +/// When merging multiple RAFS filesystems into one, special rules are needed to white out +/// files/directories in lower/parent filesystems. The whiteout specification defined by the +/// OCI image specification and Linux Overlayfs are widely adopted, so both of them are supported +/// by RAFS filesystem. +/// +/// # Overlayfs Whiteout +/// +/// In order to support rm and rmdir without changing the lower filesystem, an overlay filesystem +/// needs to record in the upper filesystem that files have been removed. This is done using +/// whiteouts and opaque directories (non-directories are always opaque). +/// +/// A whiteout is created as a character device with 0/0 device number. When a whiteout is found +/// in the upper level of a merged directory, any matching name in the lower level is ignored, +/// and the whiteout itself is also hidden. +/// +/// A directory is made opaque by setting the xattr “trusted.overlay.opaque” to “y”. Where the upper +/// filesystem contains an opaque directory, any directory in the lower filesystem with the same +/// name is ignored. +/// +/// # OCI Image Whiteout +/// - A whiteout file is an empty file with a special filename that signifies a path should be +/// deleted. +/// - A whiteout filename consists of the prefix .wh. plus the basename of the path to be deleted. +/// - As files prefixed with .wh. are special whiteout markers, it is not possible to create a +/// filesystem which has a file or directory with a name beginning with .wh.. +/// - Once a whiteout is applied, the whiteout itself MUST also be hidden. +/// - Whiteout files MUST only apply to resources in lower/parent layers. +/// - Files that are present in the same layer as a whiteout file can only be hidden by whiteout +/// files in subsequent layers. +/// - In addition to expressing that a single entry should be removed from a lower layer, layers +/// may remove all of the children using an opaque whiteout entry. +/// - An opaque whiteout entry is a file with the name .wh..wh..opq indicating that all siblings +/// are hidden in the lower layer. +#[derive(Clone, Copy, PartialEq)] +pub enum WhiteoutSpec { + /// Overlay whiteout rules according to the OCI image specification. + /// + /// https://github.com/opencontainers/image-spec/blob/master/layer.md#whiteouts + Oci, + /// Overlay whiteout rules according to the Linux Overlayfs specification. + /// + /// "whiteouts and opaque directories" in https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt + Overlayfs, + /// No whiteout, keep all content from lower/parent filesystems. + None, +} + +impl Default for WhiteoutSpec { + fn default() -> Self { + Self::Oci + } +} + +impl FromStr for WhiteoutSpec { + type Err = Error; + + fn from_str(s: &str) -> Result { + match s.to_lowercase().as_str() { + "oci" => Ok(Self::Oci), + "overlayfs" => Ok(Self::Overlayfs), + "none" => Ok(Self::None), + _ => Err(anyhow!("invalid whiteout spec")), + } + } +} + +/// RAFS filesystem overlay operation types. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum WhiteoutType { + OciOpaque, + OciRemoval, + OverlayFsOpaque, + OverlayFsRemoval, +} + +impl WhiteoutType { + pub fn is_removal(&self) -> bool { + *self == WhiteoutType::OciRemoval || *self == WhiteoutType::OverlayFsRemoval + } +} + +/// RAFS filesystem node overlay state. +#[allow(dead_code)] +#[derive(Clone, Debug, PartialEq)] +pub enum Overlay { + Lower, + UpperAddition, + UpperOpaque, + UpperRemoval, + UpperModification, +} + +impl Overlay { + pub fn is_lower_layer(&self) -> bool { + self == &Overlay::Lower + } +} + +impl Display for Overlay { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + match self { + Overlay::Lower => write!(f, "LOWER"), + Overlay::UpperAddition => write!(f, "ADDED"), + Overlay::UpperOpaque => write!(f, "OPAQUED"), + Overlay::UpperRemoval => write!(f, "REMOVED"), + Overlay::UpperModification => write!(f, "MODIFIED"), + } + } +} + +impl Node { + /// Check whether the inode is a special overlayfs whiteout file. + pub fn is_overlayfs_whiteout(&self, spec: WhiteoutSpec) -> bool { + if spec != WhiteoutSpec::Overlayfs { + return false; + } + self.inode.is_chrdev() + && nydus_utils::compact::major_dev(self.info.rdev) == 0 + && nydus_utils::compact::minor_dev(self.info.rdev) == 0 + } + + /// Check whether the inode (directory) is a overlayfs whiteout opaque. + pub fn is_overlayfs_opaque(&self, spec: WhiteoutSpec) -> bool { + if spec != WhiteoutSpec::Overlayfs || !self.is_dir() { + return false; + } + + // A directory is made opaque by setting the xattr "trusted.overlay.opaque" to "y". + if let Some(v) = self + .info + .xattrs + .get(&OsString::from(OVERLAYFS_WHITEOUT_OPAQUE)) + { + if let Ok(v) = std::str::from_utf8(v.as_slice()) { + return v == "y"; + } + } + + false + } + + /// Get whiteout type to process the inode. + pub fn whiteout_type(&self, spec: WhiteoutSpec) -> Option { + if self.overlay == Overlay::Lower { + return None; + } + + match spec { + WhiteoutSpec::Oci => { + if let Some(name) = self.name().to_str() { + if name == OCISPEC_WHITEOUT_OPAQUE { + return Some(WhiteoutType::OciOpaque); + } else if name.starts_with(OCISPEC_WHITEOUT_PREFIX) { + return Some(WhiteoutType::OciRemoval); + } + } + } + WhiteoutSpec::Overlayfs => { + if self.is_overlayfs_whiteout(spec) { + return Some(WhiteoutType::OverlayFsRemoval); + } else if self.is_overlayfs_opaque(spec) { + return Some(WhiteoutType::OverlayFsOpaque); + } + } + WhiteoutSpec::None => { + return None; + } + } + + None + } + + /// Get original filename from a whiteout filename. + pub fn origin_name(&self, t: WhiteoutType) -> Option<&OsStr> { + if let Some(name) = self.name().to_str() { + if t == WhiteoutType::OciRemoval { + // the whiteout filename prefixes the basename of the path to be deleted with ".wh.". + return Some(OsStr::from_bytes( + name[OCISPEC_WHITEOUT_PREFIX.len()..].as_bytes(), + )); + } else if t == WhiteoutType::OverlayFsRemoval { + // the whiteout file has the same name as the file to be deleted. + return Some(name.as_ref()); + } + } + + None + } +} diff --git a/rafs/src/builder/core/tree.rs b/rafs/src/builder/core/tree.rs index fc1d651e345..b98809a2865 100644 --- a/rafs/src/builder/core/tree.rs +++ b/rafs/src/builder/core/tree.rs @@ -22,8 +22,9 @@ use std::sync::Arc; use anyhow::Result; use nydus_utils::{event_tracer, root_tracer, timing_tracer}; -use super::node::{ChunkSource, Node, NodeChunk, NodeInfo, Overlay, WhiteoutSpec, WhiteoutType}; -use crate::builder::ChunkDict; +use super::node::{ChunkSource, Node, NodeChunk, NodeInfo}; +use super::overlay::{Overlay, WhiteoutType}; +use crate::builder::{ChunkDict, WhiteoutSpec}; use crate::metadata::chunk::ChunkWrapper; use crate::metadata::inode::InodeWrapper; use crate::metadata::layout::{bytes_to_os_str, RafsXAttrs}; diff --git a/rafs/src/builder/directory.rs b/rafs/src/builder/directory.rs index 42ce6ca1142..97b1b40186b 100644 --- a/rafs/src/builder/directory.rs +++ b/rafs/src/builder/directory.rs @@ -12,7 +12,8 @@ use super::core::blob::Blob; use super::core::context::{ ArtifactWriter, BlobManager, BootstrapContext, BootstrapManager, BuildContext, BuildOutput, }; -use super::core::node::{Node, Overlay}; +use super::core::node::Node; +use super::core::overlay::Overlay; use super::core::tree::Tree; use super::{build_bootstrap, dump_bootstrap, finalize_blob, Builder}; diff --git a/rafs/src/builder/mod.rs b/rafs/src/builder/mod.rs index 4735252b56a..0c636da51b4 100644 --- a/rafs/src/builder/mod.rs +++ b/rafs/src/builder/mod.rs @@ -18,7 +18,8 @@ pub use self::core::context::{ BuildContext, BuildOutput, ConversionType, }; pub use self::core::feature::{Feature, Features}; -pub use self::core::node::{ChunkSource, Overlay, WhiteoutSpec}; +pub use self::core::node::ChunkSource; +pub use self::core::overlay::{Overlay, WhiteoutSpec}; pub use self::core::prefetch::{Prefetch, PrefetchPolicy}; pub use self::core::tree::{MetadataTreeBuilder, Tree}; pub use self::directory::DirectoryBuilder; diff --git a/rafs/src/builder/stargz.rs b/rafs/src/builder/stargz.rs index ec61cbe37e8..bc55a80d47c 100644 --- a/rafs/src/builder/stargz.rs +++ b/rafs/src/builder/stargz.rs @@ -28,7 +28,8 @@ use super::core::context::{ ArtifactWriter, BlobContext, BlobManager, BootstrapContext, BootstrapManager, BuildContext, BuildOutput, }; -use super::core::node::{ChunkSource, Node, NodeChunk, NodeInfo, Overlay}; +use super::core::node::{ChunkSource, Node, NodeChunk, NodeInfo}; +use super::core::overlay::Overlay; use super::core::tree::Tree; use super::{build_bootstrap, Builder}; use crate::metadata::chunk::ChunkWrapper; diff --git a/rafs/src/builder/tarball.rs b/rafs/src/builder/tarball.rs index fe528f0eb36..ced521c409f 100644 --- a/rafs/src/builder/tarball.rs +++ b/rafs/src/builder/tarball.rs @@ -39,7 +39,8 @@ use super::core::blob::Blob; use super::core::context::{ ArtifactWriter, BlobManager, BootstrapManager, BuildContext, BuildOutput, ConversionType, }; -use super::core::node::{Node, NodeInfo, Overlay}; +use super::core::node::{Node, NodeInfo}; +use super::core::overlay::Overlay; use super::core::tree::Tree; use super::{build_bootstrap, dump_bootstrap, finalize_blob, Builder}; use crate::metadata::inode::InodeWrapper; From 009625d19f2a5e550a73851f3a2c9947f017e643 Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Sun, 5 Mar 2023 16:51:27 +0800 Subject: [PATCH 06/10] rafs: move RAFSv6 builder related code into a dedicated file Move RAFSv6 builder related code into a dedicated file. Signed-off-by: Jiang Liu --- Cargo.lock | 2 + rafs/src/builder/core/mod.rs | 2 + rafs/src/builder/core/node.rs | 755 +--------------------------------- rafs/src/builder/core/v5.rs | 99 +++++ rafs/src/builder/core/v6.rs | 668 ++++++++++++++++++++++++++++++ 5 files changed, 779 insertions(+), 747 deletions(-) create mode 100644 rafs/src/builder/core/v5.rs create mode 100644 rafs/src/builder/core/v6.rs diff --git a/Cargo.lock b/Cargo.lock index bbd8896eedc..3c821356dec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1248,6 +1248,8 @@ dependencies = [ "virtio-bindings", "virtio-queue", "vm-memory", + "vmm-sys-util", + "xattr", ] [[package]] diff --git a/rafs/src/builder/core/mod.rs b/rafs/src/builder/core/mod.rs index b4016a6721b..311625c5fc1 100644 --- a/rafs/src/builder/core/mod.rs +++ b/rafs/src/builder/core/mod.rs @@ -12,3 +12,5 @@ pub(crate) mod node; pub(crate) mod overlay; pub(crate) mod prefetch; pub(crate) mod tree; +pub(crate) mod v5; +pub(crate) mod v6; diff --git a/rafs/src/builder/core/node.rs b/rafs/src/builder/core/node.rs index aacb065efdb..7d863ce7361 100644 --- a/rafs/src/builder/core/node.rs +++ b/rafs/src/builder/core/node.rs @@ -3,12 +3,10 @@ // // SPDX-License-Identifier: Apache-2.0 -use std::collections::BTreeMap; use std::ffi::{OsStr, OsString}; use std::fmt::{self, Display, Formatter, Result as FmtResult}; use std::fs::{self, File}; -use std::io::{Read, SeekFrom, Write}; -use std::mem::size_of; +use std::io::{Read, Write}; use std::ops::Deref; #[cfg(target_os = "linux")] use std::os::linux::fs::MetadataExt; @@ -18,41 +16,22 @@ use std::os::unix::ffi::OsStrExt; use std::path::{Component, Path, PathBuf}; use std::sync::Arc; -use anyhow::{anyhow, bail, ensure, Context, Error, Result}; +use anyhow::{anyhow, Context, Error, Result}; use nydus_storage::device::BlobFeatures; use nydus_storage::meta::{BlobChunkInfoV2Ondisk, BlobMetaChunkInfo}; use nydus_utils::compress; use nydus_utils::digest::{DigestHasher, RafsDigest}; -use nydus_utils::{ - div_round_up, event_tracer, root_tracer, round_down_4k, round_up, try_round_up_4k, ByteSize, -}; +use nydus_utils::{div_round_up, event_tracer, root_tracer, try_round_up_4k, ByteSize}; use sha2::digest::Digest; -use super::chunk_dict::{ChunkDict, DigestWithBlobIndex}; -use super::context::{ArtifactWriter, BlobContext, BlobManager, BootstrapContext, BuildContext}; +use super::chunk_dict::ChunkDict; +use super::context::{ArtifactWriter, BlobContext, BlobManager, BuildContext}; use super::overlay::Overlay; -use super::tree::Tree; use crate::metadata::chunk::ChunkWrapper; -use crate::metadata::inode::{new_v6_inode, InodeWrapper}; -use crate::metadata::layout::v5::RafsV5InodeWrapper; -use crate::metadata::layout::v6::{ - align_offset, calculate_nid, RafsV6Dirent, RafsV6InodeChunkAddr, RafsV6InodeChunkHeader, - RafsV6OndiskInode, EROFS_BLOCK_SIZE, EROFS_INODE_CHUNK_BASED, EROFS_INODE_FLAT_INLINE, - EROFS_INODE_FLAT_PLAIN, -}; +use crate::metadata::inode::InodeWrapper; +use crate::metadata::layout::v6::EROFS_INODE_FLAT_PLAIN; use crate::metadata::layout::RafsXAttrs; -use crate::metadata::{Inode, RafsStore, RafsVersion}; -use crate::RafsIoWrite; - -// Filesystem may have different algorithms to calculate `i_size` for directory entries, -// which may break "repeatable build". To support repeatable build, instead of reuse the value -// provided by the source filesystem, we use our own algorithm to calculate `i_size` for directory -// entries for stable `i_size`. -// -// Rafs v6 already has its own algorithm to calculate `i_size` for directory entries, but we don't -// have directory entries for Rafs v5. So let's generate a pseudo `i_size` for Rafs v5 directory -// inode. -const RAFS_V5_VIRTUAL_ENTRY_SIZE: u64 = 8; +use crate::metadata::{Inode, RafsVersion}; /// Filesystem root path for Unix OSs. pub const ROOT_PATH_NAME: &[u8] = &[b'/']; @@ -773,721 +752,3 @@ impl Node { self.info = Arc::new(info); } } - -// Rafs v5 dedicated methods -impl Node { - /// Dump RAFS v5 inode metadata to meta blob. - pub fn dump_bootstrap_v5( - &self, - ctx: &mut BuildContext, - f_bootstrap: &mut dyn RafsIoWrite, - ) -> Result<()> { - debug!("[{}]\t{}", self.overlay, self); - - if let InodeWrapper::V5(raw_inode) = &self.inode { - // Dump inode info - let name = self.name(); - let inode = RafsV5InodeWrapper { - name, - symlink: self.info.symlink.as_deref(), - inode: raw_inode, - }; - inode - .store(f_bootstrap) - .context("failed to dump inode to bootstrap")?; - - // Dump inode xattr - if !self.info.xattrs.is_empty() { - self.info - .xattrs - .store_v5(f_bootstrap) - .context("failed to dump xattr to bootstrap")?; - ctx.has_xattr = true; - } - - // Dump chunk info - if self.is_reg() && self.inode.child_count() as usize != self.chunks.len() { - bail!("invalid chunk count {}: {}", self.chunks.len(), self); - } - for chunk in &self.chunks { - chunk - .inner - .store(f_bootstrap) - .context("failed to dump chunk info to bootstrap")?; - trace!("\t\tchunk: {} compressor {}", chunk, ctx.compressor,); - } - - Ok(()) - } else { - bail!("dump_bootstrap_v5() encounters non-v5-inode"); - } - } - - // Filesystem may have different algorithms to calculate `i_size` for directory entries, - // which may break "repeatable build". To support repeatable build, instead of reuse the value - // provided by the source filesystem, we use our own algorithm to calculate `i_size` for - // directory entries for stable `i_size`. - // - // Rafs v6 already has its own algorithm to calculate `i_size` for directory entries, but we - // don't have directory entries for Rafs v5. So let's generate a pseudo `i_size` for Rafs v5 - // directory inode. - pub fn v5_set_dir_size(&mut self, fs_version: RafsVersion, children: &[Tree]) { - if !self.is_dir() || !fs_version.is_v5() { - return; - } - - let mut d_size = 0u64; - for child in children.iter() { - d_size += child.node.inode.name_size() as u64 + RAFS_V5_VIRTUAL_ENTRY_SIZE; - } - if d_size == 0 { - self.inode.set_size(4096); - } else { - // Safe to unwrap() because we have u32 for child count. - self.inode.set_size(try_round_up_4k(d_size).unwrap()); - } - self.set_inode_blocks(); - } -} - -// Rafs v6 dedicated methods -impl Node { - /// Dump RAFS v6 inode metadata to meta blob. - pub fn dump_bootstrap_v6( - &mut self, - ctx: &mut BuildContext, - f_bootstrap: &mut dyn RafsIoWrite, - orig_meta_addr: u64, - meta_addr: u64, - chunk_cache: &mut BTreeMap>, - ) -> Result<()> { - let xattr_inline_count = self.info.xattrs.count_v6(); - ensure!( - xattr_inline_count <= u16::MAX as usize, - "size of extended attributes is too big" - ); - let mut inode = new_v6_inode( - &self.inode, - self.v6_datalayout, - xattr_inline_count as u16, - self.v6_compact_inode, - ); - - let meta_offset = meta_addr - orig_meta_addr; - // update all the inodes's offset according to the new 'meta_addr'. - self.v6_offset += meta_offset; - // The EROFS_INODE_FLAT_INLINE layout is valid for directory and symlink only, so - // `dirents_offset` is useful for these two types too, otherwise `dirents_offset` should - // always be zero. Enforce the check to avoid overflow of `dirents_offset`. - if self.is_dir() || self.is_symlink() { - self.v6_dirents_offset += meta_offset; - } - - if self.is_dir() { - self.v6_dump_dir(ctx, f_bootstrap, meta_addr, meta_offset, &mut inode)?; - } else if self.is_reg() { - self.v6_dump_file(ctx, f_bootstrap, chunk_cache, &mut inode)?; - } else if self.is_symlink() { - self.v6_dump_symlink(ctx, f_bootstrap, &mut inode)?; - } else { - f_bootstrap - .seek(SeekFrom::Start(self.v6_offset)) - .context("failed seek for dir inode")?; - inode.store(f_bootstrap).context("failed to store inode")?; - self.v6_store_xattrs(ctx, f_bootstrap)?; - } - - Ok(()) - } - - pub fn v6_dir_d_size(&self, tree: &Tree) -> Result { - ensure!(self.is_dir(), "{} is not a directory", self); - // Use length in byte, instead of length in character. - let mut d_size: u64 = (".".as_bytes().len() - + size_of::() - + "..".as_bytes().len() - + size_of::()) as u64; - - for child in tree.children.iter() { - let len = child.node.name().as_bytes().len() + size_of::(); - // erofs disk format requires dirent to be aligned with 4096. - if (d_size % EROFS_BLOCK_SIZE) + len as u64 > EROFS_BLOCK_SIZE { - d_size = div_round_up(d_size as u64, EROFS_BLOCK_SIZE) * EROFS_BLOCK_SIZE; - } - d_size += len as u64; - } - - Ok(d_size) - } - - /// Set node offset in bootstrap and return the next position. - pub fn v6_set_offset( - &mut self, - bootstrap_ctx: &mut BootstrapContext, - v6_hardlink_offset: Option, - ) { - if self.is_reg() { - if let Some(v6_hardlink_offset) = v6_hardlink_offset { - self.v6_offset = v6_hardlink_offset; - } else { - let size = self.v6_size_with_xattr() as u64; - let unit = size_of::() as u64; - // We first try to allocate space from used blocks. - // If no available used block exists, we allocate sequentially. - let total_size = round_up(size, unit) + self.inode.child_count() as u64 * unit; - self.v6_offset = bootstrap_ctx.allocate_available_block(total_size); - if self.v6_offset == 0 { - self.v6_offset = bootstrap_ctx.offset; - bootstrap_ctx.offset += size; - bootstrap_ctx.align_offset(unit); - bootstrap_ctx.offset += self.inode.child_count() as u64 * unit; - } - } - self.v6_datalayout = EROFS_INODE_CHUNK_BASED; - } else if self.is_symlink() { - self.v6_set_offset_with_tail(bootstrap_ctx, self.inode.size()); - } else { - self.v6_offset = bootstrap_ctx.offset; - bootstrap_ctx.offset += self.v6_size_with_xattr() as u64; - } - } - - pub fn v6_set_dir_offset( - &mut self, - bootstrap_ctx: &mut BootstrapContext, - d_size: u64, - ) -> Result<()> { - ensure!(self.is_dir(), "{} is not a directory", self); - - // Dir isize is the total bytes of 'dirents + names'. - self.inode.set_size(d_size); - self.v6_set_offset_with_tail(bootstrap_ctx, d_size); - - Ok(()) - } - - fn v6_size_with_xattr(&self) -> usize { - self.inode - .get_inode_size_with_xattr(&self.info.xattrs, self.v6_compact_inode) - } - - // For DIR inode, size is the total bytes of 'dirents + names'. - // For symlink, size is the length of symlink name. - fn v6_set_offset_with_tail(&mut self, bootstrap_ctx: &mut BootstrapContext, d_size: u64) { - // TODO: a hashmap of non-full blocks - - // | avail | - // +--------+-----------+----+ +-----------------------+ - // | |inode+tail | free | dirents+names | - // | | | | | | - // +--------+-----------+----+ +-----------------------+ - // - // | avail | - // +--------+-----------+----+ +-----------------------+ +---------+-------------+ - // | |inode | free | dirents+names | | tail | free | - // | | | | | | | | | - // +--------+-----------+----+ +-----------------------+ +---------+-------------+ - // - // - // | avail | - // +--------+-----------+----+ +-----------------------+ +---------+-------------+ - // | | inode + | dirents+names | | tail | free | - // | | | | | | | | - // +--------+-----------+----+ +-----------------------+ +---------+-------------+ - // - // - // | avail | - // +--------+----------------+ +--------------+--------+ +-----------------------+ - // | | inode | | inode+tail | free | | dirents+names | - // | | | | | | | | - // +--------+----------------+ +--------------+--------+ +-----------------------+ - // | inode | - // - // | avail | - // +--------+----------------+ +--------------+--------+ +-----------------------+ +-------+---------------+ - // | | inode | | inode | free | | dirents+names | | tail | free | - // | | | | | | | | | | | - // +--------+----------------+ +--------------+--------+ +-----------------------+ +-------+---------------+ - // | inode | - // - // - let inode_size = self.v6_size_with_xattr() as u64; - let tail: u64 = d_size % EROFS_BLOCK_SIZE; - - // We use a simple inline strategy here: - // If the inode size with xattr + tail data size <= EROFS_BLOCK_SIZE, - // we choose to inline it. - // Firstly, if it's bigger than EROFS_BLOCK_SIZE, - // in most cases, we can assume that the tail data size is close to EROFS_BLOCK_SIZE, - // in this condition, even if we don't inline the tail data, there won't be much waste. - // Secondly, the `available_blocks` that we maintain in the `BootstrapCtx`, - // since it contain only single blocks with some unused space, the available space can only - // be smaller than EROFS_BLOCK_SIZE, therefore we can't use our used blocks to store the - // inode plus the tail data bigger than EROFS_BLOCK_SIZE. - let should_inline = tail != 0 && (inode_size + tail) <= EROFS_BLOCK_SIZE; - - // If should inline, we first try to allocate space for the inode together with tail data - // using used blocks. - // If no available used block exists, we try to allocate space from current block. - // If current block doesn't have enough space, we append it to `available_blocks`, - // and we allocate space from the next block. - // For the remaining data, we allocate space for it sequentially. - self.v6_datalayout = if should_inline { - self.v6_offset = bootstrap_ctx.allocate_available_block(inode_size + tail); - if self.v6_offset == 0 { - let available = EROFS_BLOCK_SIZE - bootstrap_ctx.offset % EROFS_BLOCK_SIZE; - if available < inode_size + tail { - bootstrap_ctx.append_available_block(bootstrap_ctx.offset); - bootstrap_ctx.align_offset(EROFS_BLOCK_SIZE); - } - - self.v6_offset = bootstrap_ctx.offset; - bootstrap_ctx.offset += inode_size + tail; - } - - if d_size != tail { - bootstrap_ctx.append_available_block(bootstrap_ctx.offset); - bootstrap_ctx.align_offset(EROFS_BLOCK_SIZE); - } - self.v6_dirents_offset = bootstrap_ctx.offset; - bootstrap_ctx.offset += round_down_4k(d_size); - - EROFS_INODE_FLAT_INLINE - } else { - // Otherwise, we first try to allocate space for the inode from used blocks. - // If no available used block exists, we allocate space sequentially. - // Then we allocate space for all data. - self.v6_offset = bootstrap_ctx.allocate_available_block(inode_size); - if self.v6_offset == 0 { - self.v6_offset = bootstrap_ctx.offset; - bootstrap_ctx.offset += inode_size; - } - - bootstrap_ctx.append_available_block(bootstrap_ctx.offset); - bootstrap_ctx.align_offset(EROFS_BLOCK_SIZE); - self.v6_dirents_offset = bootstrap_ctx.offset; - bootstrap_ctx.offset += d_size; - bootstrap_ctx.align_offset(EROFS_BLOCK_SIZE); - - EROFS_INODE_FLAT_PLAIN - }; - - trace!( - "{:?} inode offset {} ctx offset {} d_size {} dirents_offset {} datalayout {}", - self.name(), - self.v6_offset, - bootstrap_ctx.offset, - d_size, - self.v6_dirents_offset, - self.v6_datalayout - ); - } - - fn v6_set_inode_compact(&mut self) { - if self.info.v6_force_extended_inode - || self.inode.uid() > u16::MAX as u32 - || self.inode.gid() > u16::MAX as u32 - || self.inode.nlink() > u16::MAX as u32 - || self.inode.size() > u32::MAX as u64 - || self.path().extension() == Some(OsStr::new("pyc")) - { - self.v6_compact_inode = false; - } else { - self.v6_compact_inode = true; - } - } - - fn v6_store_xattrs( - &mut self, - ctx: &mut BuildContext, - f_bootstrap: &mut dyn RafsIoWrite, - ) -> Result<()> { - if !self.info.xattrs.is_empty() { - self.info - .xattrs - .store_v6(f_bootstrap) - .context("failed to dump xattr to bootstrap")?; - ctx.has_xattr = true; - } - Ok(()) - } - - fn v6_dump_dir( - &mut self, - ctx: &mut BuildContext, - f_bootstrap: &mut dyn RafsIoWrite, - meta_addr: u64, - meta_offset: u64, - inode: &mut Box, - ) -> Result<()> { - // the 1st 4k block after dir inode. - let mut dirent_off = self.v6_dirents_offset; - inode.set_u((dirent_off / EROFS_BLOCK_SIZE) as u32); - - // Dump inode - trace!("{:?} dir inode: offset {}", self.target(), self.v6_offset); - f_bootstrap - .seek(SeekFrom::Start(self.v6_offset)) - .context("failed seek for dir inode")?; - inode.store(f_bootstrap).context("failed to store inode")?; - self.v6_store_xattrs(ctx, f_bootstrap)?; - - // Dump dirents - let mut dir_data: Vec = Vec::new(); - let mut entry_names = Vec::new(); - let mut dirents: Vec<(RafsV6Dirent, &OsString)> = Vec::new(); - let mut nameoff: u64 = 0; - let mut used: u64 = 0; - - trace!( - "{:?} self.dirents.len {}", - self.target(), - self.v6_dirents.len() - ); - // fill dir blocks one by one - for (offset, name, file_type) in self.v6_dirents.iter() { - let len = name.len() + size_of::(); - // write to bootstrap when it will exceed EROFS_BLOCK_SIZE - if used + len as u64 > EROFS_BLOCK_SIZE { - for (entry, name) in dirents.iter_mut() { - trace!("{:?} nameoff {}", name, nameoff); - entry.set_name_offset(nameoff as u16); - dir_data.extend(entry.as_ref()); - entry_names.push(*name); - // Use length in byte, instead of length in character. - // Because some characters could occupy more than one byte. - nameoff += name.as_bytes().len() as u64; - } - for name in entry_names.iter() { - dir_data.extend(name.as_bytes()); - } - - f_bootstrap - .seek(SeekFrom::Start(dirent_off as u64)) - .context("failed seek for dir inode")?; - f_bootstrap - .write(dir_data.as_slice()) - .context("failed to store dirents")?; - - dir_data.clear(); - entry_names.clear(); - dirents.clear(); - nameoff = 0; - used = 0; - // track where we're going to write. - dirent_off += EROFS_BLOCK_SIZE; - } - - trace!( - "name {:?} file type {} {:?}", - *name, - *file_type, - RafsV6Dirent::file_type(*file_type) - ); - let entry = RafsV6Dirent::new( - calculate_nid(*offset + meta_offset, meta_addr), - 0, - RafsV6Dirent::file_type(*file_type), - ); - dirents.push((entry, name)); - - nameoff += size_of::() as u64; - used += len as u64; - } - - trace!( - "{:?} used {} dir size {}", - self.target(), - used, - self.inode.size() - ); - // dump tail part if any - if used > 0 { - for (entry, name) in dirents.iter_mut() { - trace!("{:?} tail nameoff {}", name, nameoff); - entry.set_name_offset(nameoff as u16); - dir_data.extend(entry.as_ref()); - entry_names.push(*name); - nameoff += name.len() as u64; - } - for name in entry_names.iter() { - dir_data.extend(name.as_bytes()); - } - - let tail_off = match self.v6_datalayout { - EROFS_INODE_FLAT_INLINE => self.v6_offset + self.v6_size_with_xattr() as u64, - EROFS_INODE_FLAT_PLAIN => dirent_off, - _ => bail!("unsupported RAFS v6 inode layout for directory"), - }; - f_bootstrap - .seek(SeekFrom::Start(tail_off as u64)) - .context("failed seek for dir inode")?; - f_bootstrap - .write(dir_data.as_slice()) - .context("failed to store dirents")?; - } - - Ok(()) - } - - fn v6_dump_file( - &mut self, - ctx: &mut BuildContext, - f_bootstrap: &mut dyn RafsIoWrite, - chunk_cache: &mut BTreeMap>, - inode: &mut Box, - ) -> Result<()> { - let mut is_continuous = true; - let mut prev = None; - - // write chunk indexes, chunk contents has been written to blob file. - let mut chunks: Vec = Vec::new(); - for chunk in self.chunks.iter() { - let uncompressed_offset = chunk.inner.uncompressed_offset(); - let blob_idx = chunk.inner.blob_index(); - let mut v6_chunk = RafsV6InodeChunkAddr::new(); - v6_chunk.set_blob_index(blob_idx); - v6_chunk.set_blob_ci_index(chunk.inner.index()); - v6_chunk.set_block_addr((uncompressed_offset / EROFS_BLOCK_SIZE) as u32); - chunks.extend(v6_chunk.as_ref()); - chunk_cache.insert( - DigestWithBlobIndex(*chunk.inner.id(), chunk.inner.blob_index() + 1), - chunk.inner.clone(), - ); - - if let Some((prev_idx, prev_pos)) = prev { - if prev_pos + ctx.chunk_size as u64 != uncompressed_offset || prev_idx != blob_idx { - is_continuous = false; - } - } - prev = Some((blob_idx, uncompressed_offset)); - } - - // Special optimization to enable page cache sharing for EROFS. - let chunk_size = if is_continuous && inode.size() > ctx.chunk_size as u64 { - inode.size().next_power_of_two() - } else { - ctx.chunk_size as u64 - }; - let info = RafsV6InodeChunkHeader::new(chunk_size); - inode.set_u(info.to_u32()); - - f_bootstrap - .seek(SeekFrom::Start(self.v6_offset)) - .context("failed seek for dir inode")?; - inode.store(f_bootstrap).context("failed to store inode")?; - self.v6_store_xattrs(ctx, f_bootstrap)?; - - // Dump chunk indexes - let unit = size_of::() as u64; - let chunk_off = align_offset(self.v6_offset + self.v6_size_with_xattr() as u64, unit); - f_bootstrap - .seek(SeekFrom::Start(chunk_off)) - .context("failed seek for dir inode")?; - f_bootstrap - .write(chunks.as_slice()) - .context("failed to write chunkindexes")?; - - Ok(()) - } - - fn v6_dump_symlink( - &mut self, - ctx: &mut BuildContext, - f_bootstrap: &mut dyn RafsIoWrite, - inode: &mut Box, - ) -> Result<()> { - let data_off = self.v6_dirents_offset; - // TODO: check whether 'i_u' is used at all in case of inline symlink. - inode.set_u((data_off / EROFS_BLOCK_SIZE) as u32); - - f_bootstrap - .seek(SeekFrom::Start(self.v6_offset)) - .context("failed seek for symlink inode")?; - inode.store(f_bootstrap).context("failed to store inode")?; - self.v6_store_xattrs(ctx, f_bootstrap)?; - - // write symlink. - if let Some(symlink) = &self.info.symlink { - let tail_off = match self.v6_datalayout { - EROFS_INODE_FLAT_INLINE => self.v6_offset + self.v6_size_with_xattr() as u64, - EROFS_INODE_FLAT_PLAIN => data_off, - _ => bail!("unsupported RAFS v5 inode layout for symlink"), - }; - trace!("symlink write_off {}", tail_off); - f_bootstrap - .seek(SeekFrom::Start(tail_off)) - .context("failed seek for dir inode")?; - f_bootstrap - .write(symlink.as_bytes()) - .context("filed to store symlink")?; - } - - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::builder::core::context::{ArtifactStorage, BootstrapContext}; - use crate::metadata::layout::v6::{EROFS_INODE_CHUNK_BASED, EROFS_INODE_SLOT_SIZE}; - use crate::metadata::RAFS_DEFAULT_CHUNK_SIZE; - use std::fs::File; - use vmm_sys_util::{tempdir::TempDir, tempfile::TempFile}; - - #[test] - fn test_set_v6_offset() { - let pa = TempDir::new().unwrap(); - let pa_aa = TempFile::new_in(pa.as_path()).unwrap(); - let mut node = Node::new( - RafsVersion::V6, - pa.as_path().to_path_buf(), - pa_aa.as_path().to_path_buf(), - Overlay::UpperAddition, - RAFS_DEFAULT_CHUNK_SIZE as u32, - false, - false, - ) - .unwrap(); - - let bootstrap_path = TempFile::new().unwrap(); - let storage = ArtifactStorage::SingleFile(bootstrap_path.as_path().to_path_buf()); - let mut bootstrap_ctx = BootstrapContext::new(Some(storage), false).unwrap(); - bootstrap_ctx.offset = 0; - - // reg file. - // "1" is used only for testing purpose, in practice - // it's always aligned to 32 bytes. - node.v6_set_offset(&mut bootstrap_ctx, None); - assert_eq!(node.v6_offset, 0); - assert_eq!(node.v6_datalayout, EROFS_INODE_CHUNK_BASED); - assert!(node.v6_compact_inode); - assert_eq!(bootstrap_ctx.offset, 32); - - // symlink and dir are handled in the same way. - let mut dir_node = Node::new( - RafsVersion::V6, - pa.as_path().to_path_buf(), - pa.as_path().to_path_buf(), - Overlay::UpperAddition, - RAFS_DEFAULT_CHUNK_SIZE as u32, - false, - false, - ) - .unwrap(); - - dir_node - .v6_set_dir_offset(&mut bootstrap_ctx, 4064) - .unwrap(); - assert_eq!(dir_node.v6_datalayout, EROFS_INODE_FLAT_INLINE); - assert_eq!(dir_node.v6_offset, 4096); - assert_eq!(bootstrap_ctx.offset, 8192); - - dir_node - .v6_set_dir_offset(&mut bootstrap_ctx, 4096) - .unwrap(); - assert_eq!(dir_node.v6_datalayout, EROFS_INODE_FLAT_PLAIN); - assert_eq!(dir_node.v6_offset, 32); - assert_eq!(dir_node.v6_dirents_offset, 8192); - assert_eq!(bootstrap_ctx.offset, 8192 + 4096); - - dir_node - .v6_set_dir_offset(&mut bootstrap_ctx, 8160) - .unwrap(); - assert_eq!(dir_node.v6_datalayout, EROFS_INODE_FLAT_INLINE); - assert_eq!(dir_node.v6_offset, 8192 + 4096); - assert_eq!(dir_node.v6_dirents_offset, 8192 + 4096 + 4096); - assert_eq!(bootstrap_ctx.offset, 8192 + 4096 + 8192); - - dir_node - .v6_set_dir_offset(&mut bootstrap_ctx, 8161) - .unwrap(); - assert_eq!(dir_node.v6_datalayout, EROFS_INODE_FLAT_PLAIN); - assert_eq!(dir_node.v6_offset, 64); - assert_eq!(dir_node.v6_dirents_offset, 8192 + 4096 + 8192); - assert_eq!(bootstrap_ctx.offset, 8192 + 4096 + 8192 + 8192); - - dir_node - .v6_set_dir_offset(&mut bootstrap_ctx, 4096 + 3968) - .unwrap(); - assert_eq!(dir_node.v6_datalayout, EROFS_INODE_FLAT_INLINE); - assert_eq!(dir_node.v6_offset, 96); - assert_eq!(dir_node.v6_dirents_offset, 8192 + 4096 + 8192 + 8192); - assert_eq!(bootstrap_ctx.offset, 8192 + 4096 + 8192 + 8192 + 4096); - - dir_node - .v6_set_dir_offset(&mut bootstrap_ctx, 4096 + 2048) - .unwrap(); - assert_eq!(dir_node.v6_datalayout, EROFS_INODE_FLAT_INLINE); - assert_eq!(dir_node.v6_offset, 8192 + 4096 + 8192 + 8192 + 4096); - assert_eq!( - dir_node.v6_dirents_offset, - 8192 + 4096 + 8192 + 8192 + 4096 + 4096 - ); - assert_eq!( - bootstrap_ctx.offset, - 8192 + 4096 + 8192 + 8192 + 4096 + 8192 - ); - - dir_node - .v6_set_dir_offset(&mut bootstrap_ctx, 1985) - .unwrap(); - assert_eq!(dir_node.v6_datalayout, EROFS_INODE_FLAT_INLINE); - assert_eq!(dir_node.v6_offset, 8192 + 4096 + 8192 + 8192 + 4096 + 8192); - assert_eq!( - bootstrap_ctx.offset, - 8192 + 4096 + 8192 + 8192 + 4096 + 8192 + 32 + 1985 - ); - - bootstrap_ctx.align_offset(EROFS_INODE_SLOT_SIZE as u64); - dir_node - .v6_set_dir_offset(&mut bootstrap_ctx, 1984) - .unwrap(); - assert_eq!(dir_node.v6_datalayout, EROFS_INODE_FLAT_INLINE); - assert_eq!( - dir_node.v6_offset, - 8192 + 4096 + 8192 + 8192 + 4096 + 2048 + 32 - ); - assert_eq!( - bootstrap_ctx.offset, - 8192 + 4096 + 8192 + 8192 + 4096 + 8192 + round_up(32 + 1985, 32) - ); - } - - #[test] - fn test_set_v6_inode_compact() { - let pa = TempDir::new().unwrap(); - let pa_reg = TempFile::new_in(pa.as_path()).unwrap(); - let pa_pyc = pa.as_path().join("foo.pyc"); - let _ = File::create(&pa_pyc).unwrap(); - - let reg_node = Node::new( - RafsVersion::V6, - pa.as_path().to_path_buf(), - pa_reg.as_path().to_path_buf(), - Overlay::UpperAddition, - RAFS_DEFAULT_CHUNK_SIZE as u32, - false, - false, - ) - .unwrap(); - - assert!(reg_node.v6_compact_inode); - - let pyc_node = Node::new( - RafsVersion::V6, - pa.as_path().to_path_buf(), - pa_pyc.as_path().to_path_buf(), - Overlay::UpperAddition, - RAFS_DEFAULT_CHUNK_SIZE as u32, - false, - false, - ) - .unwrap(); - - assert!(!pyc_node.v6_compact_inode); - - std::fs::remove_file(&pa_pyc).unwrap(); - } -} diff --git a/rafs/src/builder/core/v5.rs b/rafs/src/builder/core/v5.rs new file mode 100644 index 00000000000..a303fc56f51 --- /dev/null +++ b/rafs/src/builder/core/v5.rs @@ -0,0 +1,99 @@ +// Copyright 2020 Ant Group. All rights reserved. +// Copyright (C) 2021 Alibaba Cloud. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +use anyhow::{bail, Context, Result}; +use nydus_utils::try_round_up_4k; + +use super::node::Node; +use crate::builder::{BuildContext, Tree}; +use crate::metadata::inode::InodeWrapper; +use crate::metadata::layout::v5::RafsV5InodeWrapper; +use crate::metadata::{RafsStore, RafsVersion}; +use crate::RafsIoWrite; + +// Filesystem may have different algorithms to calculate `i_size` for directory entries, +// which may break "repeatable build". To support repeatable build, instead of reuse the value +// provided by the source filesystem, we use our own algorithm to calculate `i_size` for directory +// entries for stable `i_size`. +// +// Rafs v6 already has its own algorithm to calculate `i_size` for directory entries, but we don't +// have directory entries for Rafs v5. So let's generate a pseudo `i_size` for Rafs v5 directory +// inode. +const RAFS_V5_VIRTUAL_ENTRY_SIZE: u64 = 8; + +impl Node { + /// Dump RAFS v5 inode metadata to meta blob. + pub fn dump_bootstrap_v5( + &self, + ctx: &mut BuildContext, + f_bootstrap: &mut dyn RafsIoWrite, + ) -> Result<()> { + debug!("[{}]\t{}", self.overlay, self); + + if let InodeWrapper::V5(raw_inode) = &self.inode { + // Dump inode info + let name = self.name(); + let inode = RafsV5InodeWrapper { + name, + symlink: self.info.symlink.as_deref(), + inode: raw_inode, + }; + inode + .store(f_bootstrap) + .context("failed to dump inode to bootstrap")?; + + // Dump inode xattr + if !self.info.xattrs.is_empty() { + self.info + .xattrs + .store_v5(f_bootstrap) + .context("failed to dump xattr to bootstrap")?; + ctx.has_xattr = true; + } + + // Dump chunk info + if self.is_reg() && self.inode.child_count() as usize != self.chunks.len() { + bail!("invalid chunk count {}: {}", self.chunks.len(), self); + } + for chunk in &self.chunks { + chunk + .inner + .store(f_bootstrap) + .context("failed to dump chunk info to bootstrap")?; + trace!("\t\tchunk: {} compressor {}", chunk, ctx.compressor,); + } + + Ok(()) + } else { + bail!("dump_bootstrap_v5() encounters non-v5-inode"); + } + } + + // Filesystem may have different algorithms to calculate `i_size` for directory entries, + // which may break "repeatable build". To support repeatable build, instead of reuse the value + // provided by the source filesystem, we use our own algorithm to calculate `i_size` for + // directory entries for stable `i_size`. + // + // Rafs v6 already has its own algorithm to calculate `i_size` for directory entries, but we + // don't have directory entries for Rafs v5. So let's generate a pseudo `i_size` for Rafs v5 + // directory inode. + pub fn v5_set_dir_size(&mut self, fs_version: RafsVersion, children: &[Tree]) { + if !self.is_dir() || !fs_version.is_v5() { + return; + } + + let mut d_size = 0u64; + for child in children.iter() { + d_size += child.node.inode.name_size() as u64 + RAFS_V5_VIRTUAL_ENTRY_SIZE; + } + if d_size == 0 { + self.inode.set_size(4096); + } else { + // Safe to unwrap() because we have u32 for child count. + self.inode.set_size(try_round_up_4k(d_size).unwrap()); + } + self.set_inode_blocks(); + } +} diff --git a/rafs/src/builder/core/v6.rs b/rafs/src/builder/core/v6.rs new file mode 100644 index 00000000000..19f89234427 --- /dev/null +++ b/rafs/src/builder/core/v6.rs @@ -0,0 +1,668 @@ +// Copyright 2020 Ant Group. All rights reserved. +// Copyright (C) 2021 Alibaba Cloud. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::BTreeMap; +use std::ffi::{OsStr, OsString}; +use std::io::SeekFrom; +use std::mem::size_of; +use std::os::unix::ffi::OsStrExt; +use std::sync::Arc; + +use anyhow::{bail, ensure, Context, Result}; +use nydus_utils::{div_round_up, round_down_4k, round_up}; + +use super::chunk_dict::DigestWithBlobIndex; +use super::node::Node; +use crate::builder::{BootstrapContext, BuildContext, Tree}; +use crate::metadata::chunk::ChunkWrapper; +use crate::metadata::inode::new_v6_inode; +use crate::metadata::layout::v6::{ + align_offset, calculate_nid, RafsV6Dirent, RafsV6InodeChunkAddr, RafsV6InodeChunkHeader, + RafsV6OndiskInode, EROFS_BLOCK_SIZE, EROFS_INODE_CHUNK_BASED, EROFS_INODE_FLAT_INLINE, + EROFS_INODE_FLAT_PLAIN, +}; +use crate::RafsIoWrite; + +// Rafs v6 dedicated methods +impl Node { + /// Dump RAFS v6 inode metadata to meta blob. + pub fn dump_bootstrap_v6( + &mut self, + ctx: &mut BuildContext, + f_bootstrap: &mut dyn RafsIoWrite, + orig_meta_addr: u64, + meta_addr: u64, + chunk_cache: &mut BTreeMap>, + ) -> Result<()> { + let xattr_inline_count = self.info.xattrs.count_v6(); + ensure!( + xattr_inline_count <= u16::MAX as usize, + "size of extended attributes is too big" + ); + let mut inode = new_v6_inode( + &self.inode, + self.v6_datalayout, + xattr_inline_count as u16, + self.v6_compact_inode, + ); + + let meta_offset = meta_addr - orig_meta_addr; + // update all the inodes's offset according to the new 'meta_addr'. + self.v6_offset += meta_offset; + // The EROFS_INODE_FLAT_INLINE layout is valid for directory and symlink only, so + // `dirents_offset` is useful for these two types too, otherwise `dirents_offset` should + // always be zero. Enforce the check to avoid overflow of `dirents_offset`. + if self.is_dir() || self.is_symlink() { + self.v6_dirents_offset += meta_offset; + } + + if self.is_dir() { + self.v6_dump_dir(ctx, f_bootstrap, meta_addr, meta_offset, &mut inode)?; + } else if self.is_reg() { + self.v6_dump_file(ctx, f_bootstrap, chunk_cache, &mut inode)?; + } else if self.is_symlink() { + self.v6_dump_symlink(ctx, f_bootstrap, &mut inode)?; + } else { + f_bootstrap + .seek(SeekFrom::Start(self.v6_offset)) + .context("failed seek for dir inode")?; + inode.store(f_bootstrap).context("failed to store inode")?; + self.v6_store_xattrs(ctx, f_bootstrap)?; + } + + Ok(()) + } + + pub fn v6_dir_d_size(&self, tree: &Tree) -> Result { + ensure!(self.is_dir(), "{} is not a directory", self); + // Use length in byte, instead of length in character. + let mut d_size: u64 = (".".as_bytes().len() + + size_of::() + + "..".as_bytes().len() + + size_of::()) as u64; + + for child in tree.children.iter() { + let len = child.node.name().as_bytes().len() + size_of::(); + // erofs disk format requires dirent to be aligned with 4096. + if (d_size % EROFS_BLOCK_SIZE) + len as u64 > EROFS_BLOCK_SIZE { + d_size = div_round_up(d_size as u64, EROFS_BLOCK_SIZE) * EROFS_BLOCK_SIZE; + } + d_size += len as u64; + } + + Ok(d_size) + } + + /// Set node offset in bootstrap and return the next position. + pub fn v6_set_offset( + &mut self, + bootstrap_ctx: &mut BootstrapContext, + v6_hardlink_offset: Option, + ) { + if self.is_reg() { + if let Some(v6_hardlink_offset) = v6_hardlink_offset { + self.v6_offset = v6_hardlink_offset; + } else { + let size = self.v6_size_with_xattr() as u64; + let unit = size_of::() as u64; + // We first try to allocate space from used blocks. + // If no available used block exists, we allocate sequentially. + let total_size = round_up(size, unit) + self.inode.child_count() as u64 * unit; + self.v6_offset = bootstrap_ctx.allocate_available_block(total_size); + if self.v6_offset == 0 { + self.v6_offset = bootstrap_ctx.offset; + bootstrap_ctx.offset += size; + bootstrap_ctx.align_offset(unit); + bootstrap_ctx.offset += self.inode.child_count() as u64 * unit; + } + } + self.v6_datalayout = EROFS_INODE_CHUNK_BASED; + } else if self.is_symlink() { + self.v6_set_offset_with_tail(bootstrap_ctx, self.inode.size()); + } else { + self.v6_offset = bootstrap_ctx.offset; + bootstrap_ctx.offset += self.v6_size_with_xattr() as u64; + } + } + + pub fn v6_set_dir_offset( + &mut self, + bootstrap_ctx: &mut BootstrapContext, + d_size: u64, + ) -> Result<()> { + ensure!(self.is_dir(), "{} is not a directory", self); + + // Dir isize is the total bytes of 'dirents + names'. + self.inode.set_size(d_size); + self.v6_set_offset_with_tail(bootstrap_ctx, d_size); + + Ok(()) + } + + fn v6_size_with_xattr(&self) -> usize { + self.inode + .get_inode_size_with_xattr(&self.info.xattrs, self.v6_compact_inode) + } + + // For DIR inode, size is the total bytes of 'dirents + names'. + // For symlink, size is the length of symlink name. + fn v6_set_offset_with_tail(&mut self, bootstrap_ctx: &mut BootstrapContext, d_size: u64) { + // TODO: a hashmap of non-full blocks + + // | avail | + // +--------+-----------+----+ +-----------------------+ + // | |inode+tail | free | dirents+names | + // | | | | | | + // +--------+-----------+----+ +-----------------------+ + // + // | avail | + // +--------+-----------+----+ +-----------------------+ +---------+-------------+ + // | |inode | free | dirents+names | | tail | free | + // | | | | | | | | | + // +--------+-----------+----+ +-----------------------+ +---------+-------------+ + // + // + // | avail | + // +--------+-----------+----+ +-----------------------+ +---------+-------------+ + // | | inode + | dirents+names | | tail | free | + // | | | | | | | | + // +--------+-----------+----+ +-----------------------+ +---------+-------------+ + // + // + // | avail | + // +--------+----------------+ +--------------+--------+ +-----------------------+ + // | | inode | | inode+tail | free | | dirents+names | + // | | | | | | | | + // +--------+----------------+ +--------------+--------+ +-----------------------+ + // | inode | + // + // | avail | + // +--------+----------------+ +--------------+--------+ +-----------------------+ +-------+---------------+ + // | | inode | | inode | free | | dirents+names | | tail | free | + // | | | | | | | | | | | + // +--------+----------------+ +--------------+--------+ +-----------------------+ +-------+---------------+ + // | inode | + // + // + let inode_size = self.v6_size_with_xattr() as u64; + let tail: u64 = d_size % EROFS_BLOCK_SIZE; + + // We use a simple inline strategy here: + // If the inode size with xattr + tail data size <= EROFS_BLOCK_SIZE, + // we choose to inline it. + // Firstly, if it's bigger than EROFS_BLOCK_SIZE, + // in most cases, we can assume that the tail data size is close to EROFS_BLOCK_SIZE, + // in this condition, even if we don't inline the tail data, there won't be much waste. + // Secondly, the `available_blocks` that we maintain in the `BootstrapCtx`, + // since it contain only single blocks with some unused space, the available space can only + // be smaller than EROFS_BLOCK_SIZE, therefore we can't use our used blocks to store the + // inode plus the tail data bigger than EROFS_BLOCK_SIZE. + let should_inline = tail != 0 && (inode_size + tail) <= EROFS_BLOCK_SIZE; + + // If should inline, we first try to allocate space for the inode together with tail data + // using used blocks. + // If no available used block exists, we try to allocate space from current block. + // If current block doesn't have enough space, we append it to `available_blocks`, + // and we allocate space from the next block. + // For the remaining data, we allocate space for it sequentially. + self.v6_datalayout = if should_inline { + self.v6_offset = bootstrap_ctx.allocate_available_block(inode_size + tail); + if self.v6_offset == 0 { + let available = EROFS_BLOCK_SIZE - bootstrap_ctx.offset % EROFS_BLOCK_SIZE; + if available < inode_size + tail { + bootstrap_ctx.append_available_block(bootstrap_ctx.offset); + bootstrap_ctx.align_offset(EROFS_BLOCK_SIZE); + } + + self.v6_offset = bootstrap_ctx.offset; + bootstrap_ctx.offset += inode_size + tail; + } + + if d_size != tail { + bootstrap_ctx.append_available_block(bootstrap_ctx.offset); + bootstrap_ctx.align_offset(EROFS_BLOCK_SIZE); + } + self.v6_dirents_offset = bootstrap_ctx.offset; + bootstrap_ctx.offset += round_down_4k(d_size); + + EROFS_INODE_FLAT_INLINE + } else { + // Otherwise, we first try to allocate space for the inode from used blocks. + // If no available used block exists, we allocate space sequentially. + // Then we allocate space for all data. + self.v6_offset = bootstrap_ctx.allocate_available_block(inode_size); + if self.v6_offset == 0 { + self.v6_offset = bootstrap_ctx.offset; + bootstrap_ctx.offset += inode_size; + } + + bootstrap_ctx.append_available_block(bootstrap_ctx.offset); + bootstrap_ctx.align_offset(EROFS_BLOCK_SIZE); + self.v6_dirents_offset = bootstrap_ctx.offset; + bootstrap_ctx.offset += d_size; + bootstrap_ctx.align_offset(EROFS_BLOCK_SIZE); + + EROFS_INODE_FLAT_PLAIN + }; + + trace!( + "{:?} inode offset {} ctx offset {} d_size {} dirents_offset {} datalayout {}", + self.name(), + self.v6_offset, + bootstrap_ctx.offset, + d_size, + self.v6_dirents_offset, + self.v6_datalayout + ); + } + + pub fn v6_set_inode_compact(&mut self) { + if self.info.v6_force_extended_inode + || self.inode.uid() > u16::MAX as u32 + || self.inode.gid() > u16::MAX as u32 + || self.inode.nlink() > u16::MAX as u32 + || self.inode.size() > u32::MAX as u64 + || self.path().extension() == Some(OsStr::new("pyc")) + { + self.v6_compact_inode = false; + } else { + self.v6_compact_inode = true; + } + } + + fn v6_store_xattrs( + &mut self, + ctx: &mut BuildContext, + f_bootstrap: &mut dyn RafsIoWrite, + ) -> Result<()> { + if !self.info.xattrs.is_empty() { + self.info + .xattrs + .store_v6(f_bootstrap) + .context("failed to dump xattr to bootstrap")?; + ctx.has_xattr = true; + } + Ok(()) + } + + fn v6_dump_dir( + &mut self, + ctx: &mut BuildContext, + f_bootstrap: &mut dyn RafsIoWrite, + meta_addr: u64, + meta_offset: u64, + inode: &mut Box, + ) -> Result<()> { + // the 1st 4k block after dir inode. + let mut dirent_off = self.v6_dirents_offset; + inode.set_u((dirent_off / EROFS_BLOCK_SIZE) as u32); + + // Dump inode + trace!("{:?} dir inode: offset {}", self.target(), self.v6_offset); + f_bootstrap + .seek(SeekFrom::Start(self.v6_offset)) + .context("failed seek for dir inode")?; + inode.store(f_bootstrap).context("failed to store inode")?; + self.v6_store_xattrs(ctx, f_bootstrap)?; + + // Dump dirents + let mut dir_data: Vec = Vec::new(); + let mut entry_names = Vec::new(); + let mut dirents: Vec<(RafsV6Dirent, &OsString)> = Vec::new(); + let mut nameoff: u64 = 0; + let mut used: u64 = 0; + + trace!( + "{:?} self.dirents.len {}", + self.target(), + self.v6_dirents.len() + ); + // fill dir blocks one by one + for (offset, name, file_type) in self.v6_dirents.iter() { + let len = name.len() + size_of::(); + // write to bootstrap when it will exceed EROFS_BLOCK_SIZE + if used + len as u64 > EROFS_BLOCK_SIZE { + for (entry, name) in dirents.iter_mut() { + trace!("{:?} nameoff {}", name, nameoff); + entry.set_name_offset(nameoff as u16); + dir_data.extend(entry.as_ref()); + entry_names.push(*name); + // Use length in byte, instead of length in character. + // Because some characters could occupy more than one byte. + nameoff += name.as_bytes().len() as u64; + } + for name in entry_names.iter() { + dir_data.extend(name.as_bytes()); + } + + f_bootstrap + .seek(SeekFrom::Start(dirent_off as u64)) + .context("failed seek for dir inode")?; + f_bootstrap + .write(dir_data.as_slice()) + .context("failed to store dirents")?; + + dir_data.clear(); + entry_names.clear(); + dirents.clear(); + nameoff = 0; + used = 0; + // track where we're going to write. + dirent_off += EROFS_BLOCK_SIZE; + } + + trace!( + "name {:?} file type {} {:?}", + *name, + *file_type, + RafsV6Dirent::file_type(*file_type) + ); + let entry = RafsV6Dirent::new( + calculate_nid(*offset + meta_offset, meta_addr), + 0, + RafsV6Dirent::file_type(*file_type), + ); + dirents.push((entry, name)); + + nameoff += size_of::() as u64; + used += len as u64; + } + + trace!( + "{:?} used {} dir size {}", + self.target(), + used, + self.inode.size() + ); + // dump tail part if any + if used > 0 { + for (entry, name) in dirents.iter_mut() { + trace!("{:?} tail nameoff {}", name, nameoff); + entry.set_name_offset(nameoff as u16); + dir_data.extend(entry.as_ref()); + entry_names.push(*name); + nameoff += name.len() as u64; + } + for name in entry_names.iter() { + dir_data.extend(name.as_bytes()); + } + + let tail_off = match self.v6_datalayout { + EROFS_INODE_FLAT_INLINE => self.v6_offset + self.v6_size_with_xattr() as u64, + EROFS_INODE_FLAT_PLAIN => dirent_off, + _ => bail!("unsupported RAFS v6 inode layout for directory"), + }; + f_bootstrap + .seek(SeekFrom::Start(tail_off as u64)) + .context("failed seek for dir inode")?; + f_bootstrap + .write(dir_data.as_slice()) + .context("failed to store dirents")?; + } + + Ok(()) + } + + fn v6_dump_file( + &mut self, + ctx: &mut BuildContext, + f_bootstrap: &mut dyn RafsIoWrite, + chunk_cache: &mut BTreeMap>, + inode: &mut Box, + ) -> Result<()> { + let mut is_continuous = true; + let mut prev = None; + + // write chunk indexes, chunk contents has been written to blob file. + let mut chunks: Vec = Vec::new(); + for chunk in self.chunks.iter() { + let uncompressed_offset = chunk.inner.uncompressed_offset(); + let blob_idx = chunk.inner.blob_index(); + let mut v6_chunk = RafsV6InodeChunkAddr::new(); + v6_chunk.set_blob_index(blob_idx); + v6_chunk.set_blob_ci_index(chunk.inner.index()); + v6_chunk.set_block_addr((uncompressed_offset / EROFS_BLOCK_SIZE) as u32); + chunks.extend(v6_chunk.as_ref()); + chunk_cache.insert( + DigestWithBlobIndex(*chunk.inner.id(), chunk.inner.blob_index() + 1), + chunk.inner.clone(), + ); + + if let Some((prev_idx, prev_pos)) = prev { + if prev_pos + ctx.chunk_size as u64 != uncompressed_offset || prev_idx != blob_idx { + is_continuous = false; + } + } + prev = Some((blob_idx, uncompressed_offset)); + } + + // Special optimization to enable page cache sharing for EROFS. + let chunk_size = if is_continuous && inode.size() > ctx.chunk_size as u64 { + inode.size().next_power_of_two() + } else { + ctx.chunk_size as u64 + }; + let info = RafsV6InodeChunkHeader::new(chunk_size); + inode.set_u(info.to_u32()); + + f_bootstrap + .seek(SeekFrom::Start(self.v6_offset)) + .context("failed seek for dir inode")?; + inode.store(f_bootstrap).context("failed to store inode")?; + self.v6_store_xattrs(ctx, f_bootstrap)?; + + // Dump chunk indexes + let unit = size_of::() as u64; + let chunk_off = align_offset(self.v6_offset + self.v6_size_with_xattr() as u64, unit); + f_bootstrap + .seek(SeekFrom::Start(chunk_off)) + .context("failed seek for dir inode")?; + f_bootstrap + .write(chunks.as_slice()) + .context("failed to write chunkindexes")?; + + Ok(()) + } + + fn v6_dump_symlink( + &mut self, + ctx: &mut BuildContext, + f_bootstrap: &mut dyn RafsIoWrite, + inode: &mut Box, + ) -> Result<()> { + let data_off = self.v6_dirents_offset; + // TODO: check whether 'i_u' is used at all in case of inline symlink. + inode.set_u((data_off / EROFS_BLOCK_SIZE) as u32); + + f_bootstrap + .seek(SeekFrom::Start(self.v6_offset)) + .context("failed seek for symlink inode")?; + inode.store(f_bootstrap).context("failed to store inode")?; + self.v6_store_xattrs(ctx, f_bootstrap)?; + + // write symlink. + if let Some(symlink) = &self.info.symlink { + let tail_off = match self.v6_datalayout { + EROFS_INODE_FLAT_INLINE => self.v6_offset + self.v6_size_with_xattr() as u64, + EROFS_INODE_FLAT_PLAIN => data_off, + _ => bail!("unsupported RAFS v5 inode layout for symlink"), + }; + trace!("symlink write_off {}", tail_off); + f_bootstrap + .seek(SeekFrom::Start(tail_off)) + .context("failed seek for dir inode")?; + f_bootstrap + .write(symlink.as_bytes()) + .context("filed to store symlink")?; + } + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::builder::{ArtifactStorage, BootstrapContext, Overlay}; + use crate::metadata::layout::v6::{EROFS_INODE_CHUNK_BASED, EROFS_INODE_SLOT_SIZE}; + use crate::metadata::{RafsVersion, RAFS_DEFAULT_CHUNK_SIZE}; + use std::fs::File; + use vmm_sys_util::{tempdir::TempDir, tempfile::TempFile}; + + #[test] + fn test_set_v6_offset() { + let pa = TempDir::new().unwrap(); + let pa_aa = TempFile::new_in(pa.as_path()).unwrap(); + let mut node = Node::new( + RafsVersion::V6, + pa.as_path().to_path_buf(), + pa_aa.as_path().to_path_buf(), + Overlay::UpperAddition, + RAFS_DEFAULT_CHUNK_SIZE as u32, + false, + false, + ) + .unwrap(); + + let bootstrap_path = TempFile::new().unwrap(); + let storage = ArtifactStorage::SingleFile(bootstrap_path.as_path().to_path_buf()); + let mut bootstrap_ctx = BootstrapContext::new(Some(storage), false).unwrap(); + bootstrap_ctx.offset = 0; + + // reg file. + // "1" is used only for testing purpose, in practice + // it's always aligned to 32 bytes. + node.v6_set_offset(&mut bootstrap_ctx, None); + assert_eq!(node.v6_offset, 0); + assert_eq!(node.v6_datalayout, EROFS_INODE_CHUNK_BASED); + assert!(node.v6_compact_inode); + assert_eq!(bootstrap_ctx.offset, 32); + + // symlink and dir are handled in the same way. + let mut dir_node = Node::new( + RafsVersion::V6, + pa.as_path().to_path_buf(), + pa.as_path().to_path_buf(), + Overlay::UpperAddition, + RAFS_DEFAULT_CHUNK_SIZE as u32, + false, + false, + ) + .unwrap(); + + dir_node + .v6_set_dir_offset(&mut bootstrap_ctx, 4064) + .unwrap(); + assert_eq!(dir_node.v6_datalayout, EROFS_INODE_FLAT_INLINE); + assert_eq!(dir_node.v6_offset, 4096); + assert_eq!(bootstrap_ctx.offset, 8192); + + dir_node + .v6_set_dir_offset(&mut bootstrap_ctx, 4096) + .unwrap(); + assert_eq!(dir_node.v6_datalayout, EROFS_INODE_FLAT_PLAIN); + assert_eq!(dir_node.v6_offset, 32); + assert_eq!(dir_node.v6_dirents_offset, 8192); + assert_eq!(bootstrap_ctx.offset, 8192 + 4096); + + dir_node + .v6_set_dir_offset(&mut bootstrap_ctx, 8160) + .unwrap(); + assert_eq!(dir_node.v6_datalayout, EROFS_INODE_FLAT_INLINE); + assert_eq!(dir_node.v6_offset, 8192 + 4096); + assert_eq!(dir_node.v6_dirents_offset, 8192 + 4096 + 4096); + assert_eq!(bootstrap_ctx.offset, 8192 + 4096 + 8192); + + dir_node + .v6_set_dir_offset(&mut bootstrap_ctx, 8161) + .unwrap(); + assert_eq!(dir_node.v6_datalayout, EROFS_INODE_FLAT_PLAIN); + assert_eq!(dir_node.v6_offset, 64); + assert_eq!(dir_node.v6_dirents_offset, 8192 + 4096 + 8192); + assert_eq!(bootstrap_ctx.offset, 8192 + 4096 + 8192 + 8192); + + dir_node + .v6_set_dir_offset(&mut bootstrap_ctx, 4096 + 3968) + .unwrap(); + assert_eq!(dir_node.v6_datalayout, EROFS_INODE_FLAT_INLINE); + assert_eq!(dir_node.v6_offset, 96); + assert_eq!(dir_node.v6_dirents_offset, 8192 + 4096 + 8192 + 8192); + assert_eq!(bootstrap_ctx.offset, 8192 + 4096 + 8192 + 8192 + 4096); + + dir_node + .v6_set_dir_offset(&mut bootstrap_ctx, 4096 + 2048) + .unwrap(); + assert_eq!(dir_node.v6_datalayout, EROFS_INODE_FLAT_INLINE); + assert_eq!(dir_node.v6_offset, 8192 + 4096 + 8192 + 8192 + 4096); + assert_eq!( + dir_node.v6_dirents_offset, + 8192 + 4096 + 8192 + 8192 + 4096 + 4096 + ); + assert_eq!( + bootstrap_ctx.offset, + 8192 + 4096 + 8192 + 8192 + 4096 + 8192 + ); + + dir_node + .v6_set_dir_offset(&mut bootstrap_ctx, 1985) + .unwrap(); + assert_eq!(dir_node.v6_datalayout, EROFS_INODE_FLAT_INLINE); + assert_eq!(dir_node.v6_offset, 8192 + 4096 + 8192 + 8192 + 4096 + 8192); + assert_eq!( + bootstrap_ctx.offset, + 8192 + 4096 + 8192 + 8192 + 4096 + 8192 + 32 + 1985 + ); + + bootstrap_ctx.align_offset(EROFS_INODE_SLOT_SIZE as u64); + dir_node + .v6_set_dir_offset(&mut bootstrap_ctx, 1984) + .unwrap(); + assert_eq!(dir_node.v6_datalayout, EROFS_INODE_FLAT_INLINE); + assert_eq!( + dir_node.v6_offset, + 8192 + 4096 + 8192 + 8192 + 4096 + 2048 + 32 + ); + assert_eq!( + bootstrap_ctx.offset, + 8192 + 4096 + 8192 + 8192 + 4096 + 8192 + round_up(32 + 1985, 32) + ); + } + + #[test] + fn test_set_v6_inode_compact() { + let pa = TempDir::new().unwrap(); + let pa_reg = TempFile::new_in(pa.as_path()).unwrap(); + let pa_pyc = pa.as_path().join("foo.pyc"); + let _ = File::create(&pa_pyc).unwrap(); + + let reg_node = Node::new( + RafsVersion::V6, + pa.as_path().to_path_buf(), + pa_reg.as_path().to_path_buf(), + Overlay::UpperAddition, + RAFS_DEFAULT_CHUNK_SIZE as u32, + false, + false, + ) + .unwrap(); + + assert!(reg_node.v6_compact_inode); + + let pyc_node = Node::new( + RafsVersion::V6, + pa.as_path().to_path_buf(), + pa_pyc.as_path().to_path_buf(), + Overlay::UpperAddition, + RAFS_DEFAULT_CHUNK_SIZE as u32, + false, + false, + ) + .unwrap(); + + assert!(!pyc_node.v6_compact_inode); + + std::fs::remove_file(&pa_pyc).unwrap(); + } +} From ab344d69c14c69ffa7f936c9cd634c7b309d3f8c Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Sun, 5 Mar 2023 17:47:15 +0800 Subject: [PATCH 07/10] rafs: refine builder/Node related code Refine builder/Node related code for maintenance. Signed-off-by: Jiang Liu --- rafs/src/builder/core/bootstrap.rs | 2 +- rafs/src/builder/core/node.rs | 361 +++++++++++++++-------------- rafs/src/builder/core/v6.rs | 86 +++---- rafs/src/builder/directory.rs | 4 +- src/bin/nydus-image/stat.rs | 5 +- 5 files changed, 243 insertions(+), 215 deletions(-) diff --git a/rafs/src/builder/core/bootstrap.rs b/rafs/src/builder/core/bootstrap.rs index cad348889a8..48eb248b445 100644 --- a/rafs/src/builder/core/bootstrap.rs +++ b/rafs/src/builder/core/bootstrap.rs @@ -150,7 +150,7 @@ impl Bootstrap { parent.inode.set_child_index(index); parent.inode.set_child_count(tree.children.len() as u32); if ctx.fs_version.is_v6() { - parent.v6_set_dir_offset(bootstrap_ctx, tree.node.v6_dir_d_size(tree)?)?; + parent.v6_set_dir_offset(bootstrap_ctx, tree.node.v6_dirent_size(tree)?)?; } } diff --git a/rafs/src/builder/core/node.rs b/rafs/src/builder/core/node.rs index 7d863ce7361..dcbf12df7d3 100644 --- a/rafs/src/builder/core/node.rs +++ b/rafs/src/builder/core/node.rs @@ -1,5 +1,5 @@ // Copyright 2020 Ant Group. All rights reserved. -// Copyright (C) 2021 Alibaba Cloud. All rights reserved. +// Copyright (C) 2021-2023 Alibaba Cloud. All rights reserved. // // SPDX-License-Identifier: Apache-2.0 @@ -16,7 +16,7 @@ use std::os::unix::ffi::OsStrExt; use std::path::{Component, Path, PathBuf}; use std::sync::Arc; -use anyhow::{anyhow, Context, Error, Result}; +use anyhow::{anyhow, bail, Context, Error, Result}; use nydus_storage::device::BlobFeatures; use nydus_storage::meta::{BlobChunkInfoV2Ondisk, BlobMetaChunkInfo}; use nydus_utils::compress; @@ -24,9 +24,7 @@ use nydus_utils::digest::{DigestHasher, RafsDigest}; use nydus_utils::{div_round_up, event_tracer, root_tracer, try_round_up_4k, ByteSize}; use sha2::digest::Digest; -use super::chunk_dict::ChunkDict; -use super::context::{ArtifactWriter, BlobContext, BlobManager, BuildContext}; -use super::overlay::Overlay; +use crate::builder::{ArtifactWriter, BlobContext, BlobManager, BuildContext, ChunkDict, Overlay}; use crate::metadata::chunk::ChunkWrapper; use crate::metadata::inode::InodeWrapper; use crate::metadata::layout::v6::EROFS_INODE_FLAT_PLAIN; @@ -34,9 +32,9 @@ use crate::metadata::layout::RafsXAttrs; use crate::metadata::{Inode, RafsVersion}; /// Filesystem root path for Unix OSs. -pub const ROOT_PATH_NAME: &[u8] = &[b'/']; +const ROOT_PATH_NAME: &[u8] = &[b'/']; -/// Where the chunk data is actually stored. +/// Source of chunk data: chunk dictionary, parent filesystem or builder. #[derive(Clone, Hash, PartialEq, Eq)] pub enum ChunkSource { /// Chunk is stored in data blob owned by current image. @@ -57,6 +55,7 @@ impl Display for ChunkSource { } } +/// Chunk information for RAFS filesystem builder. #[derive(Clone)] pub struct NodeChunk { pub source: ChunkSource, @@ -70,30 +69,35 @@ impl Display for NodeChunk { } impl NodeChunk { + /// Copy all chunk information from another `ChunkWrapper` object. pub fn copy_from(&mut self, other: &ChunkWrapper) { let mut chunk = self.inner.deref().clone(); chunk.copy_from(other); self.inner = Arc::new(chunk); } + /// Set chunk index. pub fn set_index(&mut self, index: u32) { let mut chunk = self.inner.deref().clone(); chunk.set_index(index); self.inner = Arc::new(chunk); } + /// Set blob index. pub fn set_blob_index(&mut self, index: u32) { let mut chunk = self.inner.deref().clone(); chunk.set_blob_index(index); self.inner = Arc::new(chunk); } + /// Set chunk compressed size. pub fn set_compressed_size(&mut self, size: u32) { let mut chunk = self.inner.deref().clone(); chunk.set_compressed_size(size); self.inner = Arc::new(chunk); } + /// Set file offset of chunk. pub fn set_file_offset(&mut self, offset: u64) { let mut chunk = self.inner.deref().clone(); chunk.set_file_offset(offset); @@ -101,7 +105,7 @@ impl NodeChunk { } } -/// Immutable information for a [Node] object. +/// Struct to host sharable fields of [Node]. #[derive(Clone, Default, Debug)] pub struct NodeInfo { /// Last status change time of the file, in nanoseconds. @@ -157,10 +161,10 @@ pub struct Node { pub v6_datalayout: u16, /// V6: offset to calculate nid. pub v6_offset: u64, - /// V6: information to build directory entries. - pub v6_dirents: Vec<(u64, OsString, u32)>, /// V6: offset to build directory entries. pub v6_dirents_offset: u64, + /// V6: information to build directory entries. + pub v6_dirents: Vec<(u64, OsString, u32)>, } impl Display for Node { @@ -190,56 +194,7 @@ impl Display for Node { } impl Node { - pub fn new( - version: RafsVersion, - source: PathBuf, - path: PathBuf, - overlay: Overlay, - chunk_size: u32, - explicit_uidgid: bool, - v6_force_extended_inode: bool, - ) -> Result { - let target = Self::generate_target(&path, &source); - let target_vec = Self::generate_target_vec(&target); - let info = NodeInfo { - ctime: 0, - explicit_uidgid, - src_ino: 0, - src_dev: u64::MAX, - rdev: u64::MAX, - source, - target, - path, - target_vec, - symlink: None, - xattrs: RafsXAttrs::default(), - v6_force_extended_inode, - }; - let mut node = Node { - info: Arc::new(info), - index: 0, - layer_idx: 0, - overlay, - inode: InodeWrapper::new(version), - chunks: Vec::new(), - v6_datalayout: EROFS_INODE_FLAT_PLAIN, - v6_compact_inode: false, - v6_offset: 0, - v6_dirents: Vec::new(), - v6_dirents_offset: 0, - }; - - node.build_inode(chunk_size) - .context("failed to build inode")?; - if version.is_v6() { - node.v6_set_inode_compact(); - } - - Ok(node) - } - - /// Dump data from the file assoicated with the node into the data blob, and generate chunk - /// information. + /// Dump node data into the data blob, and generate chunk information. /// /// # Arguments /// - blob_writer: optional writer to write data into the data blob. @@ -354,108 +309,6 @@ impl Node { Ok(blob_size) } - fn build_inode_xattr(&mut self) -> Result<()> { - let file_xattrs = match xattr::list(self.path()) { - Ok(x) => x, - Err(e) => { - if e.raw_os_error() == Some(libc::EOPNOTSUPP) { - return Ok(()); - } else { - return Err(anyhow!("failed to list xattr of {:?}", self.path())); - } - } - }; - - let mut info = self.info.deref().clone(); - for key in file_xattrs { - let value = xattr::get(self.path(), &key).context(format!( - "failed to get xattr {:?} of {:?}", - key, - self.path() - ))?; - info.xattrs.add(key, value.unwrap_or_default())?; - } - if !info.xattrs.is_empty() { - self.inode.set_has_xattr(true); - } - self.info = Arc::new(info); - - Ok(()) - } - - fn build_inode_stat(&mut self) -> Result<()> { - let meta = self.meta()?; - let mut info = self.info.deref().clone(); - - info.src_ino = meta.st_ino(); - info.src_dev = meta.st_dev(); - info.rdev = meta.st_rdev(); - info.ctime = meta.st_ctime(); - - self.inode.set_mode(meta.st_mode()); - if info.explicit_uidgid { - self.inode.set_uid(meta.st_uid()); - self.inode.set_gid(meta.st_gid()); - } - - // Usually the root directory is created by the build tool (nydusify/buildkit/acceld) - // and the mtime of the root directory is different for each build, which makes it - // completely impossible to achieve repeatable builds, especially in a tar build scenario - // (blob + bootstrap in one tar layer), which causes the layer hash to change and wastes - // registry storage space, so the mtime of the root directory is forced to be ignored here. - let ignore_mtime = self.is_root(); - if !ignore_mtime { - self.inode.set_mtime(meta.st_mtime() as u64); - self.inode.set_mtime_nsec(meta.st_mtime_nsec() as u32); - } - self.inode.set_projid(0); - self.inode.set_rdev(meta.st_rdev() as u32); - // Ignore actual nlink value and calculate from rootfs directory instead - self.inode.set_nlink(1); - - // Different filesystem may have different algorithms to calculate size/blocks for - // directory entries, so let's ignore the value provided by source filesystem and - // calculate it later by ourself. - if !self.is_dir() { - self.inode.set_size(meta.st_size()); - self.set_inode_blocks(); - } - self.info = Arc::new(info); - - Ok(()) - } - - fn build_inode(&mut self, chunk_size: u32) -> Result<()> { - self.inode.set_name_size(self.name().byte_size()); - - // NOTE: Always retrieve xattr before attr so that we can know the size of xattr pairs. - self.build_inode_xattr()?; - self.build_inode_stat() - .with_context(|| format!("failed to build inode {:?}", self.path()))?; - - if self.is_reg() { - // Reuse `child_count` to store `chunk_count` for normal files. - self.inode - .set_child_count(self.chunk_count(chunk_size as u64)); - } else if self.is_symlink() { - let target_path = fs::read_link(self.path())?; - let symlink: OsString = target_path.into(); - let size = symlink.byte_size(); - self.inode.set_symlink_size(size); - let mut info = self.info.deref().clone(); - info.symlink = Some(symlink); - self.info = Arc::new(info); - } - - Ok(()) - } - - fn meta(&self) -> Result { - self.path() - .symlink_metadata() - .with_context(|| format!("failed to get metadata from {:?}", self.path())) - } - fn read_file_chunk( &self, ctx: &BuildContext, @@ -490,8 +343,7 @@ impl Node { .with_context(|| format!("failed to read node file {:?}", self.path()))?; } - let chunk_id = RafsDigest::from_buf(buf, ctx.digester); - chunk.set_id(chunk_id); + chunk.set_id(RafsDigest::from_buf(buf, ctx.digester)); Ok((chunk, chunk_info)) } @@ -610,6 +462,169 @@ impl Node { } } +// build node object from a filesystem object. +impl Node { + /// Create a new instance of [Node] from a filesystem object. + pub fn from_fs_object( + version: RafsVersion, + source: PathBuf, + path: PathBuf, + overlay: Overlay, + chunk_size: u32, + explicit_uidgid: bool, + v6_force_extended_inode: bool, + ) -> Result { + let target = Self::generate_target(&path, &source); + let target_vec = Self::generate_target_vec(&target); + let info = NodeInfo { + ctime: 0, + explicit_uidgid, + src_ino: 0, + src_dev: u64::MAX, + rdev: u64::MAX, + source, + target, + path, + target_vec, + symlink: None, + xattrs: RafsXAttrs::default(), + v6_force_extended_inode, + }; + let mut node = Node { + info: Arc::new(info), + index: 0, + layer_idx: 0, + overlay, + inode: InodeWrapper::new(version), + chunks: Vec::new(), + v6_datalayout: EROFS_INODE_FLAT_PLAIN, + v6_compact_inode: false, + v6_offset: 0, + v6_dirents_offset: 0, + v6_dirents: Vec::new(), + }; + + node.build_inode(chunk_size) + .context("failed to build Node from fs object")?; + if version.is_v6() { + node.v6_set_inode_compact(); + } + + Ok(node) + } + + fn build_inode_xattr(&mut self) -> Result<()> { + let file_xattrs = match xattr::list(self.path()) { + Ok(x) => x, + Err(e) => { + if e.raw_os_error() == Some(libc::EOPNOTSUPP) { + return Ok(()); + } else { + return Err(anyhow!( + "failed to list xattr of {}, {}", + self.path().display(), + e + )); + } + } + }; + + let mut info = self.info.deref().clone(); + for key in file_xattrs { + let value = xattr::get(self.path(), &key).with_context(|| { + format!("failed to get xattr {:?} of {}", key, self.path().display()) + })?; + info.xattrs.add(key, value.unwrap_or_default())?; + } + if !info.xattrs.is_empty() { + self.inode.set_has_xattr(true); + } + self.info = Arc::new(info); + + Ok(()) + } + + fn build_inode_stat(&mut self) -> Result<()> { + let meta = self + .meta() + .with_context(|| format!("failed to get metadata of {}", self.path().display()))?; + let mut info = self.info.deref().clone(); + + info.src_ino = meta.st_ino(); + info.src_dev = meta.st_dev(); + info.rdev = meta.st_rdev(); + info.ctime = meta.st_ctime(); + + self.inode.set_mode(meta.st_mode()); + if info.explicit_uidgid { + self.inode.set_uid(meta.st_uid()); + self.inode.set_gid(meta.st_gid()); + } + + // Usually the root directory is created by the build tool (nydusify/buildkit/acceld) + // and the mtime of the root directory is different for each build, which makes it + // completely impossible to achieve repeatable builds, especially in a tar build scenario + // (blob + bootstrap in one tar layer), which causes the layer hash to change and wastes + // registry storage space, so the mtime of the root directory is forced to be ignored here. + let ignore_mtime = self.is_root(); + if !ignore_mtime { + self.inode.set_mtime(meta.st_mtime() as u64); + self.inode.set_mtime_nsec(meta.st_mtime_nsec() as u32); + } + self.inode.set_projid(0); + self.inode.set_rdev(meta.st_rdev() as u32); + // Ignore actual nlink value and calculate from rootfs directory instead + self.inode.set_nlink(1); + + // Different filesystem may have different algorithms to calculate size/blocks for + // directory entries, so let's ignore the value provided by source filesystem and + // calculate it later by ourself. + if !self.is_dir() { + self.inode.set_size(meta.st_size()); + self.set_inode_blocks(); + } + self.info = Arc::new(info); + + Ok(()) + } + + fn build_inode(&mut self, chunk_size: u32) -> Result<()> { + self.inode.set_name_size(self.name().byte_size()); + + // NOTE: Always retrieve xattr before attr so that we can know the size of xattr pairs. + self.build_inode_xattr() + .with_context(|| format!("failed to get xattr for {}", self.path().display()))?; + self.build_inode_stat() + .with_context(|| format!("failed to build inode {}", self.path().display()))?; + + if self.is_reg() { + let chunk_count = self.chunk_count(chunk_size as u64).with_context(|| { + format!("failed to get chunk count for {}", self.path().display()) + })?; + self.inode.set_child_count(chunk_count); + } else if self.is_symlink() { + let target_path = fs::read_link(self.path()).with_context(|| { + format!( + "failed to read symlink target for {}", + self.path().display() + ) + })?; + let symlink: OsString = target_path.into(); + let size = symlink.byte_size(); + self.inode.set_symlink_size(size); + self.set_symlink(symlink); + } + + Ok(()) + } + + fn meta(&self) -> Result { + self.path() + .symlink_metadata() + .with_context(|| format!("failed to get metadata of {}", self.path().display())) + } +} + // Access Methods impl Node { pub fn is_root(&self) -> bool { @@ -636,13 +651,16 @@ impl Node { self.inode.is_special() } - pub fn chunk_count(&self, chunk_size: u64) -> u32 { + pub fn chunk_count(&self, chunk_size: u64) -> Result { if self.is_reg() { let chunks = div_round_up(self.inode.size(), chunk_size); - debug_assert!(chunks < u32::MAX as u64); - chunks as u32 + if chunks > u32::MAX as u64 { + bail!("file size 0x{:x} is too big", self.inode.size()) + } else { + Ok(chunks as u32) + } } else { - 0 + Ok(0) } } @@ -735,6 +753,13 @@ impl Node { } } + /// Set symlink target for the node. + pub fn set_symlink(&mut self, symlink: OsString) { + let mut info = self.info.deref().clone(); + info.symlink = Some(symlink); + self.info = Arc::new(info); + } + /// Set extended attributes for the node. pub fn set_xattr(&mut self, xattr: RafsXAttrs) { let mut info = self.info.deref().clone(); diff --git a/rafs/src/builder/core/v6.rs b/rafs/src/builder/core/v6.rs index 19f89234427..77c6a3035e4 100644 --- a/rafs/src/builder/core/v6.rs +++ b/rafs/src/builder/core/v6.rs @@ -75,24 +75,18 @@ impl Node { Ok(()) } - pub fn v6_dir_d_size(&self, tree: &Tree) -> Result { - ensure!(self.is_dir(), "{} is not a directory", self); - // Use length in byte, instead of length in character. - let mut d_size: u64 = (".".as_bytes().len() - + size_of::() - + "..".as_bytes().len() - + size_of::()) as u64; - - for child in tree.children.iter() { - let len = child.node.name().as_bytes().len() + size_of::(); - // erofs disk format requires dirent to be aligned with 4096. - if (d_size % EROFS_BLOCK_SIZE) + len as u64 > EROFS_BLOCK_SIZE { - d_size = div_round_up(d_size as u64, EROFS_BLOCK_SIZE) * EROFS_BLOCK_SIZE; - } - d_size += len as u64; + pub fn v6_set_inode_compact(&mut self) { + if self.info.v6_force_extended_inode + || self.inode.uid() > u16::MAX as u32 + || self.inode.gid() > u16::MAX as u32 + || self.inode.nlink() > u16::MAX as u32 + || self.inode.size() > u32::MAX as u64 + || self.path().extension() == Some(OsStr::new("pyc")) + { + self.v6_compact_inode = false; + } else { + self.v6_compact_inode = true; } - - Ok(d_size) } /// Set node offset in bootstrap and return the next position. @@ -105,7 +99,7 @@ impl Node { if let Some(v6_hardlink_offset) = v6_hardlink_offset { self.v6_offset = v6_hardlink_offset; } else { - let size = self.v6_size_with_xattr() as u64; + let size = self.v6_size_with_xattr(); let unit = size_of::() as u64; // We first try to allocate space from used blocks. // If no available used block exists, we allocate sequentially. @@ -123,7 +117,7 @@ impl Node { self.v6_set_offset_with_tail(bootstrap_ctx, self.inode.size()); } else { self.v6_offset = bootstrap_ctx.offset; - bootstrap_ctx.offset += self.v6_size_with_xattr() as u64; + bootstrap_ctx.offset += self.v6_size_with_xattr(); } } @@ -141,9 +135,29 @@ impl Node { Ok(()) } - fn v6_size_with_xattr(&self) -> usize { + pub fn v6_dirent_size(&self, tree: &Tree) -> Result { + ensure!(self.is_dir(), "{} is not a directory", self); + // Use length in byte, instead of length in character. + let mut d_size: u64 = (".".as_bytes().len() + + size_of::() + + "..".as_bytes().len() + + size_of::()) as u64; + + for child in tree.children.iter() { + let len = child.node.name().as_bytes().len() + size_of::(); + // erofs disk format requires dirent to be aligned with 4096. + if (d_size % EROFS_BLOCK_SIZE) + len as u64 > EROFS_BLOCK_SIZE { + d_size = div_round_up(d_size as u64, EROFS_BLOCK_SIZE) * EROFS_BLOCK_SIZE; + } + d_size += len as u64; + } + + Ok(d_size) + } + + fn v6_size_with_xattr(&self) -> u64 { self.inode - .get_inode_size_with_xattr(&self.info.xattrs, self.v6_compact_inode) + .get_inode_size_with_xattr(&self.info.xattrs, self.v6_compact_inode) as u64 } // For DIR inode, size is the total bytes of 'dirents + names'. @@ -186,7 +200,7 @@ impl Node { // | inode | // // - let inode_size = self.v6_size_with_xattr() as u64; + let inode_size = self.v6_size_with_xattr(); let tail: u64 = d_size % EROFS_BLOCK_SIZE; // We use a simple inline strategy here: @@ -258,20 +272,6 @@ impl Node { ); } - pub fn v6_set_inode_compact(&mut self) { - if self.info.v6_force_extended_inode - || self.inode.uid() > u16::MAX as u32 - || self.inode.gid() > u16::MAX as u32 - || self.inode.nlink() > u16::MAX as u32 - || self.inode.size() > u32::MAX as u64 - || self.path().extension() == Some(OsStr::new("pyc")) - { - self.v6_compact_inode = false; - } else { - self.v6_compact_inode = true; - } - } - fn v6_store_xattrs( &mut self, ctx: &mut BuildContext, @@ -390,7 +390,7 @@ impl Node { } let tail_off = match self.v6_datalayout { - EROFS_INODE_FLAT_INLINE => self.v6_offset + self.v6_size_with_xattr() as u64, + EROFS_INODE_FLAT_INLINE => self.v6_offset + self.v6_size_with_xattr(), EROFS_INODE_FLAT_PLAIN => dirent_off, _ => bail!("unsupported RAFS v6 inode layout for directory"), }; @@ -455,7 +455,7 @@ impl Node { // Dump chunk indexes let unit = size_of::() as u64; - let chunk_off = align_offset(self.v6_offset + self.v6_size_with_xattr() as u64, unit); + let chunk_off = align_offset(self.v6_offset + self.v6_size_with_xattr(), unit); f_bootstrap .seek(SeekFrom::Start(chunk_off)) .context("failed seek for dir inode")?; @@ -485,7 +485,7 @@ impl Node { // write symlink. if let Some(symlink) = &self.info.symlink { let tail_off = match self.v6_datalayout { - EROFS_INODE_FLAT_INLINE => self.v6_offset + self.v6_size_with_xattr() as u64, + EROFS_INODE_FLAT_INLINE => self.v6_offset + self.v6_size_with_xattr(), EROFS_INODE_FLAT_PLAIN => data_off, _ => bail!("unsupported RAFS v5 inode layout for symlink"), }; @@ -515,7 +515,7 @@ mod tests { fn test_set_v6_offset() { let pa = TempDir::new().unwrap(); let pa_aa = TempFile::new_in(pa.as_path()).unwrap(); - let mut node = Node::new( + let mut node = Node::from_fs_object( RafsVersion::V6, pa.as_path().to_path_buf(), pa_aa.as_path().to_path_buf(), @@ -541,7 +541,7 @@ mod tests { assert_eq!(bootstrap_ctx.offset, 32); // symlink and dir are handled in the same way. - let mut dir_node = Node::new( + let mut dir_node = Node::from_fs_object( RafsVersion::V6, pa.as_path().to_path_buf(), pa.as_path().to_path_buf(), @@ -637,7 +637,7 @@ mod tests { let pa_pyc = pa.as_path().join("foo.pyc"); let _ = File::create(&pa_pyc).unwrap(); - let reg_node = Node::new( + let reg_node = Node::from_fs_object( RafsVersion::V6, pa.as_path().to_path_buf(), pa_reg.as_path().to_path_buf(), @@ -650,7 +650,7 @@ mod tests { assert!(reg_node.v6_compact_inode); - let pyc_node = Node::new( + let pyc_node = Node::from_fs_object( RafsVersion::V6, pa.as_path().to_path_buf(), pa_pyc.as_path().to_path_buf(), diff --git a/rafs/src/builder/directory.rs b/rafs/src/builder/directory.rs index 97b1b40186b..82743a4a0b5 100644 --- a/rafs/src/builder/directory.rs +++ b/rafs/src/builder/directory.rs @@ -45,7 +45,7 @@ impl FilesystemTreeBuilder { event_tracer!("load_from_directory", +children.len()); for child in children { let path = child.path(); - let mut child = Node::new( + let mut child = Node::from_fs_object( ctx.fs_version, ctx.source_path.clone(), path.clone(), @@ -91,7 +91,7 @@ impl DirectoryBuilder { bootstrap_ctx: &mut BootstrapContext, layer_idx: u16, ) -> Result { - let node = Node::new( + let node = Node::from_fs_object( ctx.fs_version, ctx.source_path.clone(), ctx.source_path.clone(), diff --git a/src/bin/nydus-image/stat.rs b/src/bin/nydus-image/stat.rs index 09b20a4a011..fe1eb2d2002 100644 --- a/src/bin/nydus-image/stat.rs +++ b/src/bin/nydus-image/stat.rs @@ -192,7 +192,10 @@ impl ImageStat { } for sz in 12..=20 { - image.chunk_sizes[sz - 12] += node.chunk_count(1 << sz); + match node.chunk_count(1 << sz) { + Ok(v) => image.chunk_sizes[sz - 12] += v, + Err(e) => error!("failed to get chunk size of inode, {}", e), + } } } else if node.is_dir() { image.dirs += 1; From 4372f96cfb8fa0e9d36cff32f276d16dd8732af8 Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Sun, 5 Mar 2023 23:02:07 +0800 Subject: [PATCH 08/10] rafs: refine RAFS v6 builder implementation Refine RAFS v6 builder implementation by: - introduce helper Node::v6_dump_inode() to reduce duplicated code - introduce helper BuildContext::v6_block_addr() Signed-off-by: Jiang Liu --- rafs/src/builder/core/bootstrap.rs | 4 +- rafs/src/builder/core/v6.rs | 136 ++++++++++++++++++----------- 2 files changed, 86 insertions(+), 54 deletions(-) diff --git a/rafs/src/builder/core/bootstrap.rs b/rafs/src/builder/core/bootstrap.rs index 48eb248b445..0e2d5b4dd0a 100644 --- a/rafs/src/builder/core/bootstrap.rs +++ b/rafs/src/builder/core/bootstrap.rs @@ -205,7 +205,9 @@ impl Bootstrap { // update bootstrap_ctx.offset for rafs v6. if !child.node.is_dir() && ctx.fs_version.is_v6() { - child.node.v6_set_offset(bootstrap_ctx, v6_hardlink_offset); + child + .node + .v6_set_offset(bootstrap_ctx, v6_hardlink_offset)?; bootstrap_ctx.align_offset(EROFS_INODE_SLOT_SIZE as u64); } diff --git a/rafs/src/builder/core/v6.rs b/rafs/src/builder/core/v6.rs index 77c6a3035e4..612028badd2 100644 --- a/rafs/src/builder/core/v6.rs +++ b/rafs/src/builder/core/v6.rs @@ -51,9 +51,10 @@ impl Node { let meta_offset = meta_addr - orig_meta_addr; // update all the inodes's offset according to the new 'meta_addr'. self.v6_offset += meta_offset; - // The EROFS_INODE_FLAT_INLINE layout is valid for directory and symlink only, so - // `dirents_offset` is useful for these two types too, otherwise `dirents_offset` should - // always be zero. Enforce the check to avoid overflow of `dirents_offset`. + // The EROFS_INODE_FLAT_INLINE layout is valid for directory and symlink only, + // so `dirents_offset` is useful for these two types too, otherwise `dirents_offset` + // should always be zero. + // Enforce the check to avoid overflow of `dirents_offset`. if self.is_dir() || self.is_symlink() { self.v6_dirents_offset += meta_offset; } @@ -75,6 +76,7 @@ impl Node { Ok(()) } + /// Update whether compact mode can be used for this inode or not. pub fn v6_set_inode_compact(&mut self) { if self.info.v6_force_extended_inode || self.inode.uid() > u16::MAX as u32 @@ -89,27 +91,25 @@ impl Node { } } - /// Set node offset in bootstrap and return the next position. + /// Layout the normal inode (except directory inode) into the meta blob. pub fn v6_set_offset( &mut self, bootstrap_ctx: &mut BootstrapContext, v6_hardlink_offset: Option, - ) { + ) -> Result<()> { + ensure!(!self.is_dir(), "{} is a directory", self.path().display()); if self.is_reg() { if let Some(v6_hardlink_offset) = v6_hardlink_offset { self.v6_offset = v6_hardlink_offset; } else { let size = self.v6_size_with_xattr(); let unit = size_of::() as u64; - // We first try to allocate space from used blocks. - // If no available used block exists, we allocate sequentially. let total_size = round_up(size, unit) + self.inode.child_count() as u64 * unit; + // First try to allocate from fragments of dirent pages. self.v6_offset = bootstrap_ctx.allocate_available_block(total_size); if self.v6_offset == 0 { self.v6_offset = bootstrap_ctx.offset; - bootstrap_ctx.offset += size; - bootstrap_ctx.align_offset(unit); - bootstrap_ctx.offset += self.inode.child_count() as u64 * unit; + bootstrap_ctx.offset += total_size; } } self.v6_datalayout = EROFS_INODE_CHUNK_BASED; @@ -119,14 +119,21 @@ impl Node { self.v6_offset = bootstrap_ctx.offset; bootstrap_ctx.offset += self.v6_size_with_xattr(); } + + Ok(()) } + /// Layout the directory inode and its dirents into meta blob. pub fn v6_set_dir_offset( &mut self, bootstrap_ctx: &mut BootstrapContext, d_size: u64, ) -> Result<()> { - ensure!(self.is_dir(), "{} is not a directory", self); + ensure!( + self.is_dir(), + "{} is not a directory", + self.path().display() + ); // Dir isize is the total bytes of 'dirents + names'. self.inode.set_size(d_size); @@ -135,6 +142,7 @@ impl Node { Ok(()) } + /// Calculate space needed to store dirents of the directory inode. pub fn v6_dirent_size(&self, tree: &Tree) -> Result { ensure!(self.is_dir(), "{} is not a directory", self); // Use length in byte, instead of length in character. @@ -160,11 +168,11 @@ impl Node { .get_inode_size_with_xattr(&self.info.xattrs, self.v6_compact_inode) as u64 } + // Layout symlink or directory inodes into the meta blob. + // // For DIR inode, size is the total bytes of 'dirents + names'. // For symlink, size is the length of symlink name. fn v6_set_offset_with_tail(&mut self, bootstrap_ctx: &mut BootstrapContext, d_size: u64) { - // TODO: a hashmap of non-full blocks - // | avail | // +--------+-----------+----+ +-----------------------+ // | |inode+tail | free | dirents+names | @@ -297,15 +305,12 @@ impl Node { ) -> Result<()> { // the 1st 4k block after dir inode. let mut dirent_off = self.v6_dirents_offset; - inode.set_u((dirent_off / EROFS_BLOCK_SIZE) as u32); - - // Dump inode - trace!("{:?} dir inode: offset {}", self.target(), self.v6_offset); - f_bootstrap - .seek(SeekFrom::Start(self.v6_offset)) - .context("failed seek for dir inode")?; - inode.store(f_bootstrap).context("failed to store inode")?; - self.v6_store_xattrs(ctx, f_bootstrap)?; + let blk_addr = ctx + .v6_block_addr(dirent_off) + .with_context(|| format!("failed to compute blk_addr for offset 0x{:x}", dirent_off))?; + inode.set_u(blk_addr); + self.v6_dump_inode(ctx, f_bootstrap, inode) + .context("failed to dump inode for directory")?; // Dump dirents let mut dir_data: Vec = Vec::new(); @@ -339,10 +344,10 @@ impl Node { f_bootstrap .seek(SeekFrom::Start(dirent_off as u64)) - .context("failed seek for dir inode")?; + .context("failed seek file position for writing dirent")?; f_bootstrap .write(dir_data.as_slice()) - .context("failed to store dirents")?; + .context("failed to write dirent data to meta blob")?; dir_data.clear(); entry_names.clear(); @@ -418,24 +423,30 @@ impl Node { // write chunk indexes, chunk contents has been written to blob file. let mut chunks: Vec = Vec::new(); for chunk in self.chunks.iter() { - let uncompressed_offset = chunk.inner.uncompressed_offset(); + let offset = chunk.inner.uncompressed_offset(); + let blk_addr = ctx.v6_block_addr(offset).with_context(|| { + format!( + "failed to compute blk_addr for chunk with uncompressed offset 0x{:x}", + offset + ) + })?; let blob_idx = chunk.inner.blob_index(); let mut v6_chunk = RafsV6InodeChunkAddr::new(); v6_chunk.set_blob_index(blob_idx); v6_chunk.set_blob_ci_index(chunk.inner.index()); - v6_chunk.set_block_addr((uncompressed_offset / EROFS_BLOCK_SIZE) as u32); + v6_chunk.set_block_addr(blk_addr); + chunks.extend(v6_chunk.as_ref()); chunk_cache.insert( DigestWithBlobIndex(*chunk.inner.id(), chunk.inner.blob_index() + 1), chunk.inner.clone(), ); - if let Some((prev_idx, prev_pos)) = prev { - if prev_pos + ctx.chunk_size as u64 != uncompressed_offset || prev_idx != blob_idx { + if prev_pos + ctx.chunk_size as u64 != offset || prev_idx != blob_idx { is_continuous = false; } } - prev = Some((blob_idx, uncompressed_offset)); + prev = Some((blob_idx, offset)); } // Special optimization to enable page cache sharing for EROFS. @@ -446,22 +457,17 @@ impl Node { }; let info = RafsV6InodeChunkHeader::new(chunk_size); inode.set_u(info.to_u32()); + self.v6_dump_inode(ctx, f_bootstrap, inode) + .context("failed to dump inode for file")?; - f_bootstrap - .seek(SeekFrom::Start(self.v6_offset)) - .context("failed seek for dir inode")?; - inode.store(f_bootstrap).context("failed to store inode")?; - self.v6_store_xattrs(ctx, f_bootstrap)?; - - // Dump chunk indexes let unit = size_of::() as u64; - let chunk_off = align_offset(self.v6_offset + self.v6_size_with_xattr(), unit); + let offset = align_offset(self.v6_offset + self.v6_size_with_xattr(), unit); f_bootstrap - .seek(SeekFrom::Start(chunk_off)) - .context("failed seek for dir inode")?; + .seek(SeekFrom::Start(offset)) + .with_context(|| format!("failed to seek to 0x{:x} for writing chunk data", offset))?; f_bootstrap .write(chunks.as_slice()) - .context("failed to write chunkindexes")?; + .context("failed to write chunk data for file")?; Ok(()) } @@ -472,24 +478,17 @@ impl Node { f_bootstrap: &mut dyn RafsIoWrite, inode: &mut Box, ) -> Result<()> { - let data_off = self.v6_dirents_offset; - // TODO: check whether 'i_u' is used at all in case of inline symlink. - inode.set_u((data_off / EROFS_BLOCK_SIZE) as u32); - - f_bootstrap - .seek(SeekFrom::Start(self.v6_offset)) - .context("failed seek for symlink inode")?; - inode.store(f_bootstrap).context("failed to store inode")?; - self.v6_store_xattrs(ctx, f_bootstrap)?; + let blk_addr = ctx.v6_block_addr(self.v6_dirents_offset)?; + inode.set_u(blk_addr); + self.v6_dump_inode(ctx, f_bootstrap, inode) + .context("failed to dump inode for symlink")?; - // write symlink. if let Some(symlink) = &self.info.symlink { let tail_off = match self.v6_datalayout { EROFS_INODE_FLAT_INLINE => self.v6_offset + self.v6_size_with_xattr(), - EROFS_INODE_FLAT_PLAIN => data_off, + EROFS_INODE_FLAT_PLAIN => self.v6_dirents_offset, _ => bail!("unsupported RAFS v5 inode layout for symlink"), }; - trace!("symlink write_off {}", tail_off); f_bootstrap .seek(SeekFrom::Start(tail_off)) .context("failed seek for dir inode")?; @@ -500,6 +499,37 @@ impl Node { Ok(()) } + + fn v6_dump_inode( + &mut self, + ctx: &mut BuildContext, + f_bootstrap: &mut dyn RafsIoWrite, + inode: &mut Box, + ) -> Result<()> { + f_bootstrap + .seek(SeekFrom::Start(self.v6_offset)) + .context("failed to seek file position for writing inode")?; + inode + .store(f_bootstrap) + .context("failed to write inode to meta blob")?; + self.v6_store_xattrs(ctx, f_bootstrap) + .context("failed to write extended attributes for inode") + } +} + +impl BuildContext { + pub fn v6_block_size(&self) -> u64 { + EROFS_BLOCK_SIZE + } + + pub fn v6_block_addr(&self, offset: u64) -> Result { + let blk_addr = offset / self.v6_block_size(); + if blk_addr > u32::MAX as u64 { + bail!("v6 block address 0x{:x} is too big", blk_addr) + } else { + Ok(blk_addr as u32) + } + } } #[cfg(test)] @@ -534,7 +564,7 @@ mod tests { // reg file. // "1" is used only for testing purpose, in practice // it's always aligned to 32 bytes. - node.v6_set_offset(&mut bootstrap_ctx, None); + node.v6_set_offset(&mut bootstrap_ctx, None).unwrap(); assert_eq!(node.v6_offset, 0); assert_eq!(node.v6_datalayout, EROFS_INODE_CHUNK_BASED); assert!(node.v6_compact_inode); From 7a226ce9f9340662e8ac279d4eab42acd0a111e8 Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Mon, 6 Mar 2023 16:33:43 +0800 Subject: [PATCH 09/10] rafs: refine builder Bootstrap implementation Refine builder Bootstrap implementation for maintenance. Signed-off-by: Jiang Liu --- rafs/src/builder/compact.rs | 4 +- rafs/src/builder/core/bootstrap.rs | 578 +++-------------------------- rafs/src/builder/core/v5.rs | 174 ++++++++- rafs/src/builder/core/v6.rs | 341 ++++++++++++++++- rafs/src/builder/mod.rs | 4 +- rafs/src/metadata/layout/v6.rs | 1 - src/bin/nydus-image/merge.rs | 4 +- 7 files changed, 559 insertions(+), 547 deletions(-) diff --git a/rafs/src/builder/compact.rs b/rafs/src/builder/compact.rs index 4f2ca8d953b..960ef12bad7 100644 --- a/rafs/src/builder/compact.rs +++ b/rafs/src/builder/compact.rs @@ -593,9 +593,9 @@ impl BlobCompactor { return Ok(None); } let mut _dict = HashChunkDict::new(build_ctx.digester); - let mut tree = Tree::from_bootstrap(&rs, &mut _dict)?; + let tree = Tree::from_bootstrap(&rs, &mut _dict)?; let mut bootstrap = Bootstrap::new()?; - bootstrap.build(&mut build_ctx, &mut bootstrap_ctx, &mut tree)?; + bootstrap.build(&mut build_ctx, &mut bootstrap_ctx, tree)?; let mut nodes = Vec::new(); // move out nodes std::mem::swap(&mut bootstrap_ctx.nodes, &mut nodes); diff --git a/rafs/src/builder/core/bootstrap.rs b/rafs/src/builder/core/bootstrap.rs index 0e2d5b4dd0a..27ece03a62e 100644 --- a/rafs/src/builder/core/bootstrap.rs +++ b/rafs/src/builder/core/bootstrap.rs @@ -1,41 +1,29 @@ // Copyright 2020 Ant Group. All rights reserved. +// Copyright (C) 2023 Alibaba Cloud. All rights reserved. // // SPDX-License-Identifier: Apache-2.0 -use std::collections::BTreeMap; -use std::convert::TryFrom; use std::ffi::OsString; -use std::io::SeekFrom; -use std::mem::size_of; -use anyhow::{bail, Context, Error, Result}; -use nydus_storage::device::BlobFeatures; -use nydus_utils::digest::{self, DigestHasher, RafsDigest}; +use anyhow::{Context, Error, Result}; +use nydus_utils::digest::{self, RafsDigest}; use nydus_utils::{root_tracer, timing_tracer}; use super::node::Node; use super::overlay::{WhiteoutType, OVERLAYFS_WHITEOUT_OPAQUE}; use crate::builder::{ - ArtifactStorage, BlobManager, BootstrapContext, BootstrapManager, BuildContext, ConversionType, - Tree, -}; -use crate::metadata::layout::v5::{ - RafsV5BlobTable, RafsV5ChunkInfo, RafsV5InodeTable, RafsV5SuperBlock, RafsV5XAttrsTable, -}; -use crate::metadata::layout::v6::{ - align_offset, calculate_nid, RafsV6BlobTable, RafsV6Device, RafsV6SuperBlock, - RafsV6SuperBlockExt, EROFS_BLOCK_SIZE, EROFS_DEVTABLE_OFFSET, EROFS_INODE_SLOT_SIZE, + ArtifactStorage, BlobManager, BootstrapContext, BootstrapManager, BuildContext, Tree, }; use crate::metadata::layout::{RafsBlobTable, RAFS_V5_ROOT_INODE}; -use crate::metadata::{RafsStore, RafsSuper, RafsSuperConfig}; +use crate::metadata::{RafsSuper, RafsSuperConfig}; pub(crate) const STARGZ_DEFAULT_BLOCK_SIZE: u32 = 4 << 20; -const WRITE_PADDING_DATA: [u8; 4096] = [0u8; 4096]; +/// RAFS bootstrap/meta blob builder. pub struct Bootstrap {} impl Bootstrap { - /// Create a new instance of `Bootstrap`. + /// Create a new instance of [Bootstrap]. pub fn new() -> Result { Ok(Self {}) } @@ -51,7 +39,7 @@ impl Bootstrap { /// - files/directories to be removed from the lower layer, at the head of the array. The order /// of removal operations are bottom-up, that means child files/directories is in front of its /// parent. - /// - files/directories to added/modified in into the lower layer, at the tail of the array. + /// - files/directories to added/modified into the lower layer, at the tail of the array. /// The order of addition/modification operations are top-down, that means directories is /// ahead its children. /// @@ -60,7 +48,7 @@ impl Bootstrap { &mut self, ctx: &mut BuildContext, bootstrap_ctx: &mut BootstrapContext, - tree: &mut Tree, + mut tree: Tree, ) -> Result<()> { // used to compute nid(ino) for v6 let root_offset = bootstrap_ctx.offset; @@ -77,10 +65,10 @@ impl Bootstrap { ctx.prefetch.insert_if_need(&tree.node); nodes.push(tree.node.clone()); - self.build_rafs(ctx, bootstrap_ctx, tree, &mut nodes)?; + Self::build_rafs(ctx, bootstrap_ctx, &mut tree, &mut nodes)?; if ctx.fs_version.is_v6() && !bootstrap_ctx.layered { // generate on-disk metadata layout for v6 - self.rafsv6_update_dirents(&mut nodes, tree, root_offset); + Self::v6_update_dirents(&mut nodes, &tree, root_offset); } bootstrap_ctx.nodes = nodes; @@ -128,11 +116,34 @@ impl Bootstrap { Ok(tree) } - #[allow(clippy::only_used_in_recursion)] + /// Dump the RAFS filesystem meta information to meta blob. + pub fn dump( + &mut self, + ctx: &mut BuildContext, + bootstrap_storage: &mut Option, + bootstrap_ctx: &mut BootstrapContext, + blob_table: &RafsBlobTable, + ) -> Result<()> { + match blob_table { + RafsBlobTable::V5(table) => self.v5_dump(ctx, bootstrap_ctx, table)?, + RafsBlobTable::V6(table) => self.v6_dump(ctx, bootstrap_ctx, table)?, + } + + if let Some(ArtifactStorage::FileDir(p)) = bootstrap_storage { + let bootstrap_data = bootstrap_ctx.writer.as_bytes()?; + let digest = RafsDigest::from_buf(&bootstrap_data, digest::Algorithm::Sha256); + let name = digest.to_string(); + bootstrap_ctx.writer.finalize(Some(name.clone()))?; + *bootstrap_storage = Some(ArtifactStorage::SingleFile(p.join(name))); + Ok(()) + } else { + bootstrap_ctx.writer.finalize(Some(String::default())) + } + } + /// Traverse node tree, set inode index, ino, child_index and child_count etc according to the /// RAFS metadata format, then store to nodes collection. fn build_rafs( - &mut self, ctx: &mut BuildContext, bootstrap_ctx: &mut BootstrapContext, tree: &mut Tree, @@ -140,6 +151,7 @@ impl Bootstrap { ) -> Result<()> { let index = nodes.len() as u32 + 1; let parent = &mut nodes[tree.node.index as usize - 1]; + let parent_ino = parent.inode.ino(); // Maybe the parent is not a directory in multi-layers build scenario, so we check here. if parent.is_dir() { @@ -156,22 +168,19 @@ impl Bootstrap { if ctx.fs_version.is_v6() { tree.node.v6_offset = parent.v6_offset; - // alignment for inode, which is 32 bytes; - bootstrap_ctx.align_offset(EROFS_INODE_SLOT_SIZE as u64); } // Cache dir tree for BFS walk let mut dirs: Vec<&mut Tree> = Vec::new(); - let parent_ino = parent.inode.ino(); - for child in tree.children.iter_mut() { let index = nodes.len() as u64 + 1; child.node.index = index; child.node.inode.set_parent(parent_ino); - // Hardlink handle, all hardlink nodes' ino, nlink should be the same, - // because the real_ino may be conflicted between different layers, - // so we need to find hardlink node index list in the layer where the node is located. + // Handle hardlink. + // All hardlink nodes' ino and nlink should be the same. + // We need to find hardlink node index list in the layer where the node is located + // because the real_ino may be different among different layers, let mut v6_hardlink_offset: Option = None; if let Some(indexes) = bootstrap_ctx.inode_map.get_mut(&( child.node.layer_idx, @@ -208,7 +217,6 @@ impl Bootstrap { child .node .v6_set_offset(bootstrap_ctx, v6_hardlink_offset)?; - bootstrap_ctx.align_offset(EROFS_INODE_SLOT_SIZE as u64); } // Store node for bootstrap & blob dump. @@ -240,9 +248,7 @@ impl Bootstrap { } nodes.push(child.node.clone()); } - _ => { - nodes.push(child.node.clone()); - } + _ => nodes.push(child.node.clone()), } ctx.prefetch.insert_if_need(&child.node); @@ -257,63 +263,17 @@ impl Bootstrap { // updating parent directory's nlink here is reliable since builder re-constructs // the entire tree and intends to layout all inodes into a plain array fetching // from the previously applied tree. - let parent_dir = &mut nodes[tree.node.index as usize - 1]; - parent_dir.inode.set_nlink((2 + dirs.len()) as u32); - + let parent = &mut nodes[tree.node.index as usize - 1]; + if parent.is_dir() { + parent.inode.set_nlink((2 + dirs.len()) as u32); + } for dir in dirs { - self.build_rafs(ctx, bootstrap_ctx, dir, nodes)?; + Self::build_rafs(ctx, bootstrap_ctx, dir, nodes)?; } Ok(()) } - #[allow(clippy::only_used_in_recursion)] - /// Rafsv6 update offset - fn rafsv6_update_dirents(&self, nodes: &mut Vec, tree: &mut Tree, parent_offset: u64) { - let node = &mut nodes[tree.node.index as usize - 1]; - let node_offset = node.v6_offset; - if !node.is_dir() { - return; - } - - // dot & dotdot - // Type of libc::S_IFDIR is u16 on macos, so it need a conversion - // but compiler will report useless conversion on linux platform, - // so we add an allow annotation here. - #[allow(clippy::useless_conversion)] - { - node.v6_dirents - .push((node_offset, OsString::from("."), libc::S_IFDIR.into())); - node.v6_dirents - .push((parent_offset, OsString::from(".."), libc::S_IFDIR.into())); - } - - let mut dirs: Vec<&mut Tree> = Vec::new(); - for child in tree.children.iter_mut() { - trace!( - "{:?} child {:?} offset {}, mode {}", - node.name(), - child.node.name(), - child.node.v6_offset, - child.node.inode.mode() - ); - node.v6_dirents.push(( - child.node.v6_offset, - child.node.name().to_os_string(), - child.node.inode.mode(), - )); - if child.node.is_dir() { - dirs.push(child); - } - } - node.v6_dirents - .sort_unstable_by(|a, b| a.1.as_os_str().cmp(b.1.as_os_str()) as std::cmp::Ordering); - - for dir in dirs { - self.rafsv6_update_dirents(nodes, dir, node_offset); - } - } - fn load_parent_bootstrap( &mut self, ctx: &mut BuildContext, @@ -347,448 +307,4 @@ impl Bootstrap { Ok(tree) } - - /// Calculate inode digest for directory. - fn rafsv5_digest_node( - &self, - ctx: &mut BuildContext, - bootstrap_ctx: &mut BootstrapContext, - index: usize, - ) { - let node = &bootstrap_ctx.nodes[index]; - - // We have set digest for non-directory inode in the previous dump_blob workflow. - if node.is_dir() { - let child_index = node.inode.child_index(); - let child_count = node.inode.child_count(); - let mut inode_hasher = RafsDigest::hasher(ctx.digester); - - for idx in child_index..child_index + child_count { - let child = &bootstrap_ctx.nodes[(idx - 1) as usize]; - inode_hasher.digest_update(child.inode.digest().as_ref()); - } - - bootstrap_ctx.nodes[index] - .inode - .set_digest(inode_hasher.digest_finalize()); - } - } - - pub fn dump( - &mut self, - ctx: &mut BuildContext, - bootstrap_storage: &mut Option, - bootstrap_ctx: &mut BootstrapContext, - blob_table: &RafsBlobTable, - ) -> Result<()> { - match blob_table { - RafsBlobTable::V5(table) => self.rafsv5_dump(ctx, bootstrap_ctx, table)?, - RafsBlobTable::V6(table) => self.rafsv6_dump(ctx, bootstrap_ctx, table)?, - } - - if let Some(ArtifactStorage::FileDir(p)) = bootstrap_storage { - let bootstrap_data = bootstrap_ctx.writer.as_bytes()?; - let digest = RafsDigest::from_buf(&bootstrap_data, digest::Algorithm::Sha256); - let name = digest.to_string(); - bootstrap_ctx.writer.finalize(Some(name.clone()))?; - *bootstrap_storage = Some(ArtifactStorage::SingleFile(p.join(name))); - Ok(()) - } else { - bootstrap_ctx.writer.finalize(Some(String::default())) - } - } - - /// Dump bootstrap and blob file, return (Vec, blob_size) - fn rafsv5_dump( - &mut self, - ctx: &mut BuildContext, - bootstrap_ctx: &mut BootstrapContext, - blob_table: &RafsV5BlobTable, - ) -> Result<()> { - // Set inode digest, use reverse iteration order to reduce repeated digest calculations. - for idx in (0..bootstrap_ctx.nodes.len()).rev() { - self.rafsv5_digest_node(ctx, bootstrap_ctx, idx); - } - - // Set inode table - let super_block_size = size_of::(); - let inode_table_entries = bootstrap_ctx.nodes.len() as u32; - let mut inode_table = RafsV5InodeTable::new(inode_table_entries as usize); - let inode_table_size = inode_table.size(); - - // Set prefetch table - let (prefetch_table_size, prefetch_table_entries) = if let Some(prefetch_table) = - ctx.prefetch.get_rafsv5_prefetch_table(&bootstrap_ctx.nodes) - { - (prefetch_table.size(), prefetch_table.len() as u32) - } else { - (0, 0u32) - }; - - // Set blob table, use sha256 string (length 64) as blob id if not specified - let prefetch_table_offset = super_block_size + inode_table_size; - let blob_table_offset = prefetch_table_offset + prefetch_table_size; - let blob_table_size = blob_table.size(); - let extended_blob_table_offset = blob_table_offset + blob_table_size; - let extended_blob_table_size = blob_table.extended.size(); - let extended_blob_table_entries = blob_table.extended.entries(); - - // Set super block - let mut super_block = RafsV5SuperBlock::new(); - let inodes_count = bootstrap_ctx.inode_map.len() as u64; - super_block.set_inodes_count(inodes_count); - super_block.set_inode_table_offset(super_block_size as u64); - super_block.set_inode_table_entries(inode_table_entries); - super_block.set_blob_table_offset(blob_table_offset as u64); - super_block.set_blob_table_size(blob_table_size as u32); - super_block.set_extended_blob_table_offset(extended_blob_table_offset as u64); - super_block.set_extended_blob_table_entries(u32::try_from(extended_blob_table_entries)?); - super_block.set_prefetch_table_offset(prefetch_table_offset as u64); - super_block.set_prefetch_table_entries(prefetch_table_entries); - super_block.set_compressor(ctx.compressor); - super_block.set_digester(ctx.digester); - super_block.set_chunk_size(ctx.chunk_size); - if ctx.explicit_uidgid { - super_block.set_explicit_uidgid(); - } - if ctx.conversion_type == ConversionType::EStargzIndexToRef { - super_block.set_block_size(STARGZ_DEFAULT_BLOCK_SIZE); - } - - // Set inodes and chunks - let mut inode_offset = (super_block_size - + inode_table_size - + prefetch_table_size - + blob_table_size - + extended_blob_table_size) as u32; - - let mut has_xattr = false; - for node in &mut bootstrap_ctx.nodes { - inode_table.set(node.index, inode_offset)?; - // Add inode size - inode_offset += node.inode.inode_size() as u32; - if node.inode.has_xattr() { - has_xattr = true; - if !node.info.xattrs.is_empty() { - inode_offset += (size_of::() - + node.info.xattrs.aligned_size_v5()) - as u32; - } - } - // Add chunks size - if node.is_reg() { - inode_offset += node.inode.child_count() * size_of::() as u32; - } - } - if has_xattr { - super_block.set_has_xattr(); - } - - // Dump super block - super_block - .store(bootstrap_ctx.writer.as_mut()) - .context("failed to store superblock")?; - - // Dump inode table - inode_table - .store(bootstrap_ctx.writer.as_mut()) - .context("failed to store inode table")?; - - // Dump prefetch table - if let Some(mut prefetch_table) = - ctx.prefetch.get_rafsv5_prefetch_table(&bootstrap_ctx.nodes) - { - prefetch_table - .store(bootstrap_ctx.writer.as_mut()) - .context("failed to store prefetch table")?; - } - - // Dump blob table - blob_table - .store(bootstrap_ctx.writer.as_mut()) - .context("failed to store blob table")?; - - // Dump extended blob table - blob_table - .store_extended(bootstrap_ctx.writer.as_mut()) - .context("failed to store extended blob table")?; - - // Dump inodes and chunks - timing_tracer!( - { - for node in &bootstrap_ctx.nodes { - node.dump_bootstrap_v5(ctx, bootstrap_ctx.writer.as_mut()) - .context("failed to dump bootstrap")?; - } - - Ok(()) - }, - "dump_bootstrap", - Result<()> - )?; - - Ok(()) - } - - /// Dump bootstrap and blob file, return (Vec, blob_size) - fn rafsv6_dump( - &mut self, - ctx: &mut BuildContext, - bootstrap_ctx: &mut BootstrapContext, - blob_table: &RafsV6BlobTable, - ) -> Result<()> { - // Rafs v6 disk layout - // - // EROFS_SUPER_OFFSET - // | - // +---+---------+------------+-------------+----------------------------------------------+ - // | | | | | | | | - // |1k |super |extended | blob table | prefetch table | inodes | chunk info table | - // | |block |superblock+ | | | | | - // | | |devslot | | | | | - // +---+---------+------------+-------------+----------------------------------------------+ - - let blobs = blob_table.get_all(); - let devtable_len = blobs.len() * size_of::(); - let blob_table_size = blob_table.size() as u64; - let blob_table_offset = align_offset( - (EROFS_DEVTABLE_OFFSET as u64) + devtable_len as u64, - EROFS_BLOCK_SIZE as u64, - ); - let blob_table_entries = blobs.len(); - assert!(blob_table_entries < u8::MAX as usize); - trace!( - "devtable len {} blob table offset {} blob table size {}", - devtable_len, - blob_table_offset, - blob_table_size - ); - - let (prefetch_table_offset, prefetch_table_size) = - // If blob_table_size equal to 0, there is no prefetch. - if ctx.prefetch.fs_prefetch_rule_count() > 0 && blob_table_size > 0 { - // Prefetch table is very close to blob devices table - let offset = blob_table_offset + blob_table_size; - // Each prefetched file has is nid of `u32` filled into prefetch table. - let size = ctx.prefetch.fs_prefetch_rule_count() * size_of::() as u32; - trace!("prefetch table locates at offset {} size {}", offset, size); - (offset, size) - } else { - (0, 0) - }; - - // Make the superblock's meta_blkaddr one block ahead of the inode table, - // to avoid using 0 as root nid. - // inode offset = meta_blkaddr * block_size + 32 * nid - // When using nid 0 as root nid, - // the root directory will not be shown by glibc's getdents/readdir. - // Because in some OS, ino == 0 represents corresponding file is deleted. - let orig_meta_addr = bootstrap_ctx.nodes[0].v6_offset - EROFS_BLOCK_SIZE; - let meta_addr = if blob_table_size > 0 { - align_offset( - blob_table_offset + blob_table_size + prefetch_table_size as u64, - EROFS_BLOCK_SIZE as u64, - ) - } else { - orig_meta_addr - }; - - // get devt_slotoff - let root_nid = calculate_nid( - bootstrap_ctx.nodes[0].v6_offset + (meta_addr - orig_meta_addr), - meta_addr, - ); - - // Prepare extended super block - let ext_sb_offset = bootstrap_ctx.writer.seek_current(0)?; - let mut ext_sb = RafsV6SuperBlockExt::new(); - ext_sb.set_compressor(ctx.compressor); - ext_sb.set_digester(ctx.digester); - ext_sb.set_chunk_size(ctx.chunk_size); - ext_sb.set_blob_table_offset(blob_table_offset); - ext_sb.set_blob_table_size(blob_table_size as u32); - - // collect all chunks in this bootstrap. - // HashChunkDict cannot be used here, because there will be duplicate chunks between layers, - // but there is no deduplication during the actual construction. - // Each layer uses the corresponding chunk in the blob of its own layer. - // If HashChunkDict is used here, it will cause duplication. The chunks are removed, - // resulting in incomplete chunk info. - let mut chunk_cache = BTreeMap::new(); - - // Dump bootstrap - timing_tracer!( - { - for node in &mut bootstrap_ctx.nodes { - node.dump_bootstrap_v6( - ctx, - bootstrap_ctx.writer.as_mut(), - orig_meta_addr, - meta_addr, - &mut chunk_cache, - ) - .context("failed to dump bootstrap")?; - } - - Ok(()) - }, - "dump_bootstrap", - Result<()> - )?; - Self::rafsv6_align_to_block(bootstrap_ctx)?; - - // `Node` offset might be updated during above inodes dumping. So `get_prefetch_table` after it. - let prefetch_table = ctx - .prefetch - .get_rafsv6_prefetch_table(&bootstrap_ctx.nodes, meta_addr); - if let Some(mut pt) = prefetch_table { - assert!(pt.len() * size_of::() <= prefetch_table_size as usize); - // Device slots are very close to extended super block. - ext_sb.set_prefetch_table_offset(prefetch_table_offset); - ext_sb.set_prefetch_table_size(prefetch_table_size); - bootstrap_ctx - .writer - .seek_offset(prefetch_table_offset as u64) - .context("failed seek prefetch table offset")?; - pt.store(bootstrap_ctx.writer.as_mut()).unwrap(); - } - - // TODO: get rid of the chunk info array. - // Dump chunk info array. - let chunk_table_offset = bootstrap_ctx - .writer - .seek_to_end() - .context("failed to seek to bootstrap's end for chunk table")?; - let mut chunk_table_size: u64 = 0; - for (_, chunk) in chunk_cache.iter() { - let chunk_size = chunk - .store(bootstrap_ctx.writer.as_mut()) - .context("failed to dump chunk table")?; - chunk_table_size += chunk_size as u64; - } - ext_sb.set_chunk_table(chunk_table_offset, chunk_table_size); - debug!( - "chunk_table offset {} size {}", - chunk_table_offset, chunk_table_size - ); - Self::rafsv6_align_to_block(bootstrap_ctx)?; - - // Prepare device slots. - let pos = bootstrap_ctx - .writer - .seek_to_end() - .context("failed to seek to bootstrap's end for chunk table")?; - assert_eq!(pos % EROFS_BLOCK_SIZE, 0); - let mut mapped_blkaddr = Self::align_mapped_blkaddr((pos / EROFS_BLOCK_SIZE) as u32); - let mut devtable: Vec = Vec::new(); - let mut block_count = 0u32; - let mut inlined_chunk_digest = true; - for entry in blobs.iter() { - let mut devslot = RafsV6Device::new(); - // blob id is String, which is processed by sha256.finalize(). - if entry.blob_id().is_empty() { - bail!(" blob id is empty"); - } else if entry.blob_id().len() > 64 { - bail!(format!( - "blob id length is bigger than 64 bytes, blob id {:?}", - entry.blob_id() - )); - } else if entry.uncompressed_size() / EROFS_BLOCK_SIZE > u32::MAX as u64 { - bail!(format!( - "uncompressed blob size (0x:{:x}) is too big", - entry.uncompressed_size() - )); - } - if !entry.has_feature(BlobFeatures::INLINED_CHUNK_DIGEST) { - inlined_chunk_digest = false; - } - let cnt = (entry.uncompressed_size() / EROFS_BLOCK_SIZE) as u32; - assert!(block_count.checked_add(cnt).is_some()); - block_count += cnt; - let id = entry.blob_id(); - let id = id.as_bytes(); - let mut blob_id = [0u8; 64]; - blob_id[..id.len()].copy_from_slice(id); - devslot.set_blob_id(&blob_id); - devslot.set_blocks(cnt); - devslot.set_mapped_blkaddr(mapped_blkaddr); - devtable.push(devslot); - - mapped_blkaddr = Self::align_mapped_blkaddr(mapped_blkaddr + cnt); - } - - // Dump super block - let mut sb = RafsV6SuperBlock::new(); - sb.set_inos(bootstrap_ctx.nodes.len() as u64); - sb.set_blocks(block_count); - sb.set_root_nid(root_nid as u16); - sb.set_meta_addr(meta_addr); - sb.set_extra_devices(blob_table_entries as u16); - bootstrap_ctx.writer.seek(SeekFrom::Start(0))?; - sb.store(bootstrap_ctx.writer.as_mut()) - .context("failed to store SB")?; - - // Dump extended super block. - if ctx.explicit_uidgid { - ext_sb.set_explicit_uidgid(); - } - if ctx.has_xattr { - ext_sb.set_has_xattr(); - } - if inlined_chunk_digest { - ext_sb.set_inlined_chunk_digest(); - } - bootstrap_ctx.writer.seek(SeekFrom::Start(ext_sb_offset))?; - ext_sb - .store(bootstrap_ctx.writer.as_mut()) - .context("failed to store extended super block")?; - - // Dump device slots. - bootstrap_ctx - .writer - .seek_offset(EROFS_DEVTABLE_OFFSET as u64) - .context("failed to seek devtslot")?; - for slot in devtable.iter() { - slot.store(bootstrap_ctx.writer.as_mut()) - .context("failed to store device slot")?; - } - - // Dump blob table - bootstrap_ctx - .writer - .seek_offset(blob_table_offset as u64) - .context("failed seek for extended blob table offset")?; - blob_table - .store(bootstrap_ctx.writer.as_mut()) - .context("failed to store extended blob table")?; - - Ok(()) - } - - fn rafsv6_align_to_block(bootstrap_ctx: &mut BootstrapContext) -> Result<()> { - bootstrap_ctx - .writer - .flush() - .context("failed to flush bootstrap")?; - let pos = bootstrap_ctx - .writer - .seek_to_end() - .context("failed to seek to bootstrap's end for chunk table")?; - let padding = align_offset(pos, EROFS_BLOCK_SIZE as u64) - pos; - bootstrap_ctx - .writer - .write_all(&WRITE_PADDING_DATA[0..padding as usize]) - .context("failed to write 0 to padding of bootstrap's end for chunk table")?; - bootstrap_ctx - .writer - .flush() - .context("failed to flush bootstrap")?; - Ok(()) - } - - fn align_mapped_blkaddr(addr: u32) -> u32 { - // TODO: define a const in nydus-service for 0x20_0000 - let blocks = (0x20_0000u64 / EROFS_BLOCK_SIZE) as u32; - (addr + blocks - 1) / blocks * blocks - } } diff --git a/rafs/src/builder/core/v5.rs b/rafs/src/builder/core/v5.rs index a303fc56f51..ca4b9f9f160 100644 --- a/rafs/src/builder/core/v5.rs +++ b/rafs/src/builder/core/v5.rs @@ -3,13 +3,21 @@ // // SPDX-License-Identifier: Apache-2.0 +use std::convert::TryFrom; +use std::mem::size_of; + use anyhow::{bail, Context, Result}; -use nydus_utils::try_round_up_4k; +use nydus_utils::digest::{DigestHasher, RafsDigest}; +use nydus_utils::{root_tracer, timing_tracer, try_round_up_4k}; +use super::bootstrap::STARGZ_DEFAULT_BLOCK_SIZE; use super::node::Node; -use crate::builder::{BuildContext, Tree}; +use crate::builder::{Bootstrap, BootstrapContext, BuildContext, ConversionType, Tree}; use crate::metadata::inode::InodeWrapper; -use crate::metadata::layout::v5::RafsV5InodeWrapper; +use crate::metadata::layout::v5::{ + RafsV5BlobTable, RafsV5ChunkInfo, RafsV5InodeTable, RafsV5InodeWrapper, RafsV5SuperBlock, + RafsV5XAttrsTable, +}; use crate::metadata::{RafsStore, RafsVersion}; use crate::RafsIoWrite; @@ -97,3 +105,163 @@ impl Node { self.set_inode_blocks(); } } + +impl Bootstrap { + /// Calculate inode digest for directory. + fn v5_digest_node( + &self, + ctx: &mut BuildContext, + bootstrap_ctx: &mut BootstrapContext, + index: usize, + ) { + let node = &bootstrap_ctx.nodes[index]; + + // We have set digest for non-directory inode in the previous dump_blob workflow. + if node.is_dir() { + let child_index = node.inode.child_index(); + let child_count = node.inode.child_count(); + let mut inode_hasher = RafsDigest::hasher(ctx.digester); + + for idx in child_index..child_index + child_count { + let child = &bootstrap_ctx.nodes[(idx - 1) as usize]; + inode_hasher.digest_update(child.inode.digest().as_ref()); + } + + bootstrap_ctx.nodes[index] + .inode + .set_digest(inode_hasher.digest_finalize()); + } + } + + /// Dump bootstrap and blob file, return (Vec, blob_size) + pub(crate) fn v5_dump( + &mut self, + ctx: &mut BuildContext, + bootstrap_ctx: &mut BootstrapContext, + blob_table: &RafsV5BlobTable, + ) -> Result<()> { + // Set inode digest, use reverse iteration order to reduce repeated digest calculations. + for idx in (0..bootstrap_ctx.nodes.len()).rev() { + self.v5_digest_node(ctx, bootstrap_ctx, idx); + } + + // Set inode table + let super_block_size = size_of::(); + let inode_table_entries = bootstrap_ctx.nodes.len() as u32; + let mut inode_table = RafsV5InodeTable::new(inode_table_entries as usize); + let inode_table_size = inode_table.size(); + + // Set prefetch table + let (prefetch_table_size, prefetch_table_entries) = if let Some(prefetch_table) = + ctx.prefetch.get_rafsv5_prefetch_table(&bootstrap_ctx.nodes) + { + (prefetch_table.size(), prefetch_table.len() as u32) + } else { + (0, 0u32) + }; + + // Set blob table, use sha256 string (length 64) as blob id if not specified + let prefetch_table_offset = super_block_size + inode_table_size; + let blob_table_offset = prefetch_table_offset + prefetch_table_size; + let blob_table_size = blob_table.size(); + let extended_blob_table_offset = blob_table_offset + blob_table_size; + let extended_blob_table_size = blob_table.extended.size(); + let extended_blob_table_entries = blob_table.extended.entries(); + + // Set super block + let mut super_block = RafsV5SuperBlock::new(); + let inodes_count = bootstrap_ctx.inode_map.len() as u64; + super_block.set_inodes_count(inodes_count); + super_block.set_inode_table_offset(super_block_size as u64); + super_block.set_inode_table_entries(inode_table_entries); + super_block.set_blob_table_offset(blob_table_offset as u64); + super_block.set_blob_table_size(blob_table_size as u32); + super_block.set_extended_blob_table_offset(extended_blob_table_offset as u64); + super_block.set_extended_blob_table_entries(u32::try_from(extended_blob_table_entries)?); + super_block.set_prefetch_table_offset(prefetch_table_offset as u64); + super_block.set_prefetch_table_entries(prefetch_table_entries); + super_block.set_compressor(ctx.compressor); + super_block.set_digester(ctx.digester); + super_block.set_chunk_size(ctx.chunk_size); + if ctx.explicit_uidgid { + super_block.set_explicit_uidgid(); + } + if ctx.conversion_type == ConversionType::EStargzIndexToRef { + super_block.set_block_size(STARGZ_DEFAULT_BLOCK_SIZE); + } + + // Set inodes and chunks + let mut inode_offset = (super_block_size + + inode_table_size + + prefetch_table_size + + blob_table_size + + extended_blob_table_size) as u32; + + let mut has_xattr = false; + for node in &mut bootstrap_ctx.nodes { + inode_table.set(node.index, inode_offset)?; + // Add inode size + inode_offset += node.inode.inode_size() as u32; + if node.inode.has_xattr() { + has_xattr = true; + if !node.info.xattrs.is_empty() { + inode_offset += (size_of::() + + node.info.xattrs.aligned_size_v5()) + as u32; + } + } + // Add chunks size + if node.is_reg() { + inode_offset += node.inode.child_count() * size_of::() as u32; + } + } + if has_xattr { + super_block.set_has_xattr(); + } + + // Dump super block + super_block + .store(bootstrap_ctx.writer.as_mut()) + .context("failed to store superblock")?; + + // Dump inode table + inode_table + .store(bootstrap_ctx.writer.as_mut()) + .context("failed to store inode table")?; + + // Dump prefetch table + if let Some(mut prefetch_table) = + ctx.prefetch.get_rafsv5_prefetch_table(&bootstrap_ctx.nodes) + { + prefetch_table + .store(bootstrap_ctx.writer.as_mut()) + .context("failed to store prefetch table")?; + } + + // Dump blob table + blob_table + .store(bootstrap_ctx.writer.as_mut()) + .context("failed to store blob table")?; + + // Dump extended blob table + blob_table + .store_extended(bootstrap_ctx.writer.as_mut()) + .context("failed to store extended blob table")?; + + // Dump inodes and chunks + timing_tracer!( + { + for node in &bootstrap_ctx.nodes { + node.dump_bootstrap_v5(ctx, bootstrap_ctx.writer.as_mut()) + .context("failed to dump bootstrap")?; + } + + Ok(()) + }, + "dump_bootstrap", + Result<()> + )?; + + Ok(()) + } +} diff --git a/rafs/src/builder/core/v6.rs b/rafs/src/builder/core/v6.rs index 612028badd2..ce907a4b226 100644 --- a/rafs/src/builder/core/v6.rs +++ b/rafs/src/builder/core/v6.rs @@ -11,20 +11,26 @@ use std::os::unix::ffi::OsStrExt; use std::sync::Arc; use anyhow::{bail, ensure, Context, Result}; -use nydus_utils::{div_round_up, round_down_4k, round_up}; +use nydus_utils::{div_round_up, root_tracer, round_down_4k, round_up, timing_tracer}; +use storage::device::BlobFeatures; use super::chunk_dict::DigestWithBlobIndex; use super::node::Node; -use crate::builder::{BootstrapContext, BuildContext, Tree}; +use crate::builder::{Bootstrap, BootstrapContext, BuildContext, Tree}; use crate::metadata::chunk::ChunkWrapper; use crate::metadata::inode::new_v6_inode; use crate::metadata::layout::v6::{ - align_offset, calculate_nid, RafsV6Dirent, RafsV6InodeChunkAddr, RafsV6InodeChunkHeader, - RafsV6OndiskInode, EROFS_BLOCK_SIZE, EROFS_INODE_CHUNK_BASED, EROFS_INODE_FLAT_INLINE, - EROFS_INODE_FLAT_PLAIN, + align_offset, calculate_nid, RafsV6BlobTable, RafsV6Device, RafsV6Dirent, RafsV6InodeChunkAddr, + RafsV6InodeChunkHeader, RafsV6OndiskInode, RafsV6SuperBlock, RafsV6SuperBlockExt, + EROFS_BLOCK_SIZE, EROFS_DEVTABLE_OFFSET, EROFS_INODE_CHUNK_BASED, EROFS_INODE_FLAT_INLINE, + EROFS_INODE_FLAT_PLAIN, EROFS_INODE_SLOT_SIZE, EROFS_SUPER_BLOCK_SIZE, EROFS_SUPER_OFFSET, }; +use crate::metadata::RafsStore; use crate::RafsIoWrite; +const WRITE_PADDING_DATA: [u8; 4096] = [0u8; 4096]; +const V6_BLOCK_SEG_ALIGNMENT: u64 = 0x20_0000; + // Rafs v6 dedicated methods impl Node { /// Dump RAFS v6 inode metadata to meta blob. @@ -119,6 +125,7 @@ impl Node { self.v6_offset = bootstrap_ctx.offset; bootstrap_ctx.offset += self.v6_size_with_xattr(); } + bootstrap_ctx.align_offset(EROFS_INODE_SLOT_SIZE as u64); Ok(()) } @@ -138,6 +145,7 @@ impl Node { // Dir isize is the total bytes of 'dirents + names'. self.inode.set_size(d_size); self.v6_set_offset_with_tail(bootstrap_ctx, d_size); + bootstrap_ctx.align_offset(EROFS_INODE_SLOT_SIZE as u64); Ok(()) } @@ -532,6 +540,327 @@ impl BuildContext { } } +impl Bootstrap { + pub(crate) fn v6_update_dirents(nodes: &mut Vec, tree: &Tree, parent_offset: u64) { + let node = &mut nodes[tree.node.index as usize - 1]; + let node_offset = node.v6_offset; + if !node.is_dir() { + return; + } + + // dot & dotdot + // Type of libc::S_IFDIR is u16 on macos, so it need a conversion + // but compiler will report useless conversion on linux platform, + // so we add an allow annotation here. + #[allow(clippy::useless_conversion)] + { + node.v6_dirents + .push((node_offset, OsString::from("."), libc::S_IFDIR.into())); + node.v6_dirents + .push((parent_offset, OsString::from(".."), libc::S_IFDIR.into())); + } + + let mut dirs: Vec<&Tree> = Vec::new(); + for child in tree.children.iter() { + trace!( + "{:?} child {:?} offset {}, mode {}", + node.name(), + child.node.name(), + child.node.v6_offset, + child.node.inode.mode() + ); + node.v6_dirents.push(( + child.node.v6_offset, + child.node.name().to_os_string(), + child.node.inode.mode(), + )); + if child.node.is_dir() { + dirs.push(child); + } + } + node.v6_dirents + .sort_unstable_by(|a, b| a.1.as_os_str().cmp(b.1.as_os_str()) as std::cmp::Ordering); + + for dir in dirs { + Self::v6_update_dirents(nodes, dir, node_offset); + } + } + + /// Dump bootstrap and blob file, return (Vec, blob_size) + pub(crate) fn v6_dump( + &mut self, + ctx: &mut BuildContext, + bootstrap_ctx: &mut BootstrapContext, + blob_table: &RafsV6BlobTable, + ) -> Result<()> { + // Rafs v6 disk layout + // + // EROFS_SUPER_OFFSET + // | + // +---+---------+------------+-------------+----------------------------------------------+ + // | | | | | | | | + // |1k |super |extended | blob table | prefetch table | inodes | chunk info table | + // | |block |superblock+ | | | | | + // | | |devslot | | | | | + // +---+---------+------------+-------------+----------------------------------------------+ + + let blobs = blob_table.get_all(); + let devtable_len = blobs.len() * size_of::(); + let blob_table_size = blob_table.size() as u64; + let blob_table_offset = align_offset( + (EROFS_DEVTABLE_OFFSET as u64) + devtable_len as u64, + EROFS_BLOCK_SIZE as u64, + ); + let blob_table_entries = blobs.len(); + assert!(blob_table_entries < u8::MAX as usize); + trace!( + "devtable len {} blob table offset {} blob table size {}", + devtable_len, + blob_table_offset, + blob_table_size + ); + + let (prefetch_table_offset, prefetch_table_size) = + // If blob_table_size equal to 0, there is no prefetch. + if ctx.prefetch.fs_prefetch_rule_count() > 0 && blob_table_size > 0 { + // Prefetch table is very close to blob devices table + let offset = blob_table_offset + blob_table_size; + // Each prefetched file has is nid of `u32` filled into prefetch table. + let size = ctx.prefetch.fs_prefetch_rule_count() * size_of::() as u32; + trace!("prefetch table locates at offset {} size {}", offset, size); + (offset, size) + } else { + (0, 0) + }; + + // Make the superblock's meta_blkaddr one block ahead of the inode table, + // to avoid using 0 as root nid. + // inode offset = meta_blkaddr * block_size + 32 * nid + // When using nid 0 as root nid, + // the root directory will not be shown by glibc's getdents/readdir. + // Because in some OS, ino == 0 represents corresponding file is deleted. + let orig_meta_addr = bootstrap_ctx.nodes[0].v6_offset - EROFS_BLOCK_SIZE; + let meta_addr = if blob_table_size > 0 { + align_offset( + blob_table_offset + blob_table_size + prefetch_table_size as u64, + EROFS_BLOCK_SIZE as u64, + ) + } else { + orig_meta_addr + }; + + // get devt_slotoff + let root_nid = calculate_nid( + bootstrap_ctx.nodes[0].v6_offset + (meta_addr - orig_meta_addr), + meta_addr, + ); + + // Prepare extended super block + let mut ext_sb = RafsV6SuperBlockExt::new(); + ext_sb.set_compressor(ctx.compressor); + ext_sb.set_digester(ctx.digester); + ext_sb.set_chunk_size(ctx.chunk_size); + ext_sb.set_blob_table_offset(blob_table_offset); + ext_sb.set_blob_table_size(blob_table_size as u32); + + // collect all chunks in this bootstrap. + // HashChunkDict cannot be used here, because there will be duplicate chunks between layers, + // but there is no deduplication during the actual construction. + // Each layer uses the corresponding chunk in the blob of its own layer. + // If HashChunkDict is used here, it will cause duplication. The chunks are removed, + // resulting in incomplete chunk info. + let mut chunk_cache = BTreeMap::new(); + + // Dump bootstrap + timing_tracer!( + { + for node in &mut bootstrap_ctx.nodes { + node.dump_bootstrap_v6( + ctx, + bootstrap_ctx.writer.as_mut(), + orig_meta_addr, + meta_addr, + &mut chunk_cache, + ) + .context("failed to dump bootstrap")?; + } + + Ok(()) + }, + "dump_bootstrap", + Result<()> + )?; + Self::v6_align_to_block(bootstrap_ctx)?; + + // `Node` offset might be updated during above inodes dumping. So `get_prefetch_table` after it. + let prefetch_table = ctx + .prefetch + .get_rafsv6_prefetch_table(&bootstrap_ctx.nodes, meta_addr); + if let Some(mut pt) = prefetch_table { + assert!(pt.len() * size_of::() <= prefetch_table_size as usize); + // Device slots are very close to extended super block. + ext_sb.set_prefetch_table_offset(prefetch_table_offset); + ext_sb.set_prefetch_table_size(prefetch_table_size); + bootstrap_ctx + .writer + .seek_offset(prefetch_table_offset as u64) + .context("failed seek prefetch table offset")?; + pt.store(bootstrap_ctx.writer.as_mut()).unwrap(); + } + + // TODO: get rid of the chunk info array. + // Dump chunk info array. + let chunk_table_offset = bootstrap_ctx + .writer + .seek_to_end() + .context("failed to seek to bootstrap's end for chunk table")?; + let mut chunk_table_size: u64 = 0; + for (_, chunk) in chunk_cache.iter() { + let chunk_size = chunk + .store(bootstrap_ctx.writer.as_mut()) + .context("failed to dump chunk table")?; + chunk_table_size += chunk_size as u64; + } + ext_sb.set_chunk_table(chunk_table_offset, chunk_table_size); + debug!( + "chunk_table offset {} size {}", + chunk_table_offset, chunk_table_size + ); + Self::v6_align_to_block(bootstrap_ctx)?; + + // Prepare device slots. + let mut pos = bootstrap_ctx + .writer + .seek_to_end() + .context("failed to seek to bootstrap's end for chunk table")?; + assert_eq!(pos % EROFS_BLOCK_SIZE, 0); + let mut devtable: Vec = Vec::new(); + let mut block_count = 0u32; + let mut inlined_chunk_digest = true; + for entry in blobs.iter() { + let mut devslot = RafsV6Device::new(); + // blob id is String, which is processed by sha256.finalize(). + if entry.blob_id().is_empty() { + bail!(" blob id is empty"); + } else if entry.blob_id().len() > 64 { + bail!(format!( + "blob id length is bigger than 64 bytes, blob id {:?}", + entry.blob_id() + )); + } else if entry.uncompressed_size() / ctx.v6_block_size() > u32::MAX as u64 { + bail!(format!( + "uncompressed blob size (0x:{:x}) is too big", + entry.uncompressed_size() + )); + } + if !entry.has_feature(BlobFeatures::INLINED_CHUNK_DIGEST) { + inlined_chunk_digest = false; + } + let cnt = (entry.uncompressed_size() / ctx.v6_block_size()) as u32; + if block_count.checked_add(cnt).is_none() { + bail!("Too many data blocks in RAFS filesystem, block size 0x{:x}, block count 0x{:x}", ctx.v6_block_size(), block_count as u64 + cnt as u64); + } + let mapped_blkaddr = Self::v6_align_mapped_blkaddr(ctx, pos)?; + pos += cnt as u64 * ctx.v6_block_size(); + block_count += cnt; + + let id = entry.blob_id(); + let id = id.as_bytes(); + let mut blob_id = [0u8; 64]; + blob_id[..id.len()].copy_from_slice(id); + devslot.set_blob_id(&blob_id); + devslot.set_blocks(cnt); + devslot.set_mapped_blkaddr(mapped_blkaddr); + devtable.push(devslot); + } + + // Dump super block + let mut sb = RafsV6SuperBlock::new(); + sb.set_inos(bootstrap_ctx.nodes.len() as u64); + sb.set_blocks(block_count); + sb.set_root_nid(root_nid as u16); + sb.set_meta_addr(meta_addr); + sb.set_extra_devices(blob_table_entries as u16); + bootstrap_ctx.writer.seek(SeekFrom::Start(0))?; + sb.store(bootstrap_ctx.writer.as_mut()) + .context("failed to store SB")?; + + // Dump extended super block. + if ctx.explicit_uidgid { + ext_sb.set_explicit_uidgid(); + } + if ctx.has_xattr { + ext_sb.set_has_xattr(); + } + if inlined_chunk_digest { + ext_sb.set_inlined_chunk_digest(); + } + bootstrap_ctx + .writer + .seek_offset((EROFS_SUPER_OFFSET + EROFS_SUPER_BLOCK_SIZE) as u64) + .context("failed to seek for extended super block")?; + ext_sb + .store(bootstrap_ctx.writer.as_mut()) + .context("failed to store extended super block")?; + + // Dump device slots. + bootstrap_ctx + .writer + .seek_offset(EROFS_DEVTABLE_OFFSET as u64) + .context("failed to seek devtslot")?; + for slot in devtable.iter() { + slot.store(bootstrap_ctx.writer.as_mut()) + .context("failed to store device slot")?; + } + + // Dump blob table + bootstrap_ctx + .writer + .seek_offset(blob_table_offset as u64) + .context("failed seek for extended blob table offset")?; + blob_table + .store(bootstrap_ctx.writer.as_mut()) + .context("failed to store extended blob table")?; + + Ok(()) + } + + fn v6_align_to_block(bootstrap_ctx: &mut BootstrapContext) -> Result<()> { + bootstrap_ctx + .writer + .flush() + .context("failed to flush bootstrap")?; + let pos = bootstrap_ctx + .writer + .seek_to_end() + .context("failed to seek to bootstrap's end for chunk table")?; + let padding = align_offset(pos, EROFS_BLOCK_SIZE as u64) - pos; + bootstrap_ctx + .writer + .write_all(&WRITE_PADDING_DATA[0..padding as usize]) + .context("failed to write 0 to padding of bootstrap's end for chunk table")?; + bootstrap_ctx + .writer + .flush() + .context("failed to flush bootstrap")?; + Ok(()) + } + + fn v6_align_mapped_blkaddr(ctx: &BuildContext, addr: u64) -> Result { + match addr.checked_add(V6_BLOCK_SEG_ALIGNMENT - 1) { + None => bail!("address 0x{:x} is too big", addr), + Some(v) => { + let v = (v & !(V6_BLOCK_SEG_ALIGNMENT - 1)) / ctx.v6_block_size(); + if v > u32::MAX as u64 { + bail!("address 0x{:x} is too big", addr); + } else { + Ok(v as u32) + } + } + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -642,7 +971,7 @@ mod tests { assert_eq!(dir_node.v6_offset, 8192 + 4096 + 8192 + 8192 + 4096 + 8192); assert_eq!( bootstrap_ctx.offset, - 8192 + 4096 + 8192 + 8192 + 4096 + 8192 + 32 + 1985 + 8192 + 4096 + 8192 + 8192 + 4096 + 8192 + 32 + 1985 + 31 ); bootstrap_ctx.align_offset(EROFS_INODE_SLOT_SIZE as u64); diff --git a/rafs/src/builder/mod.rs b/rafs/src/builder/mod.rs index 0c636da51b4..e8ecec9b801 100644 --- a/rafs/src/builder/mod.rs +++ b/rafs/src/builder/mod.rs @@ -55,7 +55,7 @@ fn build_bootstrap( let origin_bootstarp_offset = bootstrap_ctx.offset; // Disable prefetch and bootstrap.apply() will reset the prefetch enable/disable flag. ctx.prefetch.disable(); - bootstrap.build(ctx, bootstrap_ctx, &mut tree)?; + bootstrap.build(ctx, bootstrap_ctx, tree)?; tree = bootstrap.apply(ctx, bootstrap_ctx, bootstrap_mgr, blob_mgr, None)?; bootstrap_ctx.offset = origin_bootstarp_offset; bootstrap_ctx.layered = false; @@ -63,7 +63,7 @@ fn build_bootstrap( // Convert the hierarchy tree into an array, stored in `bootstrap_ctx.nodes`. timing_tracer!( - { bootstrap.build(ctx, bootstrap_ctx, &mut tree) }, + { bootstrap.build(ctx, bootstrap_ctx, tree) }, "build_bootstrap" )?; diff --git a/rafs/src/metadata/layout/v6.rs b/rafs/src/metadata/layout/v6.rs index 6cf13afd01b..a302c015286 100644 --- a/rafs/src/metadata/layout/v6.rs +++ b/rafs/src/metadata/layout/v6.rs @@ -535,7 +535,6 @@ impl RafsV6SuperBlockExt { impl RafsStore for RafsV6SuperBlockExt { fn store(&self, w: &mut dyn RafsIoWrite) -> Result { - w.seek_offset((EROFS_SUPER_OFFSET + EROFS_SUPER_BLOCK_SIZE) as u64)?; w.write_all(self.as_ref())?; w.seek_offset(EROFS_BLOCK_SIZE as u64)?; diff --git a/src/bin/nydus-image/merge.rs b/src/bin/nydus-image/merge.rs index 5764d539b14..7db09351e7a 100644 --- a/src/bin/nydus-image/merge.rs +++ b/src/bin/nydus-image/merge.rs @@ -243,7 +243,7 @@ impl Merger { } // Safe to unwrap because there is at least one source bootstrap. - let mut tree = tree.unwrap(); + let tree = tree.unwrap(); ctx.fs_version = fs_version; if let Some(chunk_size) = chunk_size { ctx.chunk_size = chunk_size; @@ -251,7 +251,7 @@ impl Merger { let mut bootstrap_ctx = BootstrapContext::new(Some(target.clone()), false)?; let mut bootstrap = Bootstrap::new()?; - bootstrap.build(ctx, &mut bootstrap_ctx, &mut tree)?; + bootstrap.build(ctx, &mut bootstrap_ctx, tree)?; let blob_table = blob_mgr.to_blob_table(ctx)?; let mut bootstrap_storage = Some(target.clone()); bootstrap From 2b3fcc0244632d73e2093ff4d4697a75739375c7 Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Sun, 5 Mar 2023 01:03:43 +0800 Subject: [PATCH 10/10] rafs: refine prefetch and chunk dictionary in builder Refine prefetch and chunk dictionary in builder for maintenance. Signed-off-by: Jiang Liu --- rafs/src/builder/compact.rs | 4 +- rafs/src/builder/core/blob.rs | 7 ++- rafs/src/builder/core/chunk_dict.rs | 46 ++++++++++----- rafs/src/builder/core/context.rs | 75 +++++++++++++++---------- rafs/src/builder/core/feature.rs | 86 +++++++++++++++++++---------- rafs/src/builder/core/layout.rs | 3 +- rafs/src/builder/core/node.rs | 2 +- rafs/src/builder/core/prefetch.rs | 13 ++++- rafs/src/builder/core/tree.rs | 1 + rafs/src/builder/core/v5.rs | 6 +- rafs/src/builder/core/v6.rs | 2 +- rafs/src/builder/mod.rs | 5 +- rafs/src/builder/stargz.rs | 2 +- src/bin/nydus-image/main.rs | 17 ++++-- src/bin/nydus-image/merge.rs | 2 +- 15 files changed, 174 insertions(+), 97 deletions(-) diff --git a/rafs/src/builder/compact.rs b/rafs/src/builder/compact.rs index 960ef12bad7..1c6040e5c8a 100644 --- a/rafs/src/builder/compact.rs +++ b/rafs/src/builder/compact.rs @@ -514,7 +514,7 @@ impl BlobCompactor { if blob_idx != idx as u32 { self.apply_blob_move(idx as u32, blob_idx)?; } - self.new_blob_mgr.add(ctx); + self.new_blob_mgr.add_blob(ctx); } State::Delete => { info!("delete blob {}", ori_blob_ids[idx]); @@ -543,7 +543,7 @@ impl BlobCompactor { self.apply_chunk_change(change_chunk)?; } info!("rebuild blob {} successfully", blob_ctx.blob_id); - self.new_blob_mgr.add(blob_ctx); + self.new_blob_mgr.add_blob(blob_ctx); } } } diff --git a/rafs/src/builder/core/blob.rs b/rafs/src/builder/core/blob.rs index 7ae92071a2c..128bb758b7e 100644 --- a/rafs/src/builder/core/blob.rs +++ b/rafs/src/builder/core/blob.rs @@ -20,11 +20,12 @@ use crate::builder::{ }; use crate::metadata::RAFS_MAX_CHUNK_SIZE; -pub struct Blob {} +/// Generator for RAFS data blob. +pub(crate) struct Blob {} impl Blob { /// Dump blob file and generate chunks - pub fn dump( + pub(crate) fn dump( ctx: &BuildContext, nodes: &mut [Node], blob_mgr: &mut BlobManager, @@ -202,7 +203,7 @@ impl Blob { )?; } - // Generate ToC entry for `blob.meta`. + // Generate ToC entry for `blob.meta` and write chunk digest array. if ctx.features.is_enabled(Feature::BlobToc) { let mut hasher = RafsDigest::hasher(digest::Algorithm::Sha256); let ci_data = if ctx.blob_features.contains(BlobFeatures::ZRAN) { diff --git a/rafs/src/builder/core/chunk_dict.rs b/rafs/src/builder/core/chunk_dict.rs index 4fb47ce3dfe..06f9e1a3a17 100644 --- a/rafs/src/builder/core/chunk_dict.rs +++ b/rafs/src/builder/core/chunk_dict.rs @@ -21,13 +21,27 @@ use crate::metadata::{RafsSuper, RafsSuperConfig}; #[derive(Debug, PartialEq, Eq, Hash, Ord, PartialOrd)] pub struct DigestWithBlobIndex(pub RafsDigest, pub u32); +/// Trait to manage chunk cache for chunk deduplication. pub trait ChunkDict: Sync + Send + 'static { + /// Add a chunk into the cache. fn add_chunk(&mut self, chunk: Arc, digester: digest::Algorithm); + + /// Get a cached chunk from the cache. fn get_chunk(&self, digest: &RafsDigest, uncompressed_size: u32) -> Option<&Arc>; + + /// Get all `BlobInfo` objects referenced by cached chunks. fn get_blobs(&self) -> Vec>; + + /// Get the `BlobInfo` object with inner index `idx`. fn get_blob_by_inner_idx(&self, idx: u32) -> Option<&Arc>; + + /// Associate an external index with the inner index. fn set_real_blob_idx(&self, inner_idx: u32, out_idx: u32); + + /// Get the external index associated with an inner index. fn get_real_blob_idx(&self, inner_idx: u32) -> Option; + + /// Get the digest algorithm used to generate chunk digest. fn digester(&self) -> digest::Algorithm; } @@ -63,6 +77,7 @@ impl ChunkDict for () { } } +/// An implementation of [ChunkDict] based on [HashMap]. pub struct HashChunkDict { m: HashMap, AtomicU32)>, blobs: Vec>, @@ -113,6 +128,7 @@ impl ChunkDict for HashChunkDict { } impl HashChunkDict { + /// Create a new instance of [HashChunkDict]. pub fn new(digester: digest::Algorithm) -> Self { HashChunkDict { m: Default::default(), @@ -122,11 +138,24 @@ impl HashChunkDict { } } + /// Get an immutable reference to the internal `HashMap`. pub fn hashmap(&self) -> &HashMap, AtomicU32)> { &self.m } - fn from_bootstrap_file( + /// Parse commandline argument for chunk dictionary and load chunks into the dictionary. + pub fn from_commandline_arg( + arg: &str, + config: Arc, + rafs_config: &RafsSuperConfig, + ) -> Result> { + let file_path = parse_chunk_dict_arg(arg)?; + HashChunkDict::from_bootstrap_file(&file_path, config, rafs_config) + .map(|d| Arc::new(d) as Arc) + } + + /// Load chunks from the RAFS filesystem into the chunk dictionary. + pub fn from_bootstrap_file( path: &Path, config: Arc, rafs_config: &RafsSuperConfig, @@ -205,17 +234,6 @@ pub fn parse_chunk_dict_arg(arg: &str) -> Result { } } -/// Load a chunk dictionary from external source. -pub fn import_chunk_dict( - arg: &str, - config: Arc, - rafs_config: &RafsSuperConfig, -) -> Result> { - let file_path = parse_chunk_dict_arg(arg)?; - HashChunkDict::from_bootstrap_file(&file_path, config, rafs_config) - .map(|d| Arc::new(d) as Arc) -} - #[cfg(test)] mod tests { use super::*; @@ -247,7 +265,9 @@ mod tests { chunk_size: 0x100000, explicit_uidgid: true, }; - let dict = import_chunk_dict(path, Arc::new(ConfigV2::default()), &rafs_config).unwrap(); + let dict = + HashChunkDict::from_commandline_arg(path, Arc::new(ConfigV2::default()), &rafs_config) + .unwrap(); assert!(dict.get_chunk(&RafsDigest::default(), 0).is_none()); assert_eq!(dict.get_blobs().len(), 18); diff --git a/rafs/src/builder/core/context.rs b/rafs/src/builder/core/context.rs index e25ef161da0..4bc6d0e1e3d 100644 --- a/rafs/src/builder/core/context.rs +++ b/rafs/src/builder/core/context.rs @@ -47,6 +47,7 @@ use crate::RafsIoWrite; // TODO: select BufWriter capacity by performance testing. pub const BUF_WRITER_CAPACITY: usize = 2 << 17; +/// Filesystem conversion type supported by RAFS builder. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum ConversionType { DirectoryToRafs, @@ -123,6 +124,7 @@ impl ConversionType { } } +/// Filesystem based storage configuration for artifacts. #[derive(Debug, Clone)] pub enum ArtifactStorage { // Won't rename user's specification @@ -132,6 +134,7 @@ pub enum ArtifactStorage { } impl ArtifactStorage { + /// Show file path to store the generated artifacts. pub fn display(&self) -> Display { match self { ArtifactStorage::SingleFile(p) => p.display(), @@ -148,7 +151,7 @@ impl Default for ArtifactStorage { /// ArtifactMemoryWriter provides a writer to allow writing bootstrap /// data to a byte slice in memory. -pub struct ArtifactMemoryWriter(Cursor>); +struct ArtifactMemoryWriter(Cursor>); impl Default for ArtifactMemoryWriter { fn default() -> Self { @@ -183,7 +186,7 @@ impl Write for ArtifactMemoryWriter { } } -pub struct ArtifactFileWriter(ArtifactWriter); +struct ArtifactFileWriter(ArtifactWriter); impl RafsIoWrite for ArtifactFileWriter { fn as_any(&self) -> &dyn Any { @@ -246,6 +249,7 @@ impl Write for ArtifactWriter { } impl ArtifactWriter { + /// Create a new instance of [ArtifactWriter] from a [ArtifactStorage] configuration object. pub fn new(storage: ArtifactStorage) -> Result { match storage { ArtifactStorage::SingleFile(ref p) => { @@ -298,6 +302,7 @@ impl ArtifactWriter { } } + /// Get the current write position. pub fn pos(&self) -> Result { Ok(self.pos as u64) } @@ -305,15 +310,12 @@ impl ArtifactWriter { // The `inline-bootstrap` option merges the blob and bootstrap into one // file. We need some header to index the location of the blob and bootstrap, // write_tar_header uses tar header that arranges the data as follows: - // data | tar_header | data | tar_header - // This is a tar-like structure, except that we put the tar header after the // data. The advantage is that we do not need to determine the size of the data // first, so that we can write the blob data by stream without seek to improve // the performance of the blob dump by using fifo. - pub fn write_tar_header(&mut self, name: &str, size: u64) -> Result

{ - debug!("dump rafs blob tar header {} {}", name, size); + fn write_tar_header(&mut self, name: &str, size: u64) -> Result
{ let mut header = Header::new_gnu(); header.set_path(Path::new(name))?; header.set_entry_type(EntryType::Regular); @@ -407,6 +409,7 @@ pub struct BlobContext { } impl BlobContext { + /// Create a new instance of [BlobContext]. pub fn new( blob_id: String, blob_offset: u64, @@ -479,6 +482,7 @@ impl BlobContext { blob_ctx } + /// Create a new instance of [BlobContext] from `BlobInfo` object. pub fn from(ctx: &BuildContext, blob: &BlobInfo, chunk_source: ChunkSource) -> Result { let mut compressed_blob_size = blob.compressed_size(); let mut blob_meta_size = blob.blob_meta_size(); @@ -594,6 +598,7 @@ impl BlobContext { Ok(blob_ctx) } + /// Set chunk size for the blob. pub fn set_chunk_size(&mut self, chunk_size: u32) { self.chunk_size = chunk_size; } @@ -713,14 +718,15 @@ pub struct BlobManager { current_blob_index: Option, /// Chunk dictionary to hold chunks from an extra chunk dict file. /// Used for chunk data de-duplication within the whole image. - pub global_chunk_dict: Arc, + pub(crate) global_chunk_dict: Arc, /// Chunk dictionary to hold chunks from all layers. /// Used for chunk data de-duplication between layers (with `--parent-bootstrap`) /// or within layer (with `--inline-bootstrap`). - pub layered_chunk_dict: HashChunkDict, + pub(crate) layered_chunk_dict: HashChunkDict, } impl BlobManager { + /// Create a new instance of [BlobManager]. pub fn new(digester: digest::Algorithm) -> Self { Self { blobs: Vec::new(), @@ -744,6 +750,7 @@ impl BlobManager { Ok(blob_ctx) } + /// Get the current blob object or create one if no current blob available. pub fn get_or_create_current_blob( &mut self, ctx: &BuildContext, @@ -751,12 +758,13 @@ impl BlobManager { if self.current_blob_index.is_none() { let blob_ctx = Self::new_blob_ctx(ctx)?; self.current_blob_index = Some(self.alloc_index()?); - self.add(blob_ctx); + self.add_blob(blob_ctx); } // Safe to unwrap because the blob context has been added. Ok(self.get_current_blob().unwrap()) } + /// Get the current blob object. pub fn get_current_blob(&mut self) -> Option<(u32, &mut BlobContext)> { if let Some(idx) = self.current_blob_index { Some((idx, &mut self.blobs[idx as usize])) @@ -765,10 +773,12 @@ impl BlobManager { } } + /// Set the global chunk dictionary for chunk deduplication. pub fn set_chunk_dict(&mut self, dict: Arc) { self.global_chunk_dict = dict } + /// Get the global chunk dictionary for chunk deduplication. pub fn get_chunk_dict(&self) -> Arc { self.global_chunk_dict.clone() } @@ -783,21 +793,23 @@ impl BlobManager { .with_context(|| Error::msg("too many blobs")) } - /// Add a blob context to manager - /// - /// This should be paired with Self::alloc_index() and keep in consistence. - pub fn add(&mut self, blob_ctx: BlobContext) { - self.blobs.push(blob_ctx); - } - + /// Get number of blobs managed by the manager. pub fn len(&self) -> usize { self.blobs.len() } + /// Check whether there's managed blobs. pub fn is_empty(&self) -> bool { self.blobs.is_empty() } + /// Add a blob context to manager + /// + /// This should be paired with Self::alloc_index() and keep in consistence. + pub fn add_blob(&mut self, blob_ctx: BlobContext) { + self.blobs.push(blob_ctx); + } + /// Get all blob contexts (include the blob context that does not have a blob). pub fn get_blobs(&self) -> Vec<&BlobContext> { self.blobs.iter().collect() @@ -864,7 +876,7 @@ impl BlobManager { } else { let idx = self.alloc_index()?; let ctx = BlobContext::from(ctx, blob.as_ref(), ChunkSource::Dict)?; - self.add(ctx); + self.add_blob(ctx); self.global_chunk_dict .set_real_blob_idx(blob.blob_index(), idx); } @@ -873,6 +885,7 @@ impl BlobManager { Ok(()) } + /// Generate a [RafsBlobTable] from all blobs managed by the manager. pub fn to_blob_table(&self, build_ctx: &BuildContext) -> Result { let mut blob_table = match build_ctx.fs_version { RafsVersion::V5 => RafsBlobTable::V5(RafsV5BlobTable::new()), @@ -930,22 +943,23 @@ impl BlobManager { } } -/// BootstrapContext is used to hold inmemory data of bootstrap during build. +/// BootstrapContext is used to hold in memory data of bootstrap during build. pub struct BootstrapContext { /// This build has a parent bootstrap. pub layered: bool, /// Cache node index for hardlinks, HashMap<(layer_index, real_inode, dev), Vec>. - pub inode_map: HashMap<(u16, Inode, u64), Vec>, - /// Store all nodes in ascendant ordor, indexed by (node.index - 1). + pub(crate) inode_map: HashMap<(u16, Inode, u64), Vec>, + /// Store all nodes in ascendant order, indexed by (node.index - 1). pub nodes: Vec, /// Current position to write in f_bootstrap - pub offset: u64, - pub writer: Box, + pub(crate) offset: u64, + pub(crate) writer: Box, /// Not fully used blocks - pub v6_available_blocks: Vec>, + pub(crate) v6_available_blocks: Vec>, } impl BootstrapContext { + /// Create a new instance of [BootstrapContext]. pub fn new(storage: Option, layered: bool) -> Result { let writer = if let Some(storage) = storage { Box::new(ArtifactFileWriter(ArtifactWriter::new(storage)?)) as Box @@ -965,6 +979,7 @@ impl BootstrapContext { }) } + /// Align the write position. pub fn align_offset(&mut self, align_size: u64) { if self.offset % align_size > 0 { self.offset = div_round_up(self.offset, align_size) * align_size; @@ -975,7 +990,7 @@ impl BootstrapContext { // Try to find an used block with no less than `size` space left. // If found it, return the offset where we can store data. // If not, return 0. - pub fn allocate_available_block(&mut self, size: u64) -> u64 { + pub(crate) fn allocate_available_block(&mut self, size: u64) -> u64 { if size >= EROFS_BLOCK_SIZE { return 0; } @@ -996,7 +1011,7 @@ impl BootstrapContext { } // Append the block that `offset` belongs to corresponding deque. - pub fn append_available_block(&mut self, offset: u64) { + pub(crate) fn append_available_block(&mut self, offset: u64) { if offset % EROFS_BLOCK_SIZE != 0 { let avail = EROFS_BLOCK_SIZE - offset % EROFS_BLOCK_SIZE; let idx = avail as usize / EROFS_INODE_SLOT_SIZE; @@ -1005,14 +1020,14 @@ impl BootstrapContext { } } -/// BootstrapManager is used to hold the parent bootstrap reader and create -/// new bootstrap context. +/// BootstrapManager is used to hold the parent bootstrap reader and create new bootstrap context. pub struct BootstrapManager { - pub f_parent_path: Option, - pub bootstrap_storage: Option, + pub(crate) f_parent_path: Option, + pub(crate) bootstrap_storage: Option, } impl BootstrapManager { + /// Create a new instance of [BootstrapManager] pub fn new(bootstrap_storage: Option, f_parent_path: Option) -> Self { Self { f_parent_path: f_parent_path.map(PathBuf::from), @@ -1020,6 +1035,7 @@ impl BootstrapManager { } } + /// Create a new instance of [BootstrapContext] pub fn create_ctx(&self) -> Result { BootstrapContext::new(self.bootstrap_storage.clone(), self.f_parent_path.is_some()) } @@ -1199,6 +1215,7 @@ impl fmt::Display for BuildOutput { } impl BuildOutput { + /// Create a new instance of [BuildOutput]. pub fn new( blob_mgr: &BlobManager, bootstrap_storage: &Option, diff --git a/rafs/src/builder/core/feature.rs b/rafs/src/builder/core/feature.rs index 41b4e690ff9..e5743f66f07 100644 --- a/rafs/src/builder/core/feature.rs +++ b/rafs/src/builder/core/feature.rs @@ -2,20 +2,37 @@ // // SPDX-License-Identifier: Apache-2.0 -use anyhow::{bail, Result}; use std::collections::HashSet; use std::convert::TryFrom; +use anyhow::{bail, Result}; + const ERR_UNSUPPORTED_FEATURE: &str = "unsupported feature"; -/// Feature bits for RAFS filesystem builder. -#[derive(Clone, Hash, PartialEq, Eq)] +/// Feature flags to control behavior of RAFS filesystem builder. +#[derive(Clone, Debug, Hash, PartialEq, Eq)] pub enum Feature { - /// Enable to append TOC footer to rafs blob. + /// Append a Table Of Content footer to RAFS v6 data blob, to help locate data sections. BlobToc, } -/// Feature set for RAFS filesystem builder. +impl TryFrom<&str> for Feature { + type Error = anyhow::Error; + + fn try_from(f: &str) -> Result { + match f { + "blob-toc" => Ok(Self::BlobToc), + _ => bail!( + "{} `{}`, please try upgrading to the latest nydus-image", + ERR_UNSUPPORTED_FEATURE, + f, + ), + } + } +} + +/// A set of enabled feature flags to control behavior of RAFS filesystem builder +#[derive(Clone, Debug)] pub struct Features(HashSet); impl Default for Features { @@ -30,37 +47,48 @@ impl Features { Self(HashSet::new()) } - /// Create a new instance of [Features] from a string. - pub fn from(features: &str) -> Result { - let mut list = Features::new(); - let features = features.trim(); - if features.is_empty() { - return Ok(list); - } - for feat in features.split(',') { - let feature = Feature::try_from(feat.trim())?; - list.0.insert(feature); - } - Ok(list) - } - - /// Check whether feature is enabled or not. + /// Check whether a feature is enabled or not. pub fn is_enabled(&self, feature: Feature) -> bool { self.0.contains(&feature) } } -impl TryFrom<&str> for Feature { +impl TryFrom<&str> for Features { type Error = anyhow::Error; - fn try_from(f: &str) -> Result { - match f { - "blob-toc" => Ok(Self::BlobToc), - _ => bail!( - "{} `{}`, please try upgrading to the latest nydus-image", - ERR_UNSUPPORTED_FEATURE, - f, - ), + fn try_from(features: &str) -> Result { + let mut list = Features::new(); + for feat in features.trim().split(',') { + if !feat.is_empty() { + let feature = Feature::try_from(feat.trim())?; + list.0.insert(feature); + } } + Ok(list) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_feature() { + assert_eq!(Feature::try_from("blob-toc").unwrap(), Feature::BlobToc); + Feature::try_from("unknown-feature-bit").unwrap_err(); + } + + #[test] + fn test_features() { + let features = Features::try_from("blob-toc").unwrap(); + assert!(features.is_enabled(Feature::BlobToc)); + let features = Features::try_from("blob-toc,").unwrap(); + assert!(features.is_enabled(Feature::BlobToc)); + let features = Features::try_from("blob-toc, ").unwrap(); + assert!(features.is_enabled(Feature::BlobToc)); + let features = Features::try_from("blob-toc ").unwrap(); + assert!(features.is_enabled(Feature::BlobToc)); + let features = Features::try_from(" blob-toc ").unwrap(); + assert!(features.is_enabled(Feature::BlobToc)); } } diff --git a/rafs/src/builder/core/layout.rs b/rafs/src/builder/core/layout.rs index ad48423b3ee..1d0346298c4 100644 --- a/rafs/src/builder/core/layout.rs +++ b/rafs/src/builder/core/layout.rs @@ -5,8 +5,7 @@ use anyhow::Result; use super::node::Node; -use super::overlay::Overlay; -use crate::builder::Prefetch; +use crate::builder::{Overlay, Prefetch}; #[derive(Clone)] pub struct BlobLayout {} diff --git a/rafs/src/builder/core/node.rs b/rafs/src/builder/core/node.rs index dcbf12df7d3..c03286ee061 100644 --- a/rafs/src/builder/core/node.rs +++ b/rafs/src/builder/core/node.rs @@ -434,7 +434,7 @@ impl Node { dict.set_real_blob_idx(chunk.blob_index(), blob_idx); if let Some(blob) = dict.get_blob_by_inner_idx(chunk.blob_index()) { let ctx = BlobContext::from(ctx, blob, ChunkSource::Dict)?; - blob_mgr.add(ctx); + blob_mgr.add_blob(ctx); } blob_idx }; diff --git a/rafs/src/builder/core/prefetch.rs b/rafs/src/builder/core/prefetch.rs index ccdc760cd5d..4391a74e527 100644 --- a/rafs/src/builder/core/prefetch.rs +++ b/rafs/src/builder/core/prefetch.rs @@ -13,6 +13,7 @@ use super::node::Node; use crate::metadata::layout::v5::RafsV5PrefetchTable; use crate::metadata::layout::v6::{calculate_nid, RafsV6PrefetchTable}; +/// Filesystem data prefetch policy. #[derive(Clone, Copy, Debug, PartialEq)] pub enum PrefetchPolicy { None, @@ -99,6 +100,7 @@ fn generate_patterns(input: Vec) -> Result Ok(patterns) } +/// Manage filesystem data prefetch configuration and state for builder. #[derive(Default, Clone)] pub struct Prefetch { pub policy: PrefetchPolicy, @@ -115,6 +117,7 @@ pub struct Prefetch { } impl Prefetch { + /// Create a new instance of [Prefetch]. pub fn new(policy: PrefetchPolicy) -> Result { let patterns = if policy != PrefetchPolicy::None { get_patterns().context("failed to get prefetch patterns")? @@ -130,6 +133,7 @@ impl Prefetch { }) } + /// Insert node into the prefetch list if it matches prefetch rules. pub fn insert_if_need(&mut self, node: &Node) { let path = node.target(); let index = node.index; @@ -157,14 +161,17 @@ impl Prefetch { } } + /// Check whether the node is in the prefetch list. pub fn contains(&self, node: &Node) -> bool { self.files.contains_key(node.target()) } + /// Get node index array of files in the prefetch list. pub fn get_file_indexes(&self) -> Vec { self.files.values().copied().collect() } + /// Get number of prefetch rules. pub fn fs_prefetch_rule_count(&self) -> u32 { if self.policy == PrefetchPolicy::Fs { self.patterns.values().len() as u32 @@ -174,7 +181,7 @@ impl Prefetch { } /// Generate filesystem layer prefetch list for RAFS v5. - pub fn get_rafsv5_prefetch_table(&mut self, nodes: &[Node]) -> Option { + pub fn get_v5_prefetch_table(&mut self, nodes: &[Node]) -> Option { if self.policy == PrefetchPolicy::Fs { let mut prefetch_table = RafsV5PrefetchTable::new(); for i in self.patterns.values().filter_map(|v| *v) { @@ -190,7 +197,7 @@ impl Prefetch { } /// Generate filesystem layer prefetch list for RAFS v6. - pub fn get_rafsv6_prefetch_table( + pub fn get_v6_prefetch_table( &mut self, nodes: &[Node], meta_addr: u64, @@ -222,10 +229,12 @@ impl Prefetch { } } + /// Disable filesystem data prefetch. pub fn disable(&mut self) { self.disabled = true; } + /// Reset to initialization state. pub fn clear(&mut self) { self.disabled = false; self.files.clear(); diff --git a/rafs/src/builder/core/tree.rs b/rafs/src/builder/core/tree.rs index b98809a2865..627ade87832 100644 --- a/rafs/src/builder/core/tree.rs +++ b/rafs/src/builder/core/tree.rs @@ -1,4 +1,5 @@ // Copyright 2020 Ant Group. All rights reserved. +// Copyright 2023 Alibaba Cloud. All rights reserved. // // SPDX-License-Identifier: Apache-2.0 diff --git a/rafs/src/builder/core/v5.rs b/rafs/src/builder/core/v5.rs index ca4b9f9f160..ba0c72d7a67 100644 --- a/rafs/src/builder/core/v5.rs +++ b/rafs/src/builder/core/v5.rs @@ -153,7 +153,7 @@ impl Bootstrap { // Set prefetch table let (prefetch_table_size, prefetch_table_entries) = if let Some(prefetch_table) = - ctx.prefetch.get_rafsv5_prefetch_table(&bootstrap_ctx.nodes) + ctx.prefetch.get_v5_prefetch_table(&bootstrap_ctx.nodes) { (prefetch_table.size(), prefetch_table.len() as u32) } else { @@ -230,9 +230,7 @@ impl Bootstrap { .context("failed to store inode table")?; // Dump prefetch table - if let Some(mut prefetch_table) = - ctx.prefetch.get_rafsv5_prefetch_table(&bootstrap_ctx.nodes) - { + if let Some(mut prefetch_table) = ctx.prefetch.get_v5_prefetch_table(&bootstrap_ctx.nodes) { prefetch_table .store(bootstrap_ctx.writer.as_mut()) .context("failed to store prefetch table")?; diff --git a/rafs/src/builder/core/v6.rs b/rafs/src/builder/core/v6.rs index ce907a4b226..3347cbb9261 100644 --- a/rafs/src/builder/core/v6.rs +++ b/rafs/src/builder/core/v6.rs @@ -695,7 +695,7 @@ impl Bootstrap { // `Node` offset might be updated during above inodes dumping. So `get_prefetch_table` after it. let prefetch_table = ctx .prefetch - .get_rafsv6_prefetch_table(&bootstrap_ctx.nodes, meta_addr); + .get_v6_prefetch_table(&bootstrap_ctx.nodes, meta_addr); if let Some(mut pt) = prefetch_table { assert!(pt.len() * size_of::() <= prefetch_table_size as usize); // Device slots are very close to extended super block. diff --git a/rafs/src/builder/mod.rs b/rafs/src/builder/mod.rs index e8ecec9b801..b2f6eb93bf2 100644 --- a/rafs/src/builder/mod.rs +++ b/rafs/src/builder/mod.rs @@ -11,14 +11,13 @@ use sha2::Digest; pub use self::compact::BlobCompactor; pub use self::core::bootstrap::Bootstrap; -pub use self::core::chunk_dict::{import_chunk_dict, parse_chunk_dict_arg}; -pub use self::core::chunk_dict::{ChunkDict, HashChunkDict}; +pub use self::core::chunk_dict::{parse_chunk_dict_arg, ChunkDict, HashChunkDict}; pub use self::core::context::{ ArtifactStorage, ArtifactWriter, BlobContext, BlobManager, BootstrapContext, BootstrapManager, BuildContext, BuildOutput, ConversionType, }; pub use self::core::feature::{Feature, Features}; -pub use self::core::node::ChunkSource; +pub use self::core::node::{ChunkSource, NodeChunk}; pub use self::core::overlay::{Overlay, WhiteoutSpec}; pub use self::core::prefetch::{Prefetch, PrefetchPolicy}; pub use self::core::tree::{MetadataTreeBuilder, Tree}; diff --git a/rafs/src/builder/stargz.rs b/rafs/src/builder/stargz.rs index bc55a80d47c..3dce73b54e9 100644 --- a/rafs/src/builder/stargz.rs +++ b/rafs/src/builder/stargz.rs @@ -840,7 +840,7 @@ impl Builder for StargzBuilder { // Dump blob meta Blob::dump_meta_data(ctx, &mut blob_ctx, blob_writer.as_mut().unwrap())?; if blob_ctx.uncompressed_blob_size > 0 { - blob_mgr.add(blob_ctx); + blob_mgr.add_blob(blob_ctx); } // Dump bootstrap file diff --git a/src/bin/nydus-image/main.rs b/src/bin/nydus-image/main.rs index 915e73907c1..95d800bd715 100644 --- a/src/bin/nydus-image/main.rs +++ b/src/bin/nydus-image/main.rs @@ -14,6 +14,7 @@ extern crate serde_json; #[macro_use] extern crate lazy_static; +use std::convert::TryFrom; use std::fs::{self, metadata, DirEntry, File, OpenOptions}; use std::os::unix::fs::FileTypeExt; use std::path::{Path, PathBuf}; @@ -27,9 +28,9 @@ use nydus::get_build_time_info; use nydus_api::{BuildTimeInfo, ConfigV2, LocalFsConfig}; use nydus_app::setup_logging; use nydus_rafs::builder::{ - import_chunk_dict, parse_chunk_dict_arg, ArtifactStorage, BlobCompactor, BlobManager, - BootstrapManager, BuildContext, BuildOutput, Builder, ConversionType, DirectoryBuilder, - Feature, Features, Prefetch, PrefetchPolicy, StargzBuilder, TarballBuilder, WhiteoutSpec, + parse_chunk_dict_arg, ArtifactStorage, BlobCompactor, BlobManager, BootstrapManager, + BuildContext, BuildOutput, Builder, ConversionType, DirectoryBuilder, Feature, Features, + HashChunkDict, Prefetch, PrefetchPolicy, StargzBuilder, TarballBuilder, WhiteoutSpec, }; use nydus_rafs::metadata::{RafsSuper, RafsSuperConfig, RafsVersion}; use nydus_storage::backend::localfs::LocalFs; @@ -769,7 +770,7 @@ impl Command { } } - let features = Features::from( + let features = Features::try_from( matches .get_one::("features") .map(|s| s.as_str()) @@ -817,7 +818,7 @@ impl Command { // The separate chunk dict bootstrap doesn't support blob accessible. rafs_config.internal.set_blob_accessible(false); blob_mgr.set_chunk_dict(timing_tracer!( - { import_chunk_dict(chunk_dict_arg, rafs_config, &config,) }, + { HashChunkDict::from_commandline_arg(chunk_dict_arg, rafs_config, &config,) }, "import_chunk_dict" )?); } @@ -950,7 +951,11 @@ impl Command { info!("load bootstrap {:?} successfully", bootstrap_path); let chunk_dict = match matches.get_one::("chunk-dict") { None => None, - Some(args) => Some(import_chunk_dict(args, config, &rs.meta.get_config())?), + Some(args) => Some(HashChunkDict::from_commandline_arg( + args, + config, + &rs.meta.get_config(), + )?), }; let backend = Self::get_backend(matches, "compactor")?; diff --git a/src/bin/nydus-image/merge.rs b/src/bin/nydus-image/merge.rs index 7db09351e7a..e9aa154c60b 100644 --- a/src/bin/nydus-image/merge.rs +++ b/src/bin/nydus-image/merge.rs @@ -195,7 +195,7 @@ impl Merger { } if !found { blob_idx_map.push(blob_mgr.len() as u32); - blob_mgr.add(blob_ctx); + blob_mgr.add_blob(blob_ctx); } }