Skip to content

Commit

Permalink
Merge pull request #1137 from imeoer/converter-parent-bootstrap
Browse files Browse the repository at this point in the history 
builder: support `--parent-bootstrap` for merge
  • Loading branch information
@jiangliu
jiangliu committed Mar 9, 2023
2 parents c2e08c4 + a99a41f commit 3f33613
Show file tree
Hide file tree
Showing 8 changed files with 129 additions and 44 deletions.
10 changes: 0 additions & 10 deletions rafs/src/metadata/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -718,21 +718,11 @@ impl RafsSuper {
// Old converters extracts bootstraps from data blobs with inlined bootstrap
// use blob digest as the bootstrap file name. The last blob in the blob table from
// the bootstrap has wrong blod id, so we need to fix it.
let mut fixed = false;
let blobs = rs.superblock.get_blob_infos();
for blob in blobs.iter() {
// Fix blob id for new images with old converters.
if blob.has_feature(BlobFeatures::INLINED_FS_META) {
blob.set_blob_id_from_meta_path(path.as_ref())?;
fixed = true;
}
}
if !fixed && !blob_accessible && !blobs.is_empty() {
// Fix blob id for old images with old converters.
let last = blobs.len() - 1;
let blob = &blobs[last];
if !blob.has_feature(BlobFeatures::CAP_TAR_TOC) {
rs.set_blob_id_from_meta_path(path.as_ref())?;
}
}
}
Expand Down
10 changes: 6 additions & 4 deletions smoke/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@ go 1.18
require (
github.com/containerd/containerd v1.6.18
github.com/containerd/nydus-snapshotter v0.6.1
github.com/google/uuid v1.2.0
github.com/google/uuid v1.3.0
github.com/opencontainers/go-digest v1.0.0
github.com/pkg/errors v0.9.1
github.com/pkg/xattr v0.4.9
github.com/stretchr/testify v1.8.1
golang.org/x/sys v0.4.0
golang.org/x/sys v0.5.0
)

require (
Expand All @@ -29,9 +29,11 @@ require (
github.com/sirupsen/logrus v1.9.0 // indirect
go.opencensus.io v0.23.0 // indirect
golang.org/x/mod v0.8.0 // indirect
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
golang.org/x/sync v0.1.0 // indirect
google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd // indirect
google.golang.org/grpc v1.50.1 // indirect
google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

replace github.com/containerd/nydus-snapshotter => github.com/imeoer/nydus-snapshotter v0.3.35
23 changes: 12 additions & 11 deletions smoke/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,8 +180,6 @@ github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJ
github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c=
github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
github.com/containerd/nydus-snapshotter v0.6.1 h1:G7k7EwnjFa1fUC3ywkldDt2BC3mtBPrt+omFke/Vdhk=
github.com/containerd/nydus-snapshotter v0.6.1/go.mod h1:U9m10GYZKisnSKOdgIjfkU8Ad0UTSYJ6CpP3I0SJBD0=
github.com/containerd/stargz-snapshotter/estargz v0.4.1/go.mod h1:x7Q9dg9QYb4+ELgxmo4gBUeJB0tl5dqH1Sdz0nJU1QM=
github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
Expand Down Expand Up @@ -371,8 +369,9 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
Expand Down Expand Up @@ -404,6 +403,8 @@ github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ
github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
github.com/imeoer/nydus-snapshotter v0.3.35 h1:pOW0rQrji9TpO4wsN9klpR+LA3OoNzTenbY0mUkCrHI=
github.com/imeoer/nydus-snapshotter v0.3.35/go.mod h1:haLQbzvyeVZZ59xi/DqW4uesQCAHlRW9U+GGWpiSB4s=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
Expand Down Expand Up @@ -749,7 +750,7 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
Expand All @@ -766,8 +767,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
Expand Down Expand Up @@ -838,8 +839,8 @@ golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand All @@ -849,7 +850,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
Expand Down Expand Up @@ -948,8 +949,8 @@ google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfG
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 h1:hrbNEivu7Zn1pxvHk6MBrq9iE22woVILTHqexqBxe6I=
google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd h1:e0TwkXOdbnH/1x5rc5MZ/VYyiZ4v+RdVfrGMqEwT68I=
google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
Expand Down
55 changes: 54 additions & 1 deletion smoke/tests/native_layer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,6 @@ func (n *NativeLayerTestSuite) TestMakeLayers() test.Generator {
}

func (n *NativeLayerTestSuite) testMakeLayers(ctx tool.Context, t *testing.T) {

packOption := converter.PackOption{
BuilderPath: ctx.Binary.Builder,
Compressor: ctx.Build.Compressor,
Expand Down Expand Up @@ -154,6 +153,60 @@ func (n *NativeLayerTestSuite) testMakeLayers(ctx tool.Context, t *testing.T) {
lowerLayer.Overlay(t, upperLayer)
ctx.Env.BootstrapPath = overlayBootstrap
tool.Verify(t, ctx, lowerLayer.FileTree)

// Make base layers (use as a parent bootstrap)
packOption.ChunkDictPath = ""
baseLayer1 := texture.MakeMatrixLayer(t, filepath.Join(ctx.Env.WorkDir, "source-base-1"), "1")
baseLayer1BlobDigest := baseLayer1.Pack(t, packOption, ctx.Env.BlobDir)

baseLayer2 := texture.MakeMatrixLayer(t, filepath.Join(ctx.Env.WorkDir, "source-base-2"), "2")
baseLayer2BlobDigest := baseLayer2.Pack(t, packOption, ctx.Env.BlobDir)

lowerLayer = texture.MakeLowerLayer(t, filepath.Join(ctx.Env.WorkDir, "source-lower-1"))
lowerBlobDigest = lowerLayer.Pack(t, packOption, ctx.Env.BlobDir)

upperLayer = texture.MakeUpperLayer(t, filepath.Join(ctx.Env.WorkDir, "source-upper-1"))
upperBlobDigest = upperLayer.Pack(t, packOption, ctx.Env.BlobDir)

mergeOption = converter.MergeOption{
BuilderPath: ctx.Binary.Builder,
}
baseLayerDigests, baseBootstrap := tool.MergeLayers(t, ctx, mergeOption, []converter.Layer{
{
Digest: baseLayer1BlobDigest,
},
{
Digest: baseLayer2BlobDigest,
},
})
ctx.Env.BootstrapPath = baseBootstrap
require.Equal(t, []digest.Digest{baseLayer1BlobDigest, baseLayer2BlobDigest}, baseLayerDigests)

// Test merge from a parent bootstrap
mergeOption = converter.MergeOption{
ParentBootstrapPath: baseBootstrap,
ChunkDictPath: baseBootstrap,
BuilderPath: ctx.Binary.Builder,
}
actualDigests, overlayBootstrap = tool.MergeLayers(t, ctx, mergeOption, []converter.Layer{
{
Digest: lowerBlobDigest,
},
{
Digest: upperBlobDigest,
},
})

require.Equal(t, []digest.Digest{
baseLayer1BlobDigest,
baseLayer2BlobDigest,
lowerBlobDigest,
upperBlobDigest,
}, actualDigests)

ctx.Env.BootstrapPath = overlayBootstrap
baseLayer1.Overlay(t, baseLayer2).Overlay(t, lowerLayer).Overlay(t, upperLayer)
tool.Verify(t, ctx, baseLayer1.FileTree)
}

func TestNativeLayer(t *testing.T) {
Expand Down
12 changes: 12 additions & 0 deletions smoke/tests/texture/layer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,3 +74,15 @@ func MakeUpperLayer(t *testing.T, workDir string) *tool.Layer {

return layer
}

func MakeMatrixLayer(t *testing.T, workDir, id string) *tool.Layer {
layer := tool.NewLayer(t, workDir)

// Create regular file
file1 := fmt.Sprintf("matrix-file-%s-1", id)
file2 := fmt.Sprintf("matrix-file-%s-2", id)
layer.CreateFile(t, file1, []byte(file1))
layer.CreateFile(t, file2, []byte(file2))

return layer
}
4 changes: 3 additions & 1 deletion smoke/tests/tool/layer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ func (l *Layer) PackRef(t *testing.T, ctx Context, blobDir string, compress bool
return ociBlobDigest, rafsBlobDigest
}

func (l *Layer) Overlay(t *testing.T, upper *Layer) {
func (l *Layer) Overlay(t *testing.T, upper *Layer) *Layer {
// Handle whiteout/opaque files
for upperName := range upper.FileTree {
name := filepath.Base(upperName)
Expand Down Expand Up @@ -198,6 +198,8 @@ func (l *Layer) Overlay(t *testing.T, upper *Layer) {
}
}
}

return l
}

func (l *Layer) recordFileTree(t *testing.T) {
Expand Down
11 changes: 10 additions & 1 deletion src/bin/nydus-image/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -319,11 +319,17 @@ fn prepare_cmd_args(bti_string: &'static str) -> App {
.subcommand(
App::new("merge")
.about("Merge multiple bootstraps into a overlaid bootstrap")
.arg(
Arg::new("parent-bootstrap")
.long("parent-bootstrap")
.help("File path of the parent/referenced RAFS metadata blob (optional)")
.required(false),
)
.arg(
Arg::new("bootstrap")
.long("bootstrap")
.short('B')
.help("output path of nydus overlaid bootstrap"),
.help("Output path of nydus overlaid bootstrap"),
)
.arg(
Arg::new("blob-dir")
Expand Down Expand Up @@ -921,8 +927,11 @@ impl Command {
};
ctx.configuration = config.clone();

let parent_bootstrap_path = Self::get_parent_bootstrap(matches)?;

let output = Merger::merge(
&mut ctx,
parent_bootstrap_path,
source_bootstrap_paths,
blob_digests,
blob_sizes,
Expand Down
48 changes: 32 additions & 16 deletions src/bin/nydus-image/merge.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
//
// SPDX-License-Identifier: Apache-2.0

use std::collections::HashMap;
use std::collections::HashSet;
use std::convert::TryFrom;
use std::path::{Path, PathBuf};
Expand Down Expand Up @@ -56,6 +57,7 @@ impl Merger {
#[allow(clippy::too_many_arguments)]
pub fn merge(
ctx: &mut BuildContext,
parent_bootstrap_path: Option<String>,
sources: Vec<PathBuf>,
blob_digests: Option<Vec<String>>,
blob_sizes: Option<Vec<u64>>,
Expand Down Expand Up @@ -101,6 +103,26 @@ impl Merger {
);
}

let mut tree: Option<Tree> = None;
let mut blob_mgr = BlobManager::new(ctx.digester);

// Load parent bootstrap
let mut blob_idx_map = HashMap::new();
let mut parent_layers = 0;
if let Some(parent_bootstrap_path) = &parent_bootstrap_path {
let (rs, _) =
RafsSuper::load_from_file(parent_bootstrap_path, config_v2.clone(), false, false)
.context(format!("load parent bootstrap {:?}", parent_bootstrap_path))?;
tree = Some(Tree::from_bootstrap(&rs, &mut ())?);
let blobs = rs.superblock.get_blob_infos();
for blob in &blobs {
let blob_ctx = BlobContext::from(ctx, &blob, ChunkSource::Parent)?;
blob_idx_map.insert(blob_ctx.blob_id.clone(), blob_mgr.len());
blob_mgr.add_blob(blob_ctx);
}
parent_layers = blobs.len();
}

// Get the blobs come from chunk dict bootstrap.
let mut chunk_dict_blobs = HashSet::new();
let mut config = None;
Expand All @@ -116,8 +138,6 @@ impl Merger {

let mut fs_version = RafsVersion::V6;
let mut chunk_size = None;
let mut tree: Option<Tree> = None;
let mut blob_mgr = BlobManager::new(ctx.digester);

for (layer_idx, bootstrap_path) in sources.iter().enumerate() {
let (rs, _) = RafsSuper::load_from_file(bootstrap_path, config_v2.clone(), true, false)
Expand All @@ -131,9 +151,9 @@ impl Merger {
ctx.digester = rs.meta.get_digester();
ctx.explicit_uidgid = rs.meta.explicit_uidgid();

let mut blob_idx_map = Vec::new();
let mut parent_blob_added = false;
for blob in rs.superblock.get_blob_infos() {
let blobs = &rs.superblock.get_blob_infos();
for blob in blobs {
let mut blob_ctx = BlobContext::from(ctx, &blob, ChunkSource::Parent)?;
if let Some(chunk_size) = chunk_size {
ensure!(
Expand Down Expand Up @@ -186,15 +206,8 @@ impl Merger {
}
}

let mut found = false;
for (idx, blob) in blob_mgr.get_blobs().iter().enumerate() {
if blob.blob_id == blob_ctx.blob_id {
blob_idx_map.push(idx as u32);
found = true;
}
}
if !found {
blob_idx_map.push(blob_mgr.len() as u32);
if !blob_idx_map.contains_key(&blob.blob_id()) {
blob_idx_map.insert(blob.blob_id().clone(), blob_mgr.len());
blob_mgr.add_blob(blob_ctx);
}
}
Expand All @@ -213,16 +226,19 @@ impl Merger {
))?;
for chunk in &mut node.chunks {
let origin_blob_index = chunk.inner.blob_index() as usize;
// Set the blob index of chunk to real index in blob table of final bootstrap.
chunk.set_blob_index(blob_idx_map[origin_blob_index]);
let blob_ctx = blobs[origin_blob_index].as_ref();
if let Some(blob_index) = blob_idx_map.get(&blob_ctx.blob_id()) {
// Set the blob index of chunk to real index in blob table of final bootstrap.
chunk.set_blob_index(*blob_index as u32);
}
}
// Set node's layer index to distinguish same inode number (from bootstrap)
// between different layers.
node.layer_idx = u16::try_from(layer_idx).context(format!(
"too many layers {}, limited to {}",
layer_idx,
u16::MAX
))?;
))? + parent_layers as u16;
node.overlay = Overlay::UpperAddition;
match node.whiteout_type(WhiteoutSpec::Oci) {
// Insert whiteouts at the head, so they will be handled first when
Expand Down

0 comments on commit 3f33613

Please sign in to comment.