Skip to content

Commit

Permalink
Merge pull request #970 from imeoer/refactor-nydusify-2
Browse files Browse the repository at this point in the history 
nydusify: refactor with converter package
  • Loading branch information
@jiangliu
jiangliu authored Dec 26, 2022
2 parents bbb42ec + dc83f58 commit f4dd9e1
Show file tree
Hide file tree
Showing 25 changed files with 1,710 additions and 2,083 deletions.
107 changes: 40 additions & 67 deletions contrib/nydusify/cmd/nydusify.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import (
"fmt"
"io"
"os"
"path/filepath"
"runtime"
"strings"

Expand All @@ -23,11 +22,8 @@ import (

"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/checker"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/converter"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/converter/provider"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/metrics"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/metrics/fileexporter"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/packer"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/remote"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/provider"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/utils"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/viewer"
)
Expand Down Expand Up @@ -76,7 +72,8 @@ func getBackendConfig(c *cli.Context, required bool) (string, string, error) {
}
return "", "", nil
}
possibleBackendTypes := []string{"registry", "oss", "s3"}

possibleBackendTypes := []string{"oss", "s3"}
if !isPossibleValue(possibleBackendTypes, backendType) {
return "", "", fmt.Errorf("--backend-type should be one of %v", possibleBackendTypes)
}
Expand Down Expand Up @@ -243,8 +240,8 @@ func main() {

&cli.StringFlag{
Name: "backend-type",
Value: "registry",
Usage: "Type of storage backend, possible values: 'registry', 'oss', 's3'",
Value: "",
Usage: "Type of storage backend, possible values: 'oss', 's3'",
EnvVars: []string{"BACKEND_TYPE"},
},
&cli.StringFlag{
Expand Down Expand Up @@ -392,27 +389,20 @@ func main() {
Action: func(c *cli.Context) error {
setupLogLevel(c)

target, err := getTargetReference(c)
targetRef, err := getTargetReference(c)
if err != nil {
return err
}

backendType, backendConfig, err := getBackendConfig(c, true)
backendType, backendConfig, err := getBackendConfig(c, false)
if err != nil {
return err
}

var cacheRemote *remote.Remote
cache, err := getCacheReference(c, target)
cacheRef, err := getCacheReference(c, targetRef)
if err != nil {
return err
}
if cache != "" {
cacheRemote, err = provider.DefaultRemote(cache, c.Bool("build-cache-insecure"))
if err != nil {
return err
}
}
cacheMaxRecords := c.Uint("build-cache-max-records")
if cacheMaxRecords < 1 {
return fmt.Errorf("--build-cache-max-records should be greater than 0")
Expand All @@ -428,71 +418,54 @@ func main() {
return fmt.Errorf("--fs-version should be one of %v", possibleFsVersions)
}

logger, err := provider.DefaultLogger()
if err != nil {
return err
}

sourceRemote, err := provider.DefaultRemote(c.String("source"), c.Bool("source-insecure"))
if err != nil {
return errors.Wrap(err, "Parse source reference")
}

targetPlatform := c.String("platform")
targetRemote, err := provider.DefaultRemote(target, c.Bool("target-insecure"))
if err != nil {
return err
}

prefetchPatterns, err := getPrefetchPatterns(c)
if err != nil {
return err
}

metrics.Register(fileexporter.New(filepath.Join(c.String("work-dir"), "conversion_metrics.prom")))
defer metrics.Export()
chunkDictRef := ""
chunkDict := c.String("chunk-dict")
if chunkDict != "" {
_, _, chunkDictRef, err = converter.ParseChunkDictArgs(chunkDict)
if err != nil {
return errors.Wrap(err, "parse chunk dict arguments")
}
}

opt := converter.Opt{
Logger: logger,
WorkDir: c.String("work-dir"),
NydusImagePath: c.String("nydus-image"),

TargetPlatform: targetPlatform,
Source: c.String("source"),
Target: targetRef,
SourceInsecure: c.Bool("source-insecure"),
TargetInsecure: c.Bool("target-insecure"),

SourceRemote: sourceRemote,
TargetRemote: targetRemote,
BackendType: backendType,
BackendConfig: backendConfig,
BackendForcePush: c.Bool("backend-force-push"),

CacheRemote: cacheRemote,
CacheRef: cacheRef,
CacheInsecure: c.Bool("build-cache-insecure"),
CacheMaxRecords: cacheMaxRecords,
CacheVersion: cacheVersion,

WorkDir: c.String("work-dir"),
ChunkDictRef: chunkDictRef,
ChunkDictInsecure: c.Bool("chunk-dict-insecure"),

PrefetchPatterns: prefetchPatterns,
NydusImagePath: c.String("nydus-image"),
MultiPlatform: c.Bool("multi-platform"),
DockerV2Format: c.Bool("docker-v2-format"),

BackendType: backendType,
BackendConfig: backendConfig,
BackendForcePush: c.Bool("backend-force-push"),

NydusifyVersion: version,
FsVersion: fsVersion,
FsAlignChunk: c.Bool("backend-aligned-chunk") || c.Bool("fs-align-chunk"),
Compressor: c.String("compressor"),
ChunkSize: c.String("chunk-size"),

ChunkDict: converter.ChunkDictOpt{
Args: c.String("chunk-dict"),
Insecure: c.Bool("chunk-dict-insecure"),
Platform: targetPlatform,
},
TargetPlatform: targetPlatform,
FsVersion: fsVersion,
FsAlignChunk: c.Bool("backend-aligned-chunk") || c.Bool("fs-align-chunk"),
Compressor: c.String("compressor"),
ChunkSize: c.String("chunk-size"),
}

cvt, err := converter.New(opt)
if err != nil {
return err
}

return cvt.Convert(context.Background())
return converter.Convert(context.Background(), opt)
},
},
{
Expand Down Expand Up @@ -527,7 +500,7 @@ func main() {
&cli.StringFlag{
Name: "backend-type",
Value: "",
Usage: "Type of storage backend, enable verification of file data in Nydus image if specified, possible values: 'registry', 'oss', 's3'",
Usage: "Type of storage backend, enable verification of file data in Nydus image if specified, possible values: 'oss', 's3'",
EnvVars: []string{"BACKEND_TYPE"},
},
&cli.StringFlag{
Expand Down Expand Up @@ -630,7 +603,7 @@ func main() {
Name: "backend-type",
Value: "",
Required: true,
Usage: "Type of storage backend, possible values: 'registry', 'oss', 's3'",
Usage: "Type of storage backend, possible values: 'oss', 's3'",
EnvVars: []string{"BACKEND_TYPE"},
},
&cli.StringFlag{
Expand Down Expand Up @@ -675,7 +648,7 @@ func main() {
Action: func(c *cli.Context) error {
setupLogLevel(c)

backendType, backendConfig, err := getBackendConfig(c, true)
backendType, backendConfig, err := getBackendConfig(c, false)
if err != nil {
return err
} else if backendConfig == "" {
Expand Down Expand Up @@ -742,7 +715,7 @@ func main() {
Name: "backend-type",
Value: "oss",
DefaultText: "oss",
Usage: "Type of storage backend, possible values: 'registry', 'oss', 's3'",
Usage: "Type of storage backend, possible values: 'oss', 's3'",
EnvVars: []string{"BACKEND_TYPE"},
},
&cli.StringFlag{
Expand Down
40 changes: 5 additions & 35 deletions contrib/nydusify/examples/converter/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"context"

"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/converter"
"github.com/dragonflyoss/image-service/contrib/nydusify/pkg/converter/provider"
)

func main() {
Expand All @@ -13,37 +12,13 @@ func main() {
nydusImagePath := "/path/to/nydus-image"
source := "localhost:5000/ubuntu:latest"
target := "localhost:5000/ubuntu:latest-nydus"
// Set to empty if no authorization be required
auth := "<base64_encoded_auth>"
// Set to false if using https registry
insecure := true

// Logger outputs Nydus image conversion progress
logger, err := provider.DefaultLogger()
if err != nil {
panic(err)
}

// Create remote with auth string for registry communication
sourceRemote, err := provider.DefaultRemoteWithAuth(source, insecure, auth)
if err != nil {
panic(err)
}
// Or we can create with docker config
// sourceRemote, err := provider.DefaultRemote(source, insecure)
// if err != nil {
// panic(err)
// }
targetRemote, err := provider.DefaultRemoteWithAuth(target, insecure, auth)
if err != nil {
panic(err)
}

opt := converter.Opt{
Logger: logger,
TargetPlatform: "linux/amd64",
SourceRemote: sourceRemote,
TargetRemote: targetRemote,
Source: source,
Target: target,
SourceInsecure: true,
TargetInsecure: true,

WorkDir: workDir,
PrefetchPatterns: "/",
Expand All @@ -52,12 +27,7 @@ func main() {
DockerV2Format: true,
}

cvt, err := converter.New(opt)
if err != nil {
panic(err)
}

if err := cvt.Convert(context.Background()); err != nil {
if err := converter.Convert(context.Background(), opt); err != nil {
panic(err)
}
}
Loading

0 comments on commit f4dd9e1

Please sign in to comment.