diff --git a/cmd/kcl/commands/args.go b/cmd/kcl/commands/args.go index 95e06d1..8bcf372 100644 --- a/cmd/kcl/commands/args.go +++ b/cmd/kcl/commands/args.go @@ -1,8 +1,104 @@ package cmd +import ( + "net/url" + + "kcl-lang.io/kpm/pkg/client" + "kcl-lang.io/kpm/pkg/constants" + "kcl-lang.io/kpm/pkg/opt" +) + func argsGet(a []string, n int) string { if len(a) > n { return a[n] } return "" } + +func ParseUrlFromArgs(cli *client.KpmClient, args []string) (*url.URL, error) { + var sourceUrl url.URL + + if len(args) == 0 { + if len(git) != 0 { + gitUrl, err := url.Parse(git) + if err != nil { + return nil, err + } + + gitUrl.Scheme = constants.GitScheme + query := gitUrl.Query() + if tag != "" { + query.Add(constants.Tag, tag) + } + if commit != "" { + query.Add(constants.GitCommit, commit) + } + if branch != "" { + query.Add(constants.GitBranch, branch) + } + gitUrl.RawQuery = query.Encode() + sourceUrl = *gitUrl + } else if len(oci) != 0 { + ociUrl, err := url.Parse(oci) + if err != nil { + return nil, err + } + + ociUrl.Scheme = constants.OciScheme + query := ociUrl.Query() + query.Add(constants.Tag, tag) + ociUrl.RawQuery = query.Encode() + sourceUrl = *ociUrl + } + } else { + url, err := url.Parse(args[0]) + if err != nil { + return nil, err + } + query := url.Query() + url.Opaque = "" + regOpts, err := opt.NewRegistryOptionsFrom(args[0], cli.GetSettings()) + if err != nil { + return nil, err + } + + if regOpts.Git != nil { + if url.Scheme != constants.GitScheme && url.Scheme != constants.SshScheme { + url.Scheme = constants.GitScheme + } + if tag != "" { + query.Add(constants.Tag, tag) + } + if commit != "" { + query.Add(constants.GitCommit, commit) + } + if branch != "" { + query.Add(constants.GitBranch, branch) + } + } else if regOpts.Oci != nil { + url.Scheme = constants.OciScheme + url.Host = regOpts.Oci.Reg + url.Path = regOpts.Oci.Repo + if regOpts.Oci.Tag != "" { + query.Add(constants.Tag, regOpts.Oci.Tag) + } + if tag != "" { + query.Add(constants.Tag, tag) + } + } else if regOpts.Registry != nil { + url.Scheme = constants.DefaultOciScheme + url.Host = regOpts.Registry.Reg + url.Path = regOpts.Registry.Repo + if regOpts.Registry.Tag != "" { + query.Add(constants.Tag, regOpts.Registry.Tag) + } + if tag != "" { + query.Add(constants.Tag, tag) + } + } + + url.RawQuery = query.Encode() + sourceUrl = *url + } + return &sourceUrl, nil +} diff --git a/cmd/kcl/commands/mod_add.go b/cmd/kcl/commands/mod_add.go index 2dd5eac..d7d8aad 100644 --- a/cmd/kcl/commands/mod_add.go +++ b/cmd/kcl/commands/mod_add.go @@ -28,7 +28,7 @@ const ( kcl mod add git://github.com/kcl-lang/konfig --tag v0.4.0 # Add the module dependency from the OCI Registry by oci url - kcl mod add oci://github.com/kcl-lang/konfig --tag v0.4.0 + kcl mod add oci://ghcr.io/kcl-lang/helloworld --tag 0.1.0 # Add the module dependency from the local file system by file url kcl mod add /path/to/another_module diff --git a/cmd/kcl/commands/mod_pull.go b/cmd/kcl/commands/mod_pull.go index 0de1182..b32d285 100644 --- a/cmd/kcl/commands/mod_pull.go +++ b/cmd/kcl/commands/mod_pull.go @@ -3,13 +3,29 @@ package cmd import ( "github.com/spf13/cobra" "kcl-lang.io/kpm/pkg/client" + "kcl-lang.io/kpm/pkg/downloader" ) const ( modPullDesc = `This command pulls kcl modules from the registry. ` modPullExample = ` # Pull the the module named "k8s" to the local path from the registry - kcl mod pull k8s` + kcl mod pull k8s + + # Pull the module dependency named "k8s" with the version "1.28" + kcl mod pull k8s:1.28 + + # Pull the module from the GitHub by git url + kcl mod pull git://github.com/kcl-lang/konfig --tag v0.4.0 + + # Pull the module from the OCI Registry by oci url + kcl mod pull oci://ghcr.io/kcl-lang/helloworld --tag 0.1.0 + + # Pull the module from the GitHub by flag + kcl mod pull --git https://github.com/kcl-lang/konfig --tag v0.4.0 + + # Pull the module from the OCI Registry by flag + kcl mod pull --oci https://ghcr.io/kcl-lang/helloworld --tag 0.1.0` ) // NewModPullCmd returns the mod pull command. @@ -20,14 +36,39 @@ func NewModPullCmd(cli *client.KpmClient) *cobra.Command { Long: modPullDesc, Example: modPullExample, RunE: func(_ *cobra.Command, args []string) error { - source := argsGet(args, 0) localPath := argsGet(args, 1) - return cli.PullFromOci(localPath, source, tag) + return pull(cli, args, localPath) }, SilenceUsage: true, } - cmd.Flags().StringVar(&tag, "tag", "", "git repository tag") + cmd.Flags().StringVar(&git, "git", "", "git repository url") + cmd.Flags().StringVar(&oci, "oci", "", "oci repository url") + cmd.Flags().StringVar(&tag, "tag", "", "git or oci repository tag") + cmd.Flags().StringVar(&commit, "commit", "", "git repository commit") + cmd.Flags().StringVar(&branch, "branch", "", "git repository branch") return cmd } + +func pull(cli *client.KpmClient, args []string, localPath string) error { + sourceUrl, err := ParseUrlFromArgs(cli, args) + if err != nil { + return err + } + source, err := downloader.NewSourceFromStr(sourceUrl.String()) + if err != nil { + return err + } + + _, err = cli.Pull( + client.WithPullSource(source), + client.WithLocalPath(localPath), + ) + + if err != nil { + return err + } + + return nil +} diff --git a/cmd/kcl/commands/mod_push.go b/cmd/kcl/commands/mod_push.go index 83be765..7a9ba9f 100644 --- a/cmd/kcl/commands/mod_push.go +++ b/cmd/kcl/commands/mod_push.go @@ -169,7 +169,7 @@ func pushPackage(ociUrl string, kclPkg *pkg.KclPkg, vendorMode bool, cli *client "only support url scheme 'oci://'.", ) } - ociOpts.Annotations, err = kpmoci.GenOciManifestFromPkg(kclPkg) + ociOpts.Annotations, err = kclPkg.GenOciManifestFromPkg() if err != nil { return err } diff --git a/go.mod b/go.mod index 80daf55..db9f781 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( github.com/containers/ocicrypt v1.1.10 // indirect github.com/containers/storage v1.54.0 // indirect github.com/distribution/reference v0.6.0 // indirect + github.com/elliotchance/orderedmap/v2 v2.2.0 // indirect github.com/emicklei/proto v1.13.2 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect @@ -208,3 +209,5 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) + +replace kcl-lang.io/kpm v0.9.0-rc.2 => kcl-lang.io/kpm v0.9.0-rc.2.0.20240704123051-efc7b9c78ae8 diff --git a/go.sum b/go.sum index bb9fea5..3e32788 100644 --- a/go.sum +++ b/go.sum @@ -373,6 +373,8 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= +github.com/elliotchance/orderedmap/v2 v2.2.0 h1:7/2iwO98kYT4XkOjA9mBEIwvi4KpGB4cyHeOFOnj4Vk= +github.com/elliotchance/orderedmap/v2 v2.2.0/go.mod h1:85lZyVbpGaGvHvnKa7Qhx7zncAdBIBq6u56Hb1PRU5Q= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/proto v1.13.2 h1:z/etSFO3uyXeuEsVPzfl56WNgzcvIr42aQazXaQmFZY= @@ -1689,8 +1691,8 @@ kcl-lang.io/kcl-openapi v0.6.3 h1:6Br0IBaNshRvBfmaH22Zv1miVEt6FWlmh2v/wBLb194= kcl-lang.io/kcl-openapi v0.6.3/go.mod h1:Ai9mFztCVKkRSFabczO/r5hCNdqaNtAc2ZIRxTeV0Mk= kcl-lang.io/kcl-playground v0.5.1 h1:MKQQUHgt4+2QyU2NVwa73oksOaBJGDi4keGoggA0MiU= kcl-lang.io/kcl-playground v0.5.1/go.mod h1:IFmnlw7m011ccX8OidMUfnnN2u/TWdtQGxyABRTbmow= -kcl-lang.io/kpm v0.9.0-rc.2 h1:0v76+Tk0XnWawXHLoK+XuGTp+7QxD+vNUCzteIgNE0w= -kcl-lang.io/kpm v0.9.0-rc.2/go.mod h1:mwEU/1fURutTNaO1SeGpoFh1iMf+Cc6SYyDAuIQHGNg= +kcl-lang.io/kpm v0.9.0-rc.2.0.20240704123051-efc7b9c78ae8 h1:HXGw7kK5b4xiTiinA1R10sEOSYWqPvrCFKmZdTbFAyw= +kcl-lang.io/kpm v0.9.0-rc.2.0.20240704123051-efc7b9c78ae8/go.mod h1:+8dz5o9l1DEzEYdfRetnAXhrfqKdzTQb3s99DubbHhk= kcl-lang.io/lib v0.9.0-rc.2 h1:1FcWnHKvxe5TGtEwTtd3CTg6V8aw4VdEWtYvQEp15Gk= kcl-lang.io/lib v0.9.0-rc.2/go.mod h1:tu+tzwGgHLzYZSIxUG/ntipStrxZd6OvutWYPTxS7cs= oras.land/oras-go v1.2.5 h1:XpYuAwAb0DfQsunIyMfeET92emK8km3W4yEzZvUbsTo= diff --git a/scripts/e2e/pull_pkg.sh b/scripts/e2e/pull_pkg.sh index 5732b8f..bc022d7 100755 --- a/scripts/e2e/pull_pkg.sh +++ b/scripts/e2e/pull_pkg.sh @@ -13,11 +13,11 @@ cd ./scripts/e2e/pkg_in_reg/ # Check if file exists -if [ ! -d "./ghcr.io/kcl-lang/k8s/1.28" ]; then +if [ ! -d "./oci/ghcr.io/kcl-lang/k8s/1.28" ]; then $current_dir/bin/kcl mod pull k8s:1.28 fi -if [ ! -d "./ghcr.io/kcl-lang/helloworld/0.1.1" ]; then +if [ ! -d "./oci/ghcr.io/kcl-lang/helloworld/0.1.1" ]; then $current_dir/bin/kcl mod pull helloworld:0.1.1 fi diff --git a/scripts/e2e/push_pkg.sh b/scripts/e2e/push_pkg.sh index ab0cc52..517e472 100755 --- a/scripts/e2e/push_pkg.sh +++ b/scripts/e2e/push_pkg.sh @@ -11,13 +11,13 @@ echo $current_dir $current_dir/bin/kcl registry login -u test -p 1234 localhost:5001 -cd ./scripts/e2e/pkg_in_reg/ghcr.io/kcl-lang/k8s/1.28 +cd ./scripts/e2e/pkg_in_reg/oci/ghcr.io/kcl-lang/k8s/1.28 $current_dir/bin/kcl mod push cd "$current_dir" # Push the package helloworld/0.1.1 to the registry -cd ./scripts/e2e/pkg_in_reg/ghcr.io/kcl-lang/helloworld/0.1.1 +cd ./scripts/e2e/pkg_in_reg/oci/ghcr.io/kcl-lang/helloworld/0.1.1 $current_dir/bin/kcl mod push cd "$current_dir" diff --git a/test/e2e/test_suites/test_kcl_mod_pull/stdout b/test/e2e/test_suites/test_kcl_mod_pull/stdout index da7ff38..a8aa5fd 100644 --- a/test/e2e/test_suites/test_kcl_mod_pull/stdout +++ b/test/e2e/test_suites/test_kcl_mod_pull/stdout @@ -1,4 +1,4 @@ -start to pull 'helloworld' -the lastest version '0.1.1' will be pulled -pulling 'test/helloworld:0.1.1' from 'localhost:5001/test/helloworld' -pulled 'helloworld' in '/test_space/localhost:5001/test/helloworld' successfully +start to pull oci://localhost:5001/test/helloworld +the lastest version '0.1.1' will be downloaded +downloading 'test/helloworld:0.1.1' from 'localhost:5001/test/helloworld:0.1.1' +pulled helloworld 0.1.1 successfully \ No newline at end of file diff --git a/test/e2e/test_suites/test_kcl_mod_pull_0/stdout b/test/e2e/test_suites/test_kcl_mod_pull_0/stdout index 33bda07..bead25e 100644 --- a/test/e2e/test_suites/test_kcl_mod_pull_0/stdout +++ b/test/e2e/test_suites/test_kcl_mod_pull_0/stdout @@ -1,3 +1,3 @@ -start to pull 'helloworld:0.1.1' -pulling 'test/helloworld:0.1.1' from 'localhost:5001/test/helloworld' -pulled 'helloworld:0.1.1' in '/test_space/localhost:5001/test/helloworld/0.1.1' successfully +start to pull oci://localhost:5001/test/helloworld?tag=0.1.1 +downloading 'test/helloworld:0.1.1' from 'localhost:5001/test/helloworld:0.1.1' +pulled helloworld 0.1.1 successfully \ No newline at end of file diff --git a/test/e2e/test_suites/test_kcl_mod_pull_1/input b/test/e2e/test_suites/test_kcl_mod_pull_1/input index 67782bd..5220928 100644 --- a/test/e2e/test_suites/test_kcl_mod_pull_1/input +++ b/test/e2e/test_suites/test_kcl_mod_pull_1/input @@ -1 +1 @@ -kcl mod pull oci://localhost:5001/test/helloworld:0.1.1 \ No newline at end of file +kcl mod pull oci://localhost:5001/test/helloworld --tag 0.1.1 \ No newline at end of file diff --git a/test/e2e/test_suites/test_kcl_mod_pull_1/stdout b/test/e2e/test_suites/test_kcl_mod_pull_1/stdout index 31a4e44..bead25e 100644 --- a/test/e2e/test_suites/test_kcl_mod_pull_1/stdout +++ b/test/e2e/test_suites/test_kcl_mod_pull_1/stdout @@ -1,4 +1,3 @@ -start to pull 'oci://localhost:5001/test/helloworld:0.1.1' -the lastest version '0.1.1' will be pulled -pulling '/test/helloworld:0.1.1:0.1.1' from 'localhost:5001/test/helloworld:0.1.1' -pulled 'oci://localhost:5001/test/helloworld:0.1.1' in '/test_space/localhost:5001/test/helloworld:0.1.1' successfully +start to pull oci://localhost:5001/test/helloworld?tag=0.1.1 +downloading 'test/helloworld:0.1.1' from 'localhost:5001/test/helloworld:0.1.1' +pulled helloworld 0.1.1 successfully \ No newline at end of file diff --git a/test/e2e/test_suites/test_kcl_mod_pull_2/input b/test/e2e/test_suites/test_kcl_mod_pull_2/input new file mode 100644 index 0000000..109c136 --- /dev/null +++ b/test/e2e/test_suites/test_kcl_mod_pull_2/input @@ -0,0 +1 @@ +kcl mod pull git://github.com/kcl-lang/flask-demo-kcl-manifests --commit ade147b \ No newline at end of file diff --git a/test/e2e/test_suites/test_kcl_mod_pull_2/stderr b/test/e2e/test_suites/test_kcl_mod_pull_2/stderr new file mode 100644 index 0000000..e69de29 diff --git a/test/e2e/test_suites/test_kcl_mod_pull_2/stdout b/test/e2e/test_suites/test_kcl_mod_pull_2/stdout new file mode 100644 index 0000000..e4eab44 --- /dev/null +++ b/test/e2e/test_suites/test_kcl_mod_pull_2/stdout @@ -0,0 +1,3 @@ +start to pull https://github.com/kcl-lang/flask-demo-kcl-manifests?commit=ade147b +cloning 'https://github.com/kcl-lang/flask-demo-kcl-manifests' with commit 'ade147b' +pulled flask_manifests 0.0.1 successfully \ No newline at end of file diff --git a/test/e2e/test_suites/test_kcl_mod_pull_3/input b/test/e2e/test_suites/test_kcl_mod_pull_3/input new file mode 100644 index 0000000..72a82bc --- /dev/null +++ b/test/e2e/test_suites/test_kcl_mod_pull_3/input @@ -0,0 +1 @@ +kcl mod pull --git https://github.com/kcl-lang/flask-demo-kcl-manifests --branch main \ No newline at end of file diff --git a/test/e2e/test_suites/test_kcl_mod_pull_3/stderr b/test/e2e/test_suites/test_kcl_mod_pull_3/stderr new file mode 100644 index 0000000..e69de29 diff --git a/test/e2e/test_suites/test_kcl_mod_pull_3/stdout b/test/e2e/test_suites/test_kcl_mod_pull_3/stdout new file mode 100644 index 0000000..610bfd6 --- /dev/null +++ b/test/e2e/test_suites/test_kcl_mod_pull_3/stdout @@ -0,0 +1,3 @@ +start to pull https://github.com/kcl-lang/flask-demo-kcl-manifests?branch=main +cloning 'https://github.com/kcl-lang/flask-demo-kcl-manifests' +pulled flask_manifests 0.0.1 successfully \ No newline at end of file diff --git a/test/e2e/test_suites/test_kcl_mod_pull_4/input b/test/e2e/test_suites/test_kcl_mod_pull_4/input new file mode 100644 index 0000000..724b2a7 --- /dev/null +++ b/test/e2e/test_suites/test_kcl_mod_pull_4/input @@ -0,0 +1 @@ +kcl mod pull --oci http://localhost:5001/test/helloworld --tag 0.1.1 \ No newline at end of file diff --git a/test/e2e/test_suites/test_kcl_mod_pull_4/stderr b/test/e2e/test_suites/test_kcl_mod_pull_4/stderr new file mode 100644 index 0000000..e69de29 diff --git a/test/e2e/test_suites/test_kcl_mod_pull_4/stdout b/test/e2e/test_suites/test_kcl_mod_pull_4/stdout new file mode 100644 index 0000000..bead25e --- /dev/null +++ b/test/e2e/test_suites/test_kcl_mod_pull_4/stdout @@ -0,0 +1,3 @@ +start to pull oci://localhost:5001/test/helloworld?tag=0.1.1 +downloading 'test/helloworld:0.1.1' from 'localhost:5001/test/helloworld:0.1.1' +pulled helloworld 0.1.1 successfully \ No newline at end of file