diff --git a/go.mod b/go.mod index 26613f75..ffb2cb06 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/json-iterator/go v1.1.12 github.com/muvaf/typewriter v0.0.0-20210910160850-80e49fe1eb32 github.com/pkg/errors v0.9.1 - github.com/spf13/afero v1.8.0 + github.com/spf13/afero v1.9.2 github.com/tmccombs/hcl2json v0.3.3 github.com/yuin/goldmark v1.4.13 github.com/zclconf/go-cty v1.11.0 @@ -26,6 +26,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.25.0 k8s.io/apimachinery v0.25.0 + k8s.io/client-go v0.25.0 k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed sigs.k8s.io/controller-runtime v0.12.1 sigs.k8s.io/yaml v1.3.0 @@ -34,7 +35,7 @@ require ( require ( github.com/agext/levenshtein v1.2.3 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect - github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a // indirect + github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/antchfx/xpath v1.2.0 // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -67,7 +68,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.12 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-isatty v0.0.16 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect @@ -86,8 +87,8 @@ require ( github.com/vmihailenco/msgpack/v4 v4.3.12 // indirect github.com/vmihailenco/tagparser v0.1.1 // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect - golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect + golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2 // indirect + golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect golang.org/x/term v0.0.0-20220411215600-e5f449aeb171 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect @@ -98,7 +99,6 @@ require ( gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/inf.v0 v0.9.1 // indirect k8s.io/apiextensions-apiserver v0.24.0 // indirect - k8s.io/client-go v0.25.0 // indirect k8s.io/component-base v0.25.0 // indirect k8s.io/klog/v2 v2.70.1 // indirect k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect diff --git a/go.sum b/go.sum index 10cb1bf8..59ef694b 100644 --- a/go.sum +++ b/go.sum @@ -65,8 +65,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a h1:E/8AP5dFtMhl5KPJz66Kt9G0n+7Sn41Fy1wv9/jHOrc= -github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/antchfx/htmlquery v1.2.4 h1:qLteofCMe/KGovBI6SQgmou2QNyedFUW+pE+BpeZ494= github.com/antchfx/htmlquery v1.2.4/go.mod h1:2xO6iu3EVWs7R2JYqBbp8YzG50gj/ofqs5/0VZoDZLc= github.com/antchfx/xpath v1.2.0 h1:mbwv7co+x0RwgeGAOHdrKy89GvHaGvxxBtPK0uF9Zr8= @@ -399,8 +399,9 @@ github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZb github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -522,8 +523,8 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/afero v1.8.0 h1:5MmtuhAgYeU6qpa7w7bP0dv6MBYuup0vekhSpSkoq60= -github.com/spf13/afero v1.8.0/go.mod h1:CtAatgMJh6bJEIs48Ay/FOnkljP3WeGUG0MC1RfAqwo= +github.com/spf13/afero v1.9.2 h1:j49Hj62F0n+DaZ1dDCvhABaPNSGNkt32oRFxI33IEMw= +github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= @@ -737,8 +738,8 @@ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 h1:OSnWWcOd/CtWQC2cYSBgbTSJv3ciqd8r54ySIW2y3RE= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2 h1:+jnHzr9VPj32ykQVai5DNahi9+NSp7yYuCsl5eAQtL0= +golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -822,8 +823,8 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220411215600-e5f449aeb171 h1:EH1Deb8WZJ0xc0WK//leUHXcX9aLE5SymusoTmMZye8= diff --git a/pkg/migration/converter.go b/pkg/migration/converter.go new file mode 100644 index 00000000..4ebc9f22 --- /dev/null +++ b/pkg/migration/converter.go @@ -0,0 +1,85 @@ +package migration + +import ( + "github.com/crossplane/crossplane-runtime/pkg/fieldpath" + "github.com/crossplane/crossplane-runtime/pkg/resource" + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/json" + "k8s.io/client-go/kubernetes/scheme" +) + +const ( + errFromUnstructured = "failed to convert from unstructured.Unstructured to the managed resource type" + errToUnstructured = "failed to convert from the managed resource type to unstructured.Unstructured" + errRawExtensionUnmarshal = "failed to unmarshal runtime.RawExtension" + + errFmtPavedDelete = "failed to delete fieldpath %q from paved" + errFmtNewObject = "failed to instantiate a new runtime.Object using scheme.Scheme for: %s" +) + +func CopyInto(source any, target any, targetGVK schema.GroupVersionKind, skipFieldPaths ...string) (any, error) { + u := ToUnstructured(source) + paved := fieldpath.Pave(u.Object) + skipFieldPaths = append(skipFieldPaths, "apiVersion", "kind") + for _, p := range skipFieldPaths { + if err := paved.DeleteField(p); err != nil { + return nil, errors.Wrapf(err, errFmtPavedDelete, p) + } + } + u.SetGroupVersionKind(targetGVK) + return FromUnstructured(target, u), nil +} + +func FromUnstructured(mg any, u unstructured.Unstructured) any { + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, mg); err != nil { + panic(errors.Wrap(err, errFromUnstructured)) + } + return mg +} + +func sanitizeResource(m map[string]any) map[string]any { + delete(m, "status") + if _, ok := m["metadata"]; !ok { + return m + } + metadata := m["metadata"].(map[string]any) + + if v := metadata["creationTimestamp"]; v == nil { + delete(metadata, "creationTimestamp") + } + if len(metadata) == 0 { + delete(m, "metadata") + } + return m +} + +func ToUnstructured(mg any) unstructured.Unstructured { + m, err := runtime.DefaultUnstructuredConverter.ToUnstructured(mg) + if err != nil { + panic(errors.Wrap(err, errToUnstructured)) + } + return unstructured.Unstructured{ + Object: sanitizeResource(m), + } +} + +func FromRawExtension(r runtime.RawExtension) (unstructured.Unstructured, error) { + var m map[string]interface{} + if err := json.Unmarshal(r.Raw, &m); err != nil { + return unstructured.Unstructured{}, errors.Wrap(err, errRawExtensionUnmarshal) + } + return unstructured.Unstructured{ + Object: m, + }, nil +} + +func ToManagedResource(gvk schema.GroupVersionKind, u unstructured.Unstructured) (resource.Managed, error) { + obj, err := scheme.Scheme.New(gvk) + if err != nil { + return nil, errors.Wrapf(err, errFmtNewObject, gvk) + } + return FromUnstructured(obj, u).(resource.Managed), nil +} diff --git a/pkg/migration/noop_target.go b/pkg/migration/noop_target.go new file mode 100644 index 00000000..e3a32e3b --- /dev/null +++ b/pkg/migration/noop_target.go @@ -0,0 +1,15 @@ +package migration + +type NoopTarget struct{} + +func (_ NoopTarget) Put(_ UnstructuredWithMetadata) error { + return nil +} + +func (_ NoopTarget) Patch(_ UnstructuredWithMetadata) error { + return nil +} + +func (_ NoopTarget) Delete(_ UnstructuredWithMetadata) error { + return nil +} diff --git a/pkg/migration/plan_generator.go b/pkg/migration/plan_generator.go new file mode 100644 index 00000000..dfb5b8e1 --- /dev/null +++ b/pkg/migration/plan_generator.go @@ -0,0 +1,156 @@ +/* +Copyright 2022 Upbound Inc. +*/ + +package migration + +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + + xpv1 "github.com/crossplane/crossplane/apis/apiextensions/v1" + "github.com/pkg/errors" +) + +const ( + errSourceHasNext = "failed to generate migration plan: Could not check next object from source" + errSourceNext = "failed to generate migration plan: Could not get next object from source" + errUnstructuredConvert = "failed to convert from unstructured object to v1.Composition" + errUnstructuredMarshal = "failed to marshal unstructured object to JSON" + errResourceMigrate = "failed to migrate resource" + errCompositionMigrate = "failed to migrate the composition" + errComposedTemplateBase = "failed to migrate the base of a composed template" + errComposedTemplateMigrate = "failed to migrate the composed templates of the composition" + errResourceOutput = "failed to output migrated resource" + errCompositionOutput = "failed to output migrated composition" + errPlanGeneration = "failed to generate the migration plan" +) + +type PlanGenerator struct { + source Source + target Target + registry Registry +} + +func NewPlanGenerator(source Source, target Target) PlanGenerator { + return PlanGenerator{ + source: source, + target: target, + registry: registry, + } +} + +func (pg *PlanGenerator) GeneratePlan() (*Plan, error) { + if err := pg.convert(); err != nil { + return nil, errors.Wrap(err, errPlanGeneration) + } + return &Plan{}, nil +} + +func (pg *PlanGenerator) convert() error { //nolint: gocyclo + for hasNext, err := pg.source.HasNext(); ; hasNext, err = pg.source.HasNext() { + if err != nil { + return errors.Wrap(err, errSourceHasNext) + } + if !hasNext { + break + } + o, err := pg.source.Next() + if err != nil { + return errors.Wrap(err, errSourceNext) + } + switch gvk := o.Object.GroupVersionKind(); gvk { + case xpv1.CompositionGroupVersionKind: + target, err := pg.convertComposition(o) + if err != nil { + return errors.Wrap(err, errCompositionMigrate) + } + if err := pg.target.Put(*target); err != nil { + return errors.Wrap(err, errCompositionOutput) + } + default: + targets, err := pg.convertResource(gvk, o) + if err != nil { + return errors.Wrap(err, errResourceMigrate) + } + for _, tu := range targets { + if err := pg.target.Put(tu); err != nil { + return errors.Wrap(err, errResourceOutput) + } + } + } + } + return nil +} + +func (pg *PlanGenerator) convertResource(gvk schema.GroupVersionKind, o UnstructuredWithMetadata) ([]UnstructuredWithMetadata, error) { + conv := pg.registry[gvk] + if conv == nil { + return []UnstructuredWithMetadata{o}, nil + } + mg, err := ToManagedResource(gvk, o.Object) + if err != nil { + return nil, errors.Wrap(err, errResourceMigrate) + } + resources, err := conv.Resources(mg) + if err != nil { + return nil, errors.Wrap(err, errResourceMigrate) + } + converted := make([]UnstructuredWithMetadata, 0, len(resources)) + for _, mg := range resources { + converted = append(converted, UnstructuredWithMetadata{ + Object: ToUnstructured(mg), + Metadata: o.Metadata, + }) + } + return converted, nil +} + +func (pg *PlanGenerator) convertComposition(o UnstructuredWithMetadata) (*UnstructuredWithMetadata, error) { + c := xpv1.Composition{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(o.Object.Object, &c); err != nil { + return nil, errors.Wrap(err, errUnstructuredConvert) + } + var targetResources []*xpv1.ComposedTemplate + for _, cmp := range c.Spec.Resources { + u, err := FromRawExtension(cmp.Base) + if err != nil { + return nil, errors.Wrap(err, errCompositionMigrate) + } + gvk := u.GetObjectKind().GroupVersionKind() + converted, err := pg.convertResource(gvk, UnstructuredWithMetadata{ + Object: u, + Metadata: o.Metadata, + }) + if err != nil { + return nil, errors.Wrap(err, errComposedTemplateBase) + } + cmps := make([]*xpv1.ComposedTemplate, 0, len(converted)) + for _, u := range converted { + buff, err := u.Object.MarshalJSON() + if err != nil { + return nil, errors.Wrap(err, errUnstructuredMarshal) + } + c := cmp.DeepCopy() + c.Base = runtime.RawExtension{ + Raw: buff, + } + cmps = append(cmps, c) + } + conv := pg.registry[gvk] + if conv != nil { + if err := conv.ComposedTemplates(cmp, cmps...); err != nil { + return nil, errors.Wrap(err, errComposedTemplateMigrate) + } + } + targetResources = append(targetResources, cmps...) + } + c.Spec.Resources = make([]xpv1.ComposedTemplate, 0, len(targetResources)) + for _, cmp := range targetResources { + c.Spec.Resources = append(c.Spec.Resources, *cmp) + } + return &UnstructuredWithMetadata{ + Object: ToUnstructured(&c), + Metadata: o.Metadata, + }, nil +} diff --git a/pkg/migration/registry.go b/pkg/migration/registry.go new file mode 100644 index 00000000..c15c0a68 --- /dev/null +++ b/pkg/migration/registry.go @@ -0,0 +1,40 @@ +package migration + +import ( + "github.com/crossplane/crossplane-runtime/pkg/resource" + v1 "github.com/crossplane/crossplane/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var ( + registry Registry = make(map[schema.GroupVersionKind]Converter) +) + +type ResourceConversionFn func(mg resource.Managed) ([]resource.Managed, error) +type ComposedTemplateConversionFn func(cmp v1.ComposedTemplate, convertedBase ...*v1.ComposedTemplate) error + +type Registry map[schema.GroupVersionKind]Converter + +func RegisterConverter(gvk schema.GroupVersionKind, conv Converter) { + registry[gvk] = conv +} + +type delegatingConverter struct { + rFn ResourceConversionFn + cmpFn ComposedTemplateConversionFn +} + +func (d delegatingConverter) Resources(mg resource.Managed) ([]resource.Managed, error) { + return d.rFn(mg) +} + +func (d delegatingConverter) ComposedTemplates(cmp v1.ComposedTemplate, convertedBase ...*v1.ComposedTemplate) error { + return d.cmpFn(cmp, convertedBase...) +} + +func RegisterConversionFunctions(gvk schema.GroupVersionKind, rFn ResourceConversionFn, cmpFn ComposedTemplateConversionFn) { + RegisterConverter(gvk, delegatingConverter{ + rFn: rFn, + cmpFn: cmpFn, + }) +}