From f1248b9c829e225138ab6d6234221c63092f7592 Mon Sep 17 00:00:00 2001 From: Wojciech Tyczynski Date: Fri, 4 Mar 2016 14:24:41 +0100 Subject: [PATCH] Migrate to new conversion generator --- .../conversion-gen/generators/conversion.go | 514 ++++++++++++++++++ cmd/libs/go2idl/conversion-gen/main.go | 47 ++ .../deepcopy-gen/generators/deepcopy.go | 9 +- 3 files changed, 563 insertions(+), 7 deletions(-) create mode 100644 cmd/libs/go2idl/conversion-gen/generators/conversion.go create mode 100644 cmd/libs/go2idl/conversion-gen/main.go diff --git a/cmd/libs/go2idl/conversion-gen/generators/conversion.go b/cmd/libs/go2idl/conversion-gen/generators/conversion.go new file mode 100644 index 0000000000000..9da600c8a1cc8 --- /dev/null +++ b/cmd/libs/go2idl/conversion-gen/generators/conversion.go @@ -0,0 +1,514 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generators + +import ( + "fmt" + "io" + "path/filepath" + "strings" + + "k8s.io/kubernetes/cmd/libs/go2idl/args" + "k8s.io/kubernetes/cmd/libs/go2idl/generator" + "k8s.io/kubernetes/cmd/libs/go2idl/namer" + "k8s.io/kubernetes/cmd/libs/go2idl/types" + "k8s.io/kubernetes/pkg/util/sets" + + "github.com/golang/glog" +) + +// TODO: This is created only to reduce number of changes in a single PR. +// Remove it and use PublicNamer instead. +func conversionNamer() *namer.NameStrategy { + return &namer.NameStrategy{ + Join: func(pre string, in []string, post string) string { + return strings.Join(in, "_") + }, + PrependPackageNames: 1, + } +} + +// NameSystems returns the name system used by the generators in this package. +func NameSystems() namer.NameSystems { + return namer.NameSystems{ + "public": conversionNamer(), + "raw": namer.NewRawNamer("", nil), + } +} + +// DefaultNameSystem returns the default name system for ordering the types to be +// processed by the generators in this package. +func DefaultNameSystem() string { + return "public" +} + +func getInternalTypeFor(context *generator.Context, t *types.Type) (*types.Type, bool) { + internalPackage := filepath.Dir(t.Name.Package) + if !context.Universe.Package(internalPackage).Has(t.Name.Name) { + return nil, false + } + return context.Universe.Package(internalPackage).Type(t.Name.Name), true +} + +func Packages(context *generator.Context, arguments *args.GeneratorArgs) generator.Packages { + boilerplate, err := arguments.LoadGoBoilerplate() + if err != nil { + glog.Fatalf("Failed loading boilerplate: %v", err) + } + + inputs := sets.NewString(arguments.InputDirs...) + packages := generator.Packages{} + header := append([]byte( + ` +// +build !ignore_autogenerated + +`), boilerplate...) + header = append(header, []byte( + ` +// This file was autogenerated by conversion-gen. Do not edit it manually! + +`)...) + + // We are generating conversions only for packages that are explicitly + // passed as InputDir, and only for those that have a corresponding type + // (in the directory one above) and can be automatically converted to. + for _, p := range context.Universe { + path := p.Path + if !inputs.Has(path) { + continue + } + + convertibleType := false + for _, t := range p.Types { + // Check whether this type can be auto-converted to the internal + // version. + internalType, exists := getInternalTypeFor(context, t) + if !exists { + // There is no corresponding type in the internal package. + continue + } + if isConvertible(t, internalType) && isConvertible(internalType, t) { + convertibleType = true + } + } + + if convertibleType { + packages = append(packages, + &generator.DefaultPackage{ + PackageName: filepath.Base(path), + PackagePath: path, + HeaderText: header, + GeneratorFunc: func(c *generator.Context) (generators []generator.Generator) { + generators = []generator.Generator{} + generators = append( + generators, NewGenConversion("conversion_generated", path)) + return generators + }, + FilterFunc: func(c *generator.Context, t *types.Type) bool { + return t.Name.Package == path + }, + }) + } + } + return packages +} + +func findMember(t *types.Type, name string) (types.Member, bool) { + if t.Kind != types.Struct { + return types.Member{}, false + } + for _, member := range t.Members { + if member.Name == name { + return member, true + } + } + return types.Member{}, false +} + +func isConvertible(in, out *types.Type) bool { + // FIXME: Check manually-written conversion functions. + + // If one of the types is Alias, resolve it. + if in.Kind == types.Alias { + return isConvertible(in.Underlying, out) + } + if out.Kind == types.Alias { + return isConvertible(in, out.Underlying) + } + + if in.Kind != out.Kind { + return false + } + switch in.Kind { + case types.Builtin, types.Struct, types.Map, types.Slice, types.Pointer: + default: + // We don't support conversion of other types yet. + return false + } + switch out.Kind { + case types.Builtin, types.Struct, types.Map, types.Slice, types.Pointer: + default: + // We don't support conversion of other types yet. + return false + } + + switch in.Kind { + case types.Builtin: + // FIXME: Enough to be convertible - see AWSElastic + return in.Name == out.Name + case types.Struct: + convertible := true + for _, inMember := range in.Members { + // Check if there is an out member with that name. + outMember, found := findMember(out, inMember.Name) + if !found { + return false + } + convertible = convertible && isConvertible(inMember.Type, outMember.Type) + } + return convertible + case types.Map: + return isConvertible(in.Key, out.Key) && isConvertible(in.Elem, out.Elem) + case types.Slice: + return isConvertible(in.Elem, out.Elem) + case types.Pointer: + return isConvertible(in.Elem, out.Elem) + } + glog.Fatalf("All other types should be filtered before") + return false +} + +const ( + apiPackagePath = "k8s.io/kubernetes/pkg/api" + conversionPackagePath = "k8s.io/kubernetes/pkg/conversion" +) + +// genConversion produces a file with a autogenerated conversions. +type genConversion struct { + generator.DefaultGen + targetPackage string + imports namer.ImportTracker + typesForInit []*types.Type +} + +func NewGenConversion(sanitizedName, targetPackage string) generator.Generator { + return &genConversion{ + DefaultGen: generator.DefaultGen{ + OptionalName: sanitizedName, + }, + targetPackage: targetPackage, + imports: generator.NewImportTracker(), + typesForInit: make([]*types.Type, 0), + } +} + +func (g *genConversion) Namers(c *generator.Context) namer.NameSystems { + // Have the raw namer for this file track what it imports. + return namer.NameSystems{"raw": namer.NewRawNamer(g.targetPackage, g.imports)} +} + +func (g *genConversion) convertibleOnlyWithinPackage(inType, outType *types.Type) bool { + var t *types.Type + if inType.Name.Package == g.targetPackage { + t = inType + } else { + t = outType + } + + if t.Name.Package != g.targetPackage { + return false + } + if types.ExtractCommentTags("+", t.CommentLines)["genConversion"] == "false" { + return false + } + // TODO: Consider generating functions for other kinds too. + if t.Kind != types.Struct { + return false + } + // Also, filter out private types. + if namer.IsPrivateGoName(t.Name.Name) { + return false + } + return true +} + +func (g *genConversion) Filter(c *generator.Context, t *types.Type) bool { + internalType, exists := getInternalTypeFor(c, t) + if !g.convertibleOnlyWithinPackage(t, internalType) { + return false + } + if exists && isConvertible(t, internalType) && isConvertible(internalType, t) { + g.typesForInit = append(g.typesForInit, t) + return true + } + return false +} + +func (g *genConversion) isOtherPackage(pkg string) bool { + if pkg == g.targetPackage { + return false + } + if strings.HasSuffix(pkg, `"`+g.targetPackage+`"`) { + return false + } + return true +} + +func (g *genConversion) Imports(c *generator.Context) (imports []string) { + importLines := []string{"reflect \"reflect\""} + if g.isOtherPackage(apiPackagePath) { + importLines = append(importLines, "api \""+apiPackagePath+"\"") + } + if g.isOtherPackage(conversionPackagePath) { + importLines = append(importLines, "conversion \""+conversionPackagePath+"\"") + } + for _, singleImport := range g.imports.ImportLines() { + if g.isOtherPackage(singleImport) { + importLines = append(importLines, singleImport) + } + } + return importLines +} + +func argsFromType(inType, outType *types.Type) interface{} { + return map[string]interface{}{ + "inType": inType, + "outType": outType, + } +} + +func (g *genConversion) funcNameTmpl(inType, outType *types.Type) string { + tmpl := "Convert_$.inType|public$_To_$.outType|public$" + g.imports.AddType(inType) + g.imports.AddType(outType) + return tmpl +} + +func (g *genConversion) Init(c *generator.Context, w io.Writer) error { + sw := generator.NewSnippetWriter(w, c, "$", "$") + sw.Do("func init() {\n", nil) + if g.targetPackage == apiPackagePath { + sw.Do("if err := Scheme.AddGeneratedConversionFuncs(\n", nil) + } else { + sw.Do("if err := api.Scheme.AddGeneratedConversionFuncs(\n", nil) + } + for _, t := range g.typesForInit { + internalType, _ := getInternalTypeFor(c, t) + sw.Do(fmt.Sprintf("%s,\n", g.funcNameTmpl(t, internalType)), argsFromType(t, internalType)) + sw.Do(fmt.Sprintf("%s,\n", g.funcNameTmpl(internalType, t)), argsFromType(internalType, t)) + } + sw.Do("); err != nil {\n", nil) + sw.Do("// if one of the conversion functions is malformed, detect it immediately.\n", nil) + sw.Do("panic(err)\n", nil) + sw.Do("}\n", nil) + sw.Do("}\n\n", nil) + return sw.Error() +} + +func (g *genConversion) GenerateType(c *generator.Context, t *types.Type, w io.Writer) error { + sw := generator.NewSnippetWriter(w, c, "$", "$") + internalType, _ := getInternalTypeFor(c, t) + g.generateConversion(t, internalType, sw) + g.generateConversion(internalType, t, sw) + + return sw.Error() +} + +func (g *genConversion) generateConversion(inType, outType *types.Type, sw *generator.SnippetWriter) { + funcName := g.funcNameTmpl(inType, outType) + if g.targetPackage == conversionPackagePath { + sw.Do(fmt.Sprintf("func %s(in $.inType|raw$, out *$.outType|raw$, s *Scope) error {\n", funcName), argsFromType(inType, outType)) + } else { + sw.Do(fmt.Sprintf("func %s(in $.inType|raw$, out *$.outType|raw$, s *conversion.Scope) error {\n", funcName), argsFromType(inType, outType)) + } + // FIXME: Generate defaulting. + g.generateFor(inType, outType, sw) + sw.Do("return nil\n", nil) + sw.Do("}\n\n", nil) +} + +// we use the system of shadowing 'in' and 'out' so that the same code is valid +// at any nesting level. This makes the autogenerator easy to understand, and +// the compiler shouldn't care. +func (g *genConversion) generateFor(inType, outType *types.Type, sw *generator.SnippetWriter) { + var f func(*types.Type, *types.Type, *generator.SnippetWriter) + switch inType.Kind { + case types.Builtin: + f = g.doBuiltin + case types.Map: + f = g.doMap + case types.Slice: + f = g.doSlice + case types.Struct: + f = g.doStruct + case types.Pointer: + f = g.doPointer + case types.Alias: + f = g.doAlias + default: + f = g.doUnknown + } + f(inType, outType, sw) +} + +func (g *genConversion) doBuiltin(inType, outType *types.Type, sw *generator.SnippetWriter) { + sw.Do("*out = in\n", nil) +} + +func (g *genConversion) doMap(inType, outType *types.Type, sw *generator.SnippetWriter) { + sw.Do("*out = make($.|raw$)\n", outType) + if outType.Key.IsAssignable() { + sw.Do("for key, val := range in {\n", nil) + if outType.Elem.IsAssignable() { + if inType.Elem == outType.Elem { + sw.Do("(*out)[key] = val\n", nil) + } else { + sw.Do("(*out)[key] = $.|raw$(val)\n", outType.Elem) + } + } else { + sw.Do("newVal := new($.|raw$)\n", outType.Elem) + if g.convertibleOnlyWithinPackage(inType.Elem, outType.Elem) { + funcName := g.funcNameTmpl(inType.Elem, outType.Elem) + sw.Do(fmt.Sprintf("if err := %s(val, newVal, s); err != nil {\n", funcName), argsFromType(inType.Elem, outType.Elem)) + } else { + sw.Do("// TODO: Inefficient conversion - can we improve it?\n", nil) + sw.Do("if err := s.Convert(val, newVal, 0); err != nil {\n", nil) + } + sw.Do("return err\n", nil) + sw.Do("}\n", nil) + sw.Do("(*out)[key] = *newVal\n", nil) + } + } else { + // TODO: Implement it when necessary. + sw.Do("for range in {\n", nil) + sw.Do("// FIXME: Converting unassignable keys unsupported $.|raw$\n", inType.Key) + } + sw.Do("}\n", nil) +} + +func (g *genConversion) doSlice(inType, outType *types.Type, sw *generator.SnippetWriter) { + sw.Do("*out = make($.|raw$, len(in))\n", outType) + if inType.Elem == outType.Elem && inType.Elem.Kind == types.Builtin { + sw.Do("copy(*out, in)\n", nil) + } else { + sw.Do("for i := range in {\n", nil) + if outType.Elem.IsAssignable() { + if inType.Elem == outType.Elem { + sw.Do("(*out)[i] = in[i]\n", nil) + } else { + sw.Do("(*out)[i] = $.|raw$(in[i])\n", outType.Elem) + } + } else { + if g.convertibleOnlyWithinPackage(inType.Elem, outType.Elem) { + funcName := g.funcNameTmpl(inType.Elem, outType.Elem) + sw.Do(fmt.Sprintf("if err := %s(in[i], &(*out)[i], c); err != nil {\n", funcName), argsFromType(inType.Elem, outType.Elem)) + } else { + sw.Do("// TODO: Inefficient conversion - can we improve it?\n", nil) + sw.Do("if err := s.Convert(in[i], &out[i], 0); err != nil {\n", nil) + } + sw.Do("return err\n", nil) + sw.Do("}\n", nil) + } + sw.Do("}\n", nil) + } +} + +func (g *genConversion) doStruct(inType, outType *types.Type, sw *generator.SnippetWriter) { + for _, m := range inType.Members { + outMember, _ := findMember(outType, m.Name) + args := map[string]interface{}{ + "inType": m.Type, + "outType": outMember.Type, + "name": m.Name, + } + switch m.Type.Kind { + case types.Builtin: + sw.Do("out.$.name$ = in.$.name$\n", args) + case types.Map, types.Slice, types.Pointer: + sw.Do("if in.$.name$ != nil {\n", args) + sw.Do("in, out := in.$.name$, &out.$.name$\n", args) + g.generateFor(m.Type, outMember.Type, sw) + sw.Do("} else {\n", nil) + sw.Do("out.$.name$ = nil\n", args) + sw.Do("}\n", nil) + case types.Struct: + if g.convertibleOnlyWithinPackage(m.Type, outMember.Type) { + funcName := g.funcNameTmpl(m.Type, outMember.Type) + sw.Do(fmt.Sprintf("if err := %s(in.$.name$, &out.$.name$, c); err != nil {\n", funcName), args) + } else { + sw.Do("// TODO: Inefficient conversion - can we improve it?\n", nil) + sw.Do("if err := s.Convert(in.$.name$, &out.$.name$, 0); err != nil {\n", args) + } + sw.Do("return err\n", nil) + sw.Do("}\n", nil) + case types.Alias: + if outMember.Type.IsAssignable() { + sw.Do("out.$.name$ = $.outType|raw$(in.$.name$)\n", args) + } else { + if g.convertibleOnlyWithinPackage(m.Type, outMember.Type) { + funcName := g.funcNameTmpl(m.Type, outMember.Type) + sw.Do(fmt.Sprintf("if err := %s(in.$.name$, &out.$.name$, c); err != nil {\n", funcName), args) + } else { + sw.Do("// TODO: Inefficient conversion - can we improve it?\n", nil) + sw.Do("if err := s.Convert(in.$.name$, &out.$.name$, 0); err != nil {\n", args) + } + sw.Do("return err\n", nil) + sw.Do("}\n", nil) + } + default: + if g.convertibleOnlyWithinPackage(m.Type, outMember.Type) { + funcName := g.funcNameTmpl(m.Type, outMember.Type) + sw.Do(fmt.Sprintf("if err := %s(in.$.name$, &out.$.name$, c); err != nil {\n", funcName), args) + } else { + sw.Do("// TODO: Inefficient conversion - can we improve it?\n", nil) + sw.Do("if err := s.Convert(in.$.name$, &out.$.name$, 0); err != nil {\n", args) + } + sw.Do("return err\n", nil) + sw.Do("}\n", nil) + } + } +} + +func (g *genConversion) doPointer(inType, outType *types.Type, sw *generator.SnippetWriter) { + sw.Do("*out = new($.Elem|raw$)\n", outType) + if outType.Elem.IsAssignable() { + if inType.Elem == outType.Elem { + sw.Do("**out = *in\n", nil) + } else { + sw.Do("**out = $.|raw$(*in)\n", outType.Elem) + } + } else { + if g.convertibleOnlyWithinPackage(inType.Elem, outType.Elem) { + funcName := g.funcNameTmpl(inType.Elem, outType.Elem) + sw.Do(fmt.Sprintf("if err := %s(*in, out, c); err != nil {\n", funcName), argsFromType(inType.Elem, outType.Elem)) + } else { + sw.Do("// TODO: Inefficient conversion - can we improve it?\n", nil) + sw.Do("if err := s.Convert(*in, out, 0); err != nil {\n", nil) + } + sw.Do("return err\n", nil) + sw.Do("}\n", nil) + } +} + +func (g *genConversion) doAlias(inType, outType *types.Type, sw *generator.SnippetWriter) { + // TODO: Add support for aliases. + g.doUnknown(inType, outType, sw) +} + +func (g *genConversion) doUnknown(inType, outType *types.Type, sw *generator.SnippetWriter) { + sw.Do("// FIXME: Type $.|raw$ is unsupported.\n", inType) +} diff --git a/cmd/libs/go2idl/conversion-gen/main.go b/cmd/libs/go2idl/conversion-gen/main.go new file mode 100644 index 0000000000000..e1a177821efe4 --- /dev/null +++ b/cmd/libs/go2idl/conversion-gen/main.go @@ -0,0 +1,47 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// conversion-gen is a tool for auto-generating Conversion functions. +// +// Structs in the input directories with the below line in their comments +// will be ignored during generation. +// // +genconversion=false +package main + +import ( + "k8s.io/kubernetes/cmd/libs/go2idl/args" + "k8s.io/kubernetes/cmd/libs/go2idl/conversion-gen/generators" + + "github.com/golang/glog" +) + +func main() { + arguments := args.Default() + + // Override defaults. These are Kubernetes specific input locations. + arguments.InputDirs = []string{ + "k8s.io/kubernetes/pkg/api/v1", + } + + if err := arguments.Execute( + generators.NameSystems(), + generators.DefaultNameSystem(), + generators.Packages, + ); err != nil { + glog.Fatalf("Error: %v", err) + } + glog.Info("Completed successfully.") +} diff --git a/cmd/libs/go2idl/deepcopy-gen/generators/deepcopy.go b/cmd/libs/go2idl/deepcopy-gen/generators/deepcopy.go index ce469d300f5ab..a7190397710d9 100644 --- a/cmd/libs/go2idl/deepcopy-gen/generators/deepcopy.go +++ b/cmd/libs/go2idl/deepcopy-gen/generators/deepcopy.go @@ -95,10 +95,7 @@ func Packages(context *generator.Context, arguments *args.GeneratorArgs) generat return generators }, FilterFunc: func(c *generator.Context, t *types.Type) bool { - if t.Name.Package != path { - return false - } - return copyableWithinPackage(t) + return t.Name.Package == path }, }) } @@ -111,7 +108,7 @@ const ( conversionPackagePath = "k8s.io/kubernetes/pkg/conversion" ) -// genDeepCopy produces a file with a set for a single type. +// genDeepCopy produces a file with autogenerated deep-copy functions. type genDeepCopy struct { generator.DefaultGen targetPackage string @@ -137,7 +134,6 @@ func (g *genDeepCopy) Namers(c *generator.Context) namer.NameSystems { return namer.NameSystems{"raw": namer.NewRawNamer(g.targetPackage, g.imports)} } -// Filter ignores all but one type because we're making a single file per type. func (g *genDeepCopy) Filter(c *generator.Context, t *types.Type) bool { // Filter out all types not copyable within the package. copyable := copyableWithinPackage(t) @@ -231,7 +227,6 @@ func (g *genDeepCopy) Init(c *generator.Context, w io.Writer) error { return sw.Error() } -// GenerateType makes the body of a file implementing a set for type t. func (g *genDeepCopy) GenerateType(c *generator.Context, t *types.Type, w io.Writer) error { sw := generator.NewSnippetWriter(w, c, "$", "$") funcName := g.funcNameTmpl(t)