From 9b8be2aff366edd604ade86efee5957b90b8e690 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Fri, 6 Oct 2023 16:38:30 +0200 Subject: [PATCH] re-implemented ResetProcessor to apply on map[string]any Signed-off-by: Nicolas De Loof --- loader/extends.go | 42 +++++++----- loader/fix.go | 36 ++++++++++ loader/loader.go | 97 ++++++++++++++++---------- loader/loader_test.go | 2 + loader/loader_yaml_test.go | 6 +- loader/null.go | 64 +++++------------- loader/validation.go | 82 ++++++++++++++++++++++ override/extends.go | 4 +- override/merge.go | 40 +++++------ override/merge_environment_test.go | 3 + override/merge_logging_test.go | 3 + override/merge_test.go | 24 +++---- override/uncity.go | 31 +++++---- override/uncity_test.go | 3 + paths/resolve.go | 105 +++++++++++++++++++++++++++++ transform/build.go | 38 +++++++++++ transform/canonical.go | 73 +++++++++++++------- transform/external.go | 35 ++++++++++ transform/named.go | 36 ++++++++++ transform/ports.go | 28 ++++++-- transform/services.go | 14 ++-- transform/services_test.go | 21 +++--- transform/ulimits.go | 35 ++++++++++ transform/volume.go | 49 ++++++++++++++ tree/path.go | 13 ++++ types/mapping.go | 5 +- 26 files changed, 691 insertions(+), 198 deletions(-) create mode 100644 loader/fix.go create mode 100644 loader/validation.go create mode 100644 paths/resolve.go create mode 100644 transform/build.go create mode 100644 transform/external.go create mode 100644 transform/named.go create mode 100644 transform/ulimits.go create mode 100644 transform/volume.go diff --git a/loader/extends.go b/loader/extends.go index d1a799cb9..57e222970 100644 --- a/loader/extends.go +++ b/loader/extends.go @@ -19,21 +19,22 @@ package loader import ( "context" "fmt" + "path/filepath" "github.com/compose-spec/compose-go/override" "github.com/compose-spec/compose-go/types" ) -func ApplyExtends(ctx context.Context, dict map[string]interface{}, opts *Options) error { - services := dict["services"].(map[string]interface{}) +func ApplyExtends(ctx context.Context, dict map[string]any, opts *Options, post ...PostProcessor) error { + services := dict["services"].(map[string]any) for name, s := range services { - service := s.(map[string]interface{}) + service := s.(map[string]any) x, ok := service["extends"] if !ok { continue } - extends := x.(map[string]interface{}) - var base interface{} + extends := x.(map[string]any) + var base any ref := extends["service"].(string) if file, ok := extends["file"]; ok { path := file.(string) @@ -43,15 +44,18 @@ func ApplyExtends(ctx context.Context, dict map[string]interface{}, opts *Option if err != nil { return err } - source, err := loadYamlModel(ctx, []types.ConfigFile{ - {Filename: local}, + source, err := loadYamlModel(ctx, types.ConfigDetails{ + WorkingDir: filepath.Dir(path), + ConfigFiles: []types.ConfigFile{ + {Filename: local}, + }, }, opts) if err != nil { return err } - services := source["services"].([]interface{}) + services := source["services"].([]any) for _, s := range services { - service := s.(map[string]interface{}) + service := s.(map[string]any) if service["name"] == ref { base = service break @@ -71,7 +75,15 @@ func ApplyExtends(ctx context.Context, dict map[string]interface{}, opts *Option return fmt.Errorf("cannot extend service %q in %s: service not found", name, "filename") //TODO track filename } } - merged, err := override.ExtendService(deepClone(base).(map[string]interface{}), service) + source := deepClone(base).(map[string]any) + for _, processor := range post { + processor.Apply(map[string]any{ + "services": map[string]any{ + name: source, + }, + }) + } + merged, err := override.ExtendService(source, service) if err != nil { return err } @@ -81,16 +93,16 @@ func ApplyExtends(ctx context.Context, dict map[string]interface{}, opts *Option return nil } -func deepClone(value interface{}) interface{} { +func deepClone(value any) any { switch v := value.(type) { - case []interface{}: - cp := make([]interface{}, len(v)) + case []any: + cp := make([]any, len(v)) for i, e := range v { cp[i] = deepClone(e) } return cp - case map[string]interface{}: - cp := make(map[string]interface{}, len(v)) + case map[string]any: + cp := make(map[string]any, len(v)) for k, e := range v { cp[k] = deepClone(e) } diff --git a/loader/fix.go b/loader/fix.go new file mode 100644 index 000000000..7a6e88d81 --- /dev/null +++ b/loader/fix.go @@ -0,0 +1,36 @@ +/* + Copyright 2020 The Compose Specification Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package loader + +// fixEmptyNotNull is a workaround for https://github.com/xeipuuv/gojsonschema/issues/141 +// as go-yaml `[]` will load as a `[]any(nil)`, which is not the same as an empty array +func fixEmptyNotNull(value any) interface{} { + switch v := value.(type) { + case []any: + if v == nil { + return []any{} + } + for i, e := range v { + v[i] = fixEmptyNotNull(e) + } + case map[string]any: + for k, e := range v { + v[k] = fixEmptyNotNull(e) + } + } + return value +} diff --git a/loader/loader.go b/loader/loader.go index a6299660f..fcabcce84 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -22,7 +22,7 @@ import ( "fmt" "io" "os" - paths "path" + "path" "path/filepath" "reflect" "regexp" @@ -33,6 +33,7 @@ import ( "github.com/compose-spec/compose-go/format" interp "github.com/compose-spec/compose-go/interpolation" "github.com/compose-spec/compose-go/override" + "github.com/compose-spec/compose-go/paths" "github.com/compose-spec/compose-go/schema" "github.com/compose-spec/compose-go/template" "github.com/compose-spec/compose-go/transform" @@ -51,9 +52,9 @@ type Options struct { SkipInterpolation bool // Skip normalization SkipNormalization bool - // Resolve paths + // Resolve path ResolvePaths bool - // Convert Windows paths + // Convert Windows path ConvertWindowsPaths bool // Skip consistency check SkipConsistencyCheck bool @@ -265,12 +266,12 @@ func LoadWithContext(ctx context.Context, configDetails types.ConfigDetails, opt return load(ctx, configDetails, opts, nil) } -func loadYamlModel(ctx context.Context, configFiles []types.ConfigFile, opts *Options) (map[string]interface{}, error) { +func loadYamlModel(ctx context.Context, config types.ConfigDetails, opts *Options) (map[string]interface{}, error) { var ( dict = map[string]interface{}{} err error ) - for _, file := range configFiles { + for _, file := range config.ConfigFiles { if len(file.Content) == 0 { content, err := os.ReadFile(file.Filename) if err != nil { @@ -282,8 +283,8 @@ func loadYamlModel(ctx context.Context, configFiles []types.ConfigFile, opts *Op r := bytes.NewReader(file.Content) decoder := yaml.NewDecoder(r) for { - var cfg map[string]interface{} - processor := ResetProcessor{target: &cfg} + var raw interface{} + processor := ResetProcessor{target: &raw} err := decoder.Decode(&processor) if err == io.EOF { break @@ -292,6 +293,24 @@ func loadYamlModel(ctx context.Context, configFiles []types.ConfigFile, opts *Op return nil, err } + converted, err := convertToStringKeysRecursive(raw, "") + if err != nil { + return nil, err + } + cfg, ok := converted.(map[string]interface{}) + if !ok { + return nil, errors.New("Top-level object must be a mapping") + } + + if opts.Interpolate != nil && !opts.SkipInterpolation { + cfg, err = interp.Interpolate(cfg, *opts.Interpolate) + if err != nil { + return nil, err + } + } + + fixEmptyNotNull(cfg) + if !opts.SkipValidation { if err := schema.Validate(cfg); err != nil { return nil, fmt.Errorf("validating %s: %w", file.Filename, err) @@ -302,6 +321,14 @@ func loadYamlModel(ctx context.Context, configFiles []types.ConfigFile, opts *Op if err != nil { return nil, err } + + if !opts.SkipExtends { + err = ApplyExtends(ctx, cfg, opts, &processor) + if err != nil { + return nil, err + } + } + dict, err = override.Merge(dict, cfg) if err != nil { return nil, err @@ -309,32 +336,27 @@ func loadYamlModel(ctx context.Context, configFiles []types.ConfigFile, opts *Op } } - if opts.Interpolate != nil && !opts.SkipInterpolation { - dict, err = interp.Interpolate(dict, *opts.Interpolate) - if err != nil { - return nil, err - } - } - - if !opts.SkipExtends { - err = ApplyExtends(ctx, dict, opts) - if err != nil { - return nil, err - } + dict, err = override.EnforceUnicity(dict) + if err != nil { + return nil, err } - dict, err = override.EnforceUnicity(dict) + dict, err = transform.Canonical(dict) if err != nil { return nil, err } dict = groupXFieldsIntoExtensions(dict) - dict, err = transform.Canonical(dict) - if err != nil { + // TODO(ndeloof) shall we implement this as a Validate func on types ? + if err := Validate(dict); err != nil { return nil, err } + if opts.ResolvePaths { + paths.ResolveRelativePaths(dict, config.WorkingDir) + } + return dict, nil } @@ -350,7 +372,7 @@ func load(ctx context.Context, configDetails types.ConfigDetails, opts *Options, includeRefs := make(map[string][]types.IncludeConfig) - dict, err := loadYamlModel(ctx, configDetails.ConfigFiles, opts) + dict, err := loadYamlModel(ctx, configDetails, opts) if err != nil { return nil, err } @@ -523,8 +545,15 @@ func groupXFieldsIntoExtensions(dict map[string]interface{}) map[string]interfac extras[key] = value delete(dict, key) } - if d, ok := value.(map[string]interface{}); ok { - dict[key] = groupXFieldsIntoExtensions(d) + switch v := value.(type) { + case map[string]interface{}: + dict[key] = groupXFieldsIntoExtensions(v) + case []interface{}: + for i, e := range v { + if m, ok := e.(map[string]interface{}); ok { + v[i] = groupXFieldsIntoExtensions(m) + } + } } } if len(extras) > 0 { @@ -816,9 +845,9 @@ func loadServiceWithExtends(ctx context.Context, filename, name string, services return nil, err } - // Make paths relative to the importing Compose file. Note that we - // make the paths relative to `file` rather than `baseFilePath` so - // that the resulting paths won't be absolute if `file` isn't an + // Make path relative to the importing Compose file. Note that we + // make the path relative to `file` rather than `baseFilePath` so + // that the resulting path won't be absolute if `file` isn't an // absolute path. baseFileParent := filepath.Dir(file) @@ -860,8 +889,8 @@ func LoadService(name string, serviceDict map[string]interface{}) (*types.Servic return serviceConfig, nil } -// Windows paths, c:\\my\\path\\shiny, need to be changed to be compatible with -// the Engine. Volume paths are expected to be linux style /c/my/path/shiny/ +// Windows path, c:\\my\\path\\shiny, need to be changed to be compatible with +// the Engine. Volume path are expected to be linux style /c/my/path/shiny/ func convertVolumePath(volume types.ServiceVolumeConfig) types.ServiceVolumeConfig { volumeName := strings.ToLower(filepath.VolumeName(volume.Source)) if len(volumeName) != 2 { @@ -875,15 +904,15 @@ func convertVolumePath(volume types.ServiceVolumeConfig) types.ServiceVolumeConf return volume } -func resolveMaybeUnixPath(workingDir string, path string) string { - filePath := expandUser(path) +func resolveMaybeUnixPath(workingDir string, p string) string { + filePath := expandUser(p) // Check if source is an absolute path (either Unix or Windows), to // handle a Windows client with a Unix daemon or vice-versa. // // Note that this is not required for Docker for Windows when specifying // a local Windows path, because Docker for Windows translates the Windows // path into a valid path within the VM. - if !paths.IsAbs(filePath) && !isAbs(filePath) { + if !path.IsAbs(filePath) && !isAbs(filePath) { filePath = absPath(workingDir, filePath) } return filePath @@ -1128,7 +1157,7 @@ func cleanTarget(target string) string { if target == "" { return "" } - return paths.Clean(target) + return path.Clean(target) } var transformBuildConfig TransformerFunc = func(data interface{}) (interface{}, error) { diff --git a/loader/loader_test.go b/loader/loader_test.go index a3f4a1753..3a320cd75 100644 --- a/loader/loader_test.go +++ b/loader/loader_test.go @@ -752,6 +752,7 @@ services: volumes: - source: data type: volume + target: /data read_only: $thebool volume: nocopy: $thebool @@ -852,6 +853,7 @@ networks: { Source: "data", Type: "volume", + Target: "/data", ReadOnly: true, Volume: &types.ServiceVolumeVolume{NoCopy: true}, }, diff --git a/loader/loader_yaml_test.go b/loader/loader_yaml_test.go index 1b5270309..b54e2d24b 100644 --- a/loader/loader_yaml_test.go +++ b/loader/loader_yaml_test.go @@ -25,8 +25,8 @@ import ( ) func TestParseYAMLFiles(t *testing.T) { - model, err := loadYamlModel(context.TODO(), - []types.ConfigFile{ + model, err := loadYamlModel(context.TODO(), types.ConfigDetails{ + ConfigFiles: []types.ConfigFile{ {Filename: "test.yaml", Content: []byte(` services: @@ -43,7 +43,7 @@ services: image: bar command: echo world init: false -`)}}, &Options{}) +`)}}}, &Options{}) assert.NilError(t, err) assert.DeepEqual(t, model, map[string]interface{}{ "services": map[string]interface{}{ diff --git a/loader/null.go b/loader/null.go index 58dce5393..2e0994471 100644 --- a/loader/null.go +++ b/loader/null.go @@ -18,9 +18,7 @@ package loader import ( "fmt" - "reflect" "strconv" - "strings" "github.com/compose-spec/compose-go/tree" "gopkg.in/yaml.v3" @@ -74,70 +72,38 @@ func (p *ResetProcessor) resolveReset(node *yaml.Node, path tree.Path) (*yaml.No // Apply finds the go attributes matching recorded paths and reset them to zero value func (p *ResetProcessor) Apply(target any) error { - return p.applyNullOverrides(reflect.ValueOf(target), tree.NewPath()) + return p.applyNullOverrides(target, tree.NewPath()) } // applyNullOverrides set val to Zero if it matches any of the recorded paths -func (p *ResetProcessor) applyNullOverrides(val reflect.Value, path tree.Path) error { - val = reflect.Indirect(val) - if !val.IsValid() { - return nil - } - typ := val.Type() - switch { - case typ.Kind() == reflect.Map: - iter := val.MapRange() +func (p *ResetProcessor) applyNullOverrides(target any, path tree.Path) error { + switch v := target.(type) { + case map[string]any: KEYS: - for iter.Next() { - k := iter.Key() - next := path.Next(k.String()) + for k, e := range v { + next := path.Next(k) for _, pattern := range p.paths { if next.Matches(pattern) { - val.SetMapIndex(k, reflect.Value{}) + delete(v, k) continue KEYS } } - return p.applyNullOverrides(iter.Value(), next) + err := p.applyNullOverrides(e, next) + if err != nil { + return err + } } - case typ.Kind() == reflect.Slice: + case []any: ITER: - for i := 0; i < val.Len(); i++ { + for i, e := range v { next := path.Next(fmt.Sprintf("[%d]", i)) for _, pattern := range p.paths { if next.Matches(pattern) { - continue ITER + // TODO(ndeloof) support removal from sequence } } - // TODO(ndeloof) support removal from sequence - return p.applyNullOverrides(val.Index(i), next) - } - - case typ.Kind() == reflect.Struct: - FIELDS: - for i := 0; i < typ.NumField(); i++ { - field := typ.Field(i) - name := field.Name - attr := strings.ToLower(name) - tag := field.Tag.Get("yaml") - tag = strings.Split(tag, ",")[0] - if tag != "" && tag != "-" { - attr = tag - } - next := path.Next(attr) - f := val.Field(i) - for _, pattern := range p.paths { - if next.Matches(pattern) { - f := f - if !f.CanSet() { - return fmt.Errorf("can't override attribute %s", name) - } - // f.SetZero() requires go 1.20 - f.Set(reflect.Zero(f.Type())) - continue FIELDS - } - } - err := p.applyNullOverrides(f, next) + err := p.applyNullOverrides(e, next) if err != nil { return err } diff --git a/loader/validation.go b/loader/validation.go new file mode 100644 index 000000000..89cffc456 --- /dev/null +++ b/loader/validation.go @@ -0,0 +1,82 @@ +/* + Copyright 2020 The Compose Specification Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package loader + +import ( + "fmt" + + "github.com/compose-spec/compose-go/tree" +) + +type checkerFunc func(value any, p tree.Path) error + +var checks = map[tree.Path]checkerFunc{ + "volumes.*": checkExternalVolume, +} + +func Validate(dict map[string]any) error { + return check(dict, tree.NewPath()) +} + +func check(value any, p tree.Path) error { + for pattern, fn := range checks { + if p.Matches(pattern) { + return fn(value, p) + } + } + switch v := value.(type) { + case map[string]any: + for k, v := range v { + err := check(v, p.Next(k)) + if err != nil { + return err + } + } + case []any: + for _, e := range v { + err := check(e, p.Next("[]")) + if err != nil { + return err + } + } + } + return nil +} + +func checkExternalVolume(value any, p tree.Path) error { + switch v := value.(type) { + case map[string]any: + if _, ok := v["external"]; !ok { + return nil + } + for k, e := range v { + switch k { + case "name", extensions: + continue + case "external": + vname := v["name"] + ename, ok := e.(map[string]any)["name"] + if ok && vname != nil && ename != vname { + return fmt.Errorf("volume %s: volume.external.name and volume.name conflict; only use volume.name", p.Last()) + } + default: + return externalVolumeError(p.Last(), k) + } + } + } + return nil +} diff --git a/override/extends.go b/override/extends.go index b46ddad4d..795269ad8 100644 --- a/override/extends.go +++ b/override/extends.go @@ -18,10 +18,10 @@ package override import "github.com/compose-spec/compose-go/tree" -func ExtendService(base, override map[string]interface{}) (map[string]interface{}, error) { +func ExtendService(base, override map[string]any) (map[string]any, error) { yaml, err := mergeYaml(base, override, tree.NewPath("services.x")) if err != nil { return nil, err } - return yaml.(map[string]interface{}), nil + return yaml.(map[string]any), nil } diff --git a/override/merge.go b/override/merge.go index cbe4adc04..7d355ada0 100644 --- a/override/merge.go +++ b/override/merge.go @@ -24,15 +24,15 @@ import ( ) // Merge applies overrides to a config model -func Merge(right, left map[string]interface{}) (map[string]interface{}, error) { +func Merge(right, left map[string]any) (map[string]any, error) { merged, err := mergeYaml(right, left, tree.NewPath()) if err != nil { return nil, err } - return merged.(map[string]interface{}), nil + return merged.(map[string]any), nil } -type merger func(interface{}, interface{}, tree.Path) (interface{}, error) +type merger func(any, any, tree.Path) (any, error) // mergeSpecials defines the custom rules applied by compose when merging yaml trees var mergeSpecials = map[tree.Path]merger{} @@ -45,8 +45,8 @@ func init() { mergeSpecials["services.*.environment"] = mergeEnvironment } -// mergeYaml merges map[string]interface{} yaml trees handling special rules -func mergeYaml(e interface{}, o interface{}, p tree.Path) (interface{}, error) { +// mergeYaml merges map[string]any yaml trees handling special rules +func mergeYaml(e any, o any, p tree.Path) (any, error) { for pattern, merger := range mergeSpecials { if p.Matches(pattern) { merged, err := merger(e, o, p) @@ -57,14 +57,14 @@ func mergeYaml(e interface{}, o interface{}, p tree.Path) (interface{}, error) { } } switch value := e.(type) { - case map[string]interface{}: - other, ok := o.(map[string]interface{}) + case map[string]any: + other, ok := o.(map[string]any) if !ok { return nil, fmt.Errorf("cannont override %s", p) } return mergeMappings(value, other, p) - case []interface{}: - other, ok := o.([]interface{}) + case []any: + other, ok := o.([]any) if !ok { return nil, fmt.Errorf("cannont override %s", p) } @@ -74,7 +74,7 @@ func mergeYaml(e interface{}, o interface{}, p tree.Path) (interface{}, error) { } } -func mergeMappings(mapping map[string]interface{}, other map[string]interface{}, p tree.Path) (map[string]interface{}, error) { +func mergeMappings(mapping map[string]any, other map[string]any, p tree.Path) (map[string]any, error) { for k, v := range other { next := p.Next(k) e, ok := mapping[k] @@ -92,9 +92,9 @@ func mergeMappings(mapping map[string]interface{}, other map[string]interface{}, } // logging driver options are merged only when both compose file define the same driver -func mergeLogging(c interface{}, o interface{}, p tree.Path) (interface{}, error) { - config := c.(map[string]interface{}) - other := o.(map[string]interface{}) +func mergeLogging(c any, o any, p tree.Path) (any, error) { + config := c.(map[string]any) + other := o.(map[string]any) // we override logging config if source and override have the same driver set, or none d, ok1 := other["driver"] o, ok2 := config["driver"] @@ -105,16 +105,16 @@ func mergeLogging(c interface{}, o interface{}, p tree.Path) (interface{}, error } // environment must be first converted into yaml sequence syntax so we can append -func mergeEnvironment(c interface{}, o interface{}, p tree.Path) (interface{}, error) { +func mergeEnvironment(c any, o any, p tree.Path) (any, error) { right := convertIntoSequence(c) left := convertIntoSequence(o) return append(right, left...), nil } -func convertIntoSequence(value interface{}) []interface{} { +func convertIntoSequence(value any) []any { switch v := value.(type) { - case map[string]interface{}: - seq := make([]interface{}, len(v)) + case map[string]any: + seq := make([]any, len(v)) i := 0 for k, v := range v { if v == nil { @@ -124,16 +124,16 @@ func convertIntoSequence(value interface{}) []interface{} { } i++ } - slices.SortFunc(seq, func(a, b interface{}) bool { + slices.SortFunc(seq, func(a, b any) bool { return a.(string) < b.(string) }) return seq - case []interface{}: + case []any: return v } return nil } -func override(c interface{}, other interface{}, p tree.Path) (interface{}, error) { +func override(c any, other any, p tree.Path) (any, error) { return other, nil } diff --git a/override/merge_environment_test.go b/override/merge_environment_test.go index 1df6e06a3..67f25dfcb 100644 --- a/override/merge_environment_test.go +++ b/override/merge_environment_test.go @@ -1,9 +1,12 @@ /* Copyright 2020 The Compose Specification Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/override/merge_logging_test.go b/override/merge_logging_test.go index 2ba7d324b..c4c04a629 100644 --- a/override/merge_logging_test.go +++ b/override/merge_logging_test.go @@ -1,9 +1,12 @@ /* Copyright 2020 The Compose Specification Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/override/merge_test.go b/override/merge_test.go index 0aafaccd7..77af93ca9 100644 --- a/override/merge_test.go +++ b/override/merge_test.go @@ -1,9 +1,12 @@ /* Copyright 2020 The Compose Specification Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,31 +25,26 @@ import ( // override using the same logging driver will override driver options func Test_mergeOverrides(t *testing.T) { - configs := []string{` + right := ` services: test: image: foo scale: 1 -`, ` +` + left := ` services: test: image: bar -`, ` -services: - test: scale: 2 -`} +` expected := ` services: test: image: bar scale: 2 ` - models := make([]map[string]interface{}, len(configs)) - for i, config := range configs { - models[i] = unmarshall(t, config) - } - got, err := Merge(models...) + + got, err := Merge(unmarshall(t, right), unmarshall(t, left)) assert.NilError(t, err) assert.DeepEqual(t, got, unmarshall(t, expected)) } @@ -57,8 +55,8 @@ func assertMergeYaml(t *testing.T, right string, left string, want string) { assert.DeepEqual(t, got, unmarshall(t, want)) } -func unmarshall(t *testing.T, s string) map[string]interface{} { - var val map[string]interface{} +func unmarshall(t *testing.T, s string) map[string]any { + var val map[string]any err := yaml.Unmarshal([]byte(s), &val) assert.NilError(t, err) return val diff --git a/override/uncity.go b/override/uncity.go index 5dfa2fd86..f1b29cb93 100644 --- a/override/uncity.go +++ b/override/uncity.go @@ -17,13 +17,14 @@ package override import ( + "fmt" "strings" "github.com/compose-spec/compose-go/format" "github.com/compose-spec/compose-go/tree" ) -type indexer func(interface{}) (string, error) +type indexer func(any, tree.Path) (string, error) // mergeSpecials defines the custom rules applied by compose when merging yaml trees var unique = map[tree.Path]indexer{} @@ -34,17 +35,17 @@ func init() { } // EnforceUnicity removes redefinition of elements declared in a sequence -func EnforceUnicity(value map[string]interface{}) (map[string]interface{}, error) { +func EnforceUnicity(value map[string]any) (map[string]any, error) { uniq, err := enforceUnicity(value, tree.NewPath()) if err != nil { return nil, err } - return uniq.(map[string]interface{}), nil + return uniq.(map[string]any), nil } -func enforceUnicity(value interface{}, p tree.Path) (interface{}, error) { +func enforceUnicity(value any, p tree.Path) (any, error) { switch v := value.(type) { - case map[string]interface{}: + case map[string]any: for k, e := range v { u, err := enforceUnicity(e, p.Next(k)) if err != nil { @@ -53,13 +54,13 @@ func enforceUnicity(value interface{}, p tree.Path) (interface{}, error) { v[k] = u } return v, nil - case []interface{}: + case []any: for pattern, indexer := range unique { if p.Matches(pattern) { - var seq []interface{} + var seq []any keys := map[string]int{} - for _, entry := range v { - key, err := indexer(entry) + for i, entry := range v { + key, err := indexer(entry, p.Next(fmt.Sprintf("[%d]", i))) if err != nil { return nil, err } @@ -77,7 +78,7 @@ func enforceUnicity(value interface{}, p tree.Path) (interface{}, error) { return value, nil } -func environmentIndexer(y interface{}) (string, error) { +func environmentIndexer(y any, p tree.Path) (string, error) { value := y.(string) key, _, found := strings.Cut(value, "=") if !found { @@ -86,10 +87,14 @@ func environmentIndexer(y interface{}) (string, error) { return key, nil } -func volumeIndexer(y interface{}) (string, error) { +func volumeIndexer(y any, p tree.Path) (string, error) { switch value := y.(type) { - case map[string]interface{}: - return value["target"].(string), nil + case map[string]any: + target, ok := value["target"].(string) + if !ok { + return "", fmt.Errorf("service volume %s is missing a mount target", p) + } + return target, nil case string: volume, err := format.ParseVolume(value) if err != nil { diff --git a/override/uncity_test.go b/override/uncity_test.go index 92177d2ce..3ad46f154 100644 --- a/override/uncity_test.go +++ b/override/uncity_test.go @@ -1,9 +1,12 @@ /* Copyright 2020 The Compose Specification Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/paths/resolve.go b/paths/resolve.go new file mode 100644 index 000000000..a4f042ad8 --- /dev/null +++ b/paths/resolve.go @@ -0,0 +1,105 @@ +/* + Copyright 2020 The Compose Specification Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package paths + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/compose-spec/compose-go/tree" +) + +type resolver func(any) (any, error) + +// ResolveRelativePaths make relative paths absolute +func ResolveRelativePaths(project map[string]any, base string) error { + r := relativePathsResolver{workingDir: base} + r.resolvers = map[tree.Path]resolver{ + "services.*.build.context": r.absPath, // TODO(ndeloof) need to detect remote + "services.*.build.additional_contexts.*": r.absPath, // TODO(ndeloof) need to detect remote + "services.*.env_file": r.absPath, + "services.*.extends.file": r.absPath, + "services.*.develop.watch.*.path": r.absPath, + "services.*.volume.source": r.absPath, // TODO(ndeloof) bind only, maybe unix path + "config.file": r.absPath, + "secret.file": r.absPath, + "include.path": r.absPath, + "include.project_directory": r.absPath, + "include.env_file": r.absPath, + } + _, err := r.resolveRelativePaths(project, tree.NewPath()) + return err +} + +type relativePathsResolver struct { + workingDir string + resolvers map[tree.Path]resolver +} + +func (r *relativePathsResolver) resolveRelativePaths(value any, p tree.Path) (any, error) { + for pattern, resolver := range r.resolvers { + if p.Matches(pattern) { + return resolver(value) + } + } + switch value.(type) { + case map[string]any: + mapping := value.(map[string]any) + for k, v := range mapping { + resolved, err := r.resolveRelativePaths(v, p.Next(k)) + if err != nil { + return nil, err + } + mapping[k] = resolved + } + case []any: + sequence := value.([]any) + for i, v := range sequence { + resolved, err := r.resolveRelativePaths(v, p.Next("[]")) + if err != nil { + return nil, err + } + sequence[i] = resolved + } + } + return value, nil +} + +func (r *relativePathsResolver) absPath(value any) (any, error) { + switch v := value.(type) { + case []any: + for i, s := range v { + abs, err := r.absPath(s) + if err != nil { + return nil, err + } + v[i] = abs + } + case string: + if strings.HasPrefix(v, "~") { + home, _ := os.UserHomeDir() + return filepath.Join(home, v[1:]), nil + } + if filepath.IsAbs(v) { + return v, nil + } + return filepath.Join(r.workingDir, v), nil + } + return nil, fmt.Errorf("unexpected type %T", value) +} diff --git a/transform/build.go b/transform/build.go new file mode 100644 index 000000000..268691a0b --- /dev/null +++ b/transform/build.go @@ -0,0 +1,38 @@ +/* + Copyright 2020 The Compose Specification Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package transform + +import ( + "github.com/compose-spec/compose-go/tree" + "github.com/pkg/errors" +) + +func transformBuild(data any, p tree.Path) (any, error) { + switch v := data.(type) { + case map[string]any: + if _, ok := v["context"]; !ok { + v["context"] = "." // TODO(ndeloof) maybe we miss an explicit "set-defaults" loading phase + } + return v, nil + case string: + return map[string]any{ + "context": v, + }, nil + default: + return data, errors.Errorf("invalid type %T for build", v) + } +} diff --git a/transform/canonical.go b/transform/canonical.go index 4b8f22eaa..bdc16735b 100644 --- a/transform/canonical.go +++ b/transform/canonical.go @@ -20,26 +20,37 @@ import ( "github.com/compose-spec/compose-go/tree" ) -type transformFunc func(data interface{}, p tree.Path) (interface{}, error) +type transformFunc func(data any, p tree.Path) (any, error) var transformers = map[tree.Path]transformFunc{} func init() { transformers["services"] = makeServicesSlice transformers["services.*.networks"] = transformServiceNetworks - transformers["services.*.ports"] = trasformPorts + transformers["services.*.volumes.*"] = transformVolume + transformers["services.*.ports"] = transformPorts + transformers["services.*.build"] = transformBuild + transformers["services.*.ulimits.*"] = transformUlimits + transformers["volumes.*.external"] = transformExternal + transformers["networks.*.external"] = transformExternal + transformers["secrets.*.external"] = transformExternal + transformers["configs.*.external"] = transformExternal + transformers["volumes.*"] = transformNamed + transformers["networks.*"] = transformNamed + transformers["configs.*"] = transformNamed + transformers["secrets.*"] = transformNamed } // Canonical transforms a compose model into canonical syntax -func Canonical(yaml map[string]interface{}) (map[string]interface{}, error) { +func Canonical(yaml map[string]any) (map[string]any, error) { canonical, err := transform(yaml, tree.NewPath()) if err != nil { return nil, err } - return canonical.(map[string]interface{}), nil + return canonical.(map[string]any), nil } -func transform(data interface{}, p tree.Path) (interface{}, error) { +func transform(data any, p tree.Path) (any, error) { for pattern, transformer := range transformers { if p.Matches(pattern) { t, err := transformer(data, p) @@ -49,28 +60,42 @@ func transform(data interface{}, p tree.Path) (interface{}, error) { return t, nil } } - switch data.(type) { - case map[string]interface{}: - mapping := data.(map[string]interface{}) - for k, v := range mapping { - t, err := transform(v, p.Next(k)) - if err != nil { - return nil, err - } - mapping[k] = t + switch v := data.(type) { + case map[string]any: + a, err := transformMapping(v, p) + if err != nil { + return a, err } - return mapping, nil - case []interface{}: - sequence := data.([]interface{}) - for i, e := range sequence { - t, err := transform(e, p.Next("[]")) - if err != nil { - return nil, err - } - sequence[i] = t + return v, nil + case []any: + a, err := transformSequence(v, p) + if err != nil { + return a, err } - return sequence, nil + return v, nil default: return data, nil } } + +func transformSequence(v []any, p tree.Path) (any, error) { + for i, e := range v { + t, err := transform(e, p.Next("[]")) + if err != nil { + return nil, err + } + v[i] = t + } + return v, nil +} + +func transformMapping(v map[string]any, p tree.Path) (any, error) { + for k, e := range v { + t, err := transform(e, p.Next(k)) + if err != nil { + return nil, err + } + v[k] = t + } + return v, nil +} diff --git a/transform/external.go b/transform/external.go new file mode 100644 index 000000000..b8b8ea319 --- /dev/null +++ b/transform/external.go @@ -0,0 +1,35 @@ +/* + Copyright 2020 The Compose Specification Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package transform + +import ( + "github.com/compose-spec/compose-go/tree" + "github.com/pkg/errors" +) + +func transformExternal(data any, p tree.Path) (any, error) { + switch v := data.(type) { + case map[string]any: + return v, nil + case bool: + return map[string]any{ + "external": v, + }, nil + default: + return data, errors.Errorf("invalid type %T for external", v) + } +} diff --git a/transform/named.go b/transform/named.go new file mode 100644 index 000000000..27f55e46f --- /dev/null +++ b/transform/named.go @@ -0,0 +1,36 @@ +/* + Copyright 2020 The Compose Specification Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package transform + +import ( + "github.com/compose-spec/compose-go/tree" + "github.com/pkg/errors" +) + +func transformNamed(data any, p tree.Path) (any, error) { + switch v := data.(type) { + case map[string]any: + if _, ok := v["name"]; !ok { + v["name"] = p.Last() + } + return transformMapping(v, p) + case nil: + return nil, nil + default: + return data, errors.Errorf("invalid type %T for %s", v, p) + } +} diff --git a/transform/ports.go b/transform/ports.go index 96215e445..7a51ba5b0 100644 --- a/transform/ports.go +++ b/transform/ports.go @@ -1,3 +1,19 @@ +/* + Copyright 2020 The Compose Specification Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + package transform import ( @@ -9,13 +25,13 @@ import ( "github.com/pkg/errors" ) -func trasformPorts(data interface{}, p tree.Path) (interface{}, error) { +func transformPorts(data any, p tree.Path) (any, error) { switch entries := data.(type) { - case []interface{}: + case []any: // We process the list instead of individual items here. // The reason is that one entry might be mapped to multiple ServicePortConfig. // Therefore we take an input of a list and return an output of a list. - var ports []interface{} + var ports []any for _, entry := range entries { switch value := entry.(type) { case int: @@ -24,7 +40,7 @@ func trasformPorts(data interface{}, p tree.Path) (interface{}, error) { return data, err } for _, v := range parsed { - m := map[string]interface{}{} + m := map[string]any{} err := mapstructure.Decode(v, &m) if err != nil { return nil, err @@ -37,14 +53,14 @@ func trasformPorts(data interface{}, p tree.Path) (interface{}, error) { return data, err } for _, v := range parsed { - m := map[string]interface{}{} + m := map[string]any{} err := mapstructure.Decode(v, &m) if err != nil { return nil, err } ports = append(ports, m) } - case map[string]interface{}: + case map[string]any: ports = append(ports, value) default: return data, errors.Errorf("invalid type %T for port", value) diff --git a/transform/services.go b/transform/services.go index 34c3bfc7d..84de19565 100644 --- a/transform/services.go +++ b/transform/services.go @@ -20,12 +20,12 @@ import ( "github.com/compose-spec/compose-go/tree" ) -func makeServicesSlice(data interface{}, p tree.Path) (interface{}, error) { - services := data.(map[string]interface{}) - servicesAsSlice := make([]interface{}, len(services)) +func makeServicesSlice(data any, p tree.Path) (any, error) { + services := data.(map[string]any) + servicesAsSlice := make([]any, len(services)) i := 0 for name, it := range services { - config := it.(map[string]interface{}) + config := it.(map[string]any) config["name"] = name if _, ok := config["scale"]; !ok { config["scale"] = 1 // TODO(ndeloof) we should make Scale a *int @@ -40,9 +40,9 @@ func makeServicesSlice(data interface{}, p tree.Path) (interface{}, error) { return servicesAsSlice, nil } -func transformServiceNetworks(data interface{}, p tree.Path) (interface{}, error) { - if slice, ok := data.([]interface{}); ok { - networks := make(map[string]interface{}, len(slice)) +func transformServiceNetworks(data any, p tree.Path) (any, error) { + if slice, ok := data.([]any); ok { + networks := make(map[string]any, len(slice)) for _, net := range slice { networks[net.(string)] = nil } diff --git a/transform/services_test.go b/transform/services_test.go index be54d300f..76132458f 100644 --- a/transform/services_test.go +++ b/transform/services_test.go @@ -26,8 +26,8 @@ import ( ) func TestMakeServiceSlide(t *testing.T) { - var mapping interface{} - yaml.Unmarshal([]byte(` + var mapping any + err := yaml.Unmarshal([]byte(` foo: image: foo bar: @@ -35,26 +35,27 @@ bar: zot: image: zot `), &mapping) + assert.NilError(t, err) slice, err := makeServicesSlice(mapping, tree.NewPath("services")) assert.NilError(t, err) - services := slice.([]interface{}) - slices.SortFunc(services, func(a, b interface{}) bool { - right := a.(map[string]interface{}) - left := b.(map[string]interface{}) + services := slice.([]any) + slices.SortFunc(services, func(a, b any) bool { + right := a.(map[string]any) + left := b.(map[string]any) return right["name"].(string) < left["name"].(string) }) - assert.DeepEqual(t, services, []interface{}{ - map[string]interface{}{ + assert.DeepEqual(t, services, []any{ + map[string]any{ "name": "bar", "image": "bar", }, - map[string]interface{}{ + map[string]any{ "name": "foo", "image": "foo", }, - map[string]interface{}{ + map[string]any{ "name": "zot", "image": "zot", }, diff --git a/transform/ulimits.go b/transform/ulimits.go new file mode 100644 index 000000000..5fa16c961 --- /dev/null +++ b/transform/ulimits.go @@ -0,0 +1,35 @@ +/* + Copyright 2020 The Compose Specification Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package transform + +import ( + "github.com/compose-spec/compose-go/tree" + "github.com/pkg/errors" +) + +func transformUlimits(data any, p tree.Path) (any, error) { + switch v := data.(type) { + case map[string]any: + return v, nil + case int: + return map[string]any{ + "single": v, + }, nil + default: + return data, errors.Errorf("invalid type %T for external", v) + } +} diff --git a/transform/volume.go b/transform/volume.go new file mode 100644 index 000000000..025bca796 --- /dev/null +++ b/transform/volume.go @@ -0,0 +1,49 @@ +/* + Copyright 2020 The Compose Specification Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package transform + +import ( + "github.com/compose-spec/compose-go/format" + "github.com/compose-spec/compose-go/tree" + "github.com/mitchellh/mapstructure" + "github.com/pkg/errors" +) + +func transformVolume(data any, p tree.Path) (any, error) { + switch v := data.(type) { + case map[string]any: + return v, nil + case string: + volume, err := format.ParseVolume(v) //TODO(ndeloof) ParseVolume should not rely on types and return map[string] + if err != nil { + return nil, err + } + + yaml := map[string]any{} + decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{ + TagName: "yaml", + Result: &yaml, + }) + if err != nil { + return nil, err + } + + return yaml, decoder.Decode(volume) + default: + return data, errors.Errorf("invalid type %T for build", v) + } +} diff --git a/tree/path.go b/tree/path.go index 4f91a14b6..b73038b1b 100644 --- a/tree/path.go +++ b/tree/path.go @@ -67,3 +67,16 @@ func (p Path) Matches(pattern Path) bool { } return true } + +func (p Path) Last() string { + parts := p.Parts() + return parts[len(parts)-1] +} + +func (p Path) Parent() Path { + index := strings.LastIndex(string(p), pathSeparator) + if index > 0 { + return p[0:index] + } + return "" +} diff --git a/types/mapping.go b/types/mapping.go index 32667258f..43e471405 100644 --- a/types/mapping.go +++ b/types/mapping.go @@ -85,9 +85,10 @@ func (m *MappingWithEquals) DecodeMapstructure(value interface{}) error { for _, s := range v { k, e, ok := strings.Cut(fmt.Sprint(s), "=") if !ok { - return fmt.Errorf("invalid label %q", v) + mapping[k] = nil + } else { + mapping[k] = mappingValue(e) } - mapping[k] = mappingValue(e) } *m = mapping default: