From febf5ed8ddd3bb57db83b3d000ebf661d90c5a39 Mon Sep 17 00:00:00 2001 From: magodo Date: Wed, 18 Aug 2021 12:01:17 +0800 Subject: [PATCH] resolve dependency via arm template and add it into hcl --- go.mod | 2 + internal/armtemplate/armtemplate.go | 185 +++++++---- internal/armtemplate/armtemplate_test.go | 388 +++++++++++++++++++++++ internal/meta.go | 133 ++++++-- internal/run.go | 9 +- 5 files changed, 619 insertions(+), 98 deletions(-) create mode 100644 internal/armtemplate/armtemplate_test.go diff --git a/go.mod b/go.mod index 4a114a2..6257afb 100644 --- a/go.mod +++ b/go.mod @@ -10,4 +10,6 @@ require ( github.com/hashicorp/hcl/v2 v2.10.1 github.com/hashicorp/terraform-exec v0.14.1-0.20210812105923-7fa6ba66697a github.com/hashicorp/terraform-schema v0.0.0-20210804102346-b9355678f0bc + github.com/stretchr/testify v1.7.0 // indirect + github.com/zclconf/go-cty v1.9.0 // indirect ) diff --git a/internal/armtemplate/armtemplate.go b/internal/armtemplate/armtemplate.go index af304bb..4a6fa7f 100644 --- a/internal/armtemplate/armtemplate.go +++ b/internal/armtemplate/armtemplate.go @@ -1,84 +1,139 @@ package armtemplate -type Template struct { - Schema string `json:"$schema"` - ContentVersion string `json:"contentVersion"` - ApiProfile string `json:"apiProfile,omitempty"` - Parameters map[string]Parameter `json:"parameters,omitempty"` - Variables map[string]interface{} `json:"variables,omitempty"` - Functions []Function `json:"functions,omitempty"` - Resources []Resource `json:"resources"` - Outputs []Output `json:"outputs,omitempty"` -} - -type ParameterType string - -const ( - ParameterTypeString ParameterType = "string" - ParameterTypeSecureString = "securestring" - ParameterTypeInt = "int" - ParameterTypeBool = "bool" - ParameterTypeObject = "object" - ParameterTypeSecureObject = "secureObject" - ParameterTypeArray = "array" +import ( + "encoding/json" + "fmt" + "regexp" + "strings" ) -type Parameter struct { - Type ParameterType `json:"type"` - DefaultValue interface{} `json:"defaultValue,omitempty"` - AllowedValues []interface{} `json:"allowedValues,omitempty"` - MinValue int `json:"minValue,omitempty"` - MaxValue int `json:"maxValue,omitempty"` - MinLength int `json:"minLength,omitempty"` - MaxLength int `json:"maxLength,omitempty"` - Metadata *ParameterMetadata `json:"metadata,omitempty"` +type Template struct { + Resources []Resource `json:"resources"` } -type ParameterMetadata struct { - Description string `json:"description,omitempty"` +type Resource struct { + ResourceId + DependsOn Dependencies `json:"dependsOn,omitempty"` } -type Function struct { - Namespace string `json:"namespace"` - Members map[string]FunctionMember `json:"members"` +type ResourceId struct { + Type string `json:"type"` + Name string `json:"name"` } -type FunctionMember struct { - Parameters []FunctionParameter `json:"parameters,omitempty"` - Output FunctionOutput `json:"output"` -} +var ResourceGroupId = ResourceId{} + +func NewResourceId(id string) (*ResourceId, error) { + id = strings.TrimPrefix(id, "/") + id = strings.TrimSuffix(id, "/") + segs := strings.Split(id, "/") + if len(segs)%2 != 0 { + return nil, fmt.Errorf("invalid resource id format of %q: amount of segments is not even", id) + } + // ==4: resource group + // >=8: general resources resides in a resource group + if len(segs) != 4 && len(segs) < 8 { + return nil, fmt.Errorf("invalid resource id format of %q: amount of segments is too small", id) + } + if segs[0] != "subscriptions" { + return nil, fmt.Errorf("invalid resource id format of %q: the 1st segment is not subscriptions", id) + } + segs = segs[2:] + if !strings.EqualFold(segs[0], "resourcegroups") { + return nil, fmt.Errorf("invalid resource id format of %q: the 2nd segment is not resourcegroups (case insensitive)", id) + } + segs = segs[2:] + + if len(segs) == 0 { + return &ResourceGroupId, nil + } + + if segs[0] != "providers" { + return nil, fmt.Errorf("invalid resource id format of %q: the 3rd segment is not providers", id) + } + providerName := segs[1] + segs = segs[2:] + + t := []string{providerName} + n := []string{} -type FunctionParameter struct { - Name string `json:"name"` - Type ParameterType `json:"type"` + for i := 0; i < len(segs); i += 2 { + t = append(t, segs[0]) + n = append(n, segs[1]) + } + + return &ResourceId{ + Type: strings.Join(t, "/"), + Name: strings.Join(n, "/"), + }, nil } -type FunctionOutput struct { - Type ParameterType `json:"type"` - Value interface{} `json:"value"` +// ID returns the azure resource id +func (res ResourceId) ID(sub, rg string) string { + typeSegs := strings.Split(res.Type, "/") + nameSegs := strings.Split(res.Name, "/") + + out := []string{"subscriptions", sub, "resourceGroups", rg} + if len(typeSegs) != 0 { + if len(typeSegs)-1 != len(nameSegs) { + panic(fmt.Sprintf("The resource of type %q and name %q is not a valid identifier", res.Type, res.Name)) + } + out = append(out, typeSegs[0]) + for i := 0; i < len(nameSegs); i++ { + out = append(out, typeSegs[i+1]) + out = append(out, nameSegs[i]) + } + } + return strings.Join(out, "/") } -type Resource struct { - Condition *bool `json:"condition,omitempty"` - Type string `json:"type"` - ApiVersion string `json:"apiVersion"` - Name string `json:"name"` - Comments string `json:"comments,omitempty"` - Location string `json:"location,omitempty"` - DependsOn []Dependency `json:"dependsOn,omitempty"` - Tags map[string]string `json:"tags,omitempty"` - Sku *ResourceSku `json:"sku,omitempty"` - Kind string `json:"kind,omitempty"` - Scope string `json:"scope,omitempty"` - //TODO Copy +type Dependencies []ResourceId + +func (deps *Dependencies) UnmarshalJSON(b []byte) error { + var dependenciesRaw []string + if err := json.Unmarshal(b, &dependenciesRaw); err != nil { + return err + } + + for _, dep := range dependenciesRaw { + matches := regexp.MustCompile(`^\[resourceId\(([^,]+), (.+)\)]$`).FindAllStringSubmatch(dep, 1) + if len(matches) == 0 { + panic(fmt.Sprintf("the dependency %q is not valid (no match)", dep)) + } + m := matches[0] + if len(m) != 3 { + panic(fmt.Sprintf("the dependency %q is not valid (the matched one has invalid form)", dep)) + } + + tlit, nlit := m[1], m[2] + + t := strings.Trim(tlit, "' ") + + var names []string + for _, seg := range strings.Split(nlit, ",") { + names = append(names, strings.Trim(seg, "' ")) + } + n := strings.Join(names, "/") + + *deps = append(*deps, ResourceId{ + Type: t, + Name: n, + }) + } + return nil } -type Dependency string +type DependencyInfo map[ResourceId][]ResourceId + +func (tpl Template) DependencyInfo(rgName string) DependencyInfo { + s := map[ResourceId][]ResourceId{} + for _, res := range tpl.Resources { + if len(res.DependsOn) == 0 { + s[res.ResourceId] = []ResourceId{ResourceGroupId} + continue + } -type ResourceSku struct { - Name string `json:"name"` - Tier string `json:"tier,omitempty"` - Size string `json:"size,omitempty"` - Family string `json:family,omitempty"` - Capacity string `json:"capacity,omitempty"` + s[res.ResourceId] = res.DependsOn + } + return s } diff --git a/internal/armtemplate/armtemplate_test.go b/internal/armtemplate/armtemplate_test.go new file mode 100644 index 0000000..990cc76 --- /dev/null +++ b/internal/armtemplate/armtemplate_test.go @@ -0,0 +1,388 @@ +package armtemplate_test + +import ( + "encoding/json" + "testing" + + "github.com/magodo/tfy/internal/armtemplate" + "github.com/stretchr/testify/require" +) + +func TestUnmarshalTemplate(t *testing.T) { + cases := []struct { + name string + input string + expect armtemplate.Template + }{ + { + name: "one level dependency", + input: ` +{ + "resources": [ + { + "type": "Microsoft.Storage/storageAccounts", + "name": "a" + }, + { + "type": "Microsoft.Storage/storageAccounts/fileServices", + "name": "a/default", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', 'a')]" + ] + } + ] +} +`, + expect: armtemplate.Template{ + Resources: []armtemplate.Resource{ + { + ResourceId: armtemplate.ResourceId{ + Type: "Microsoft.Storage/storageAccounts", + Name: "a", + }, + DependsOn: nil, + }, + { + ResourceId: armtemplate.ResourceId{ + Type: "Microsoft.Storage/storageAccounts/fileServices", + Name: "a/default", + }, + DependsOn: armtemplate.Dependencies{ + { + Type: "Microsoft.Storage/storageAccounts", + Name: "a", + }, + }, + }, + }, + }, + }, + { + name: "multi-level dependency", + input: ` +{ + "resources": [ + { + "type": "Microsoft.Network/networkInterfaces", + "name": "nic", + "dependsOn": [ + "[resourceId('Microsoft.Network/publicIPAddresses', 'pip')]", + "[resourceId('Microsoft.Network/virtualNetworks/subnets', 'vnet', 'subnet')]", + "[resourceId('Microsoft.Network/networkSecurityGroups', 'nsg')]" + ] + }, + { + "type": "Microsoft.Network/virtualNetworks/subnets", + "name": "vnet/subnet", + "dependsOn": [ + "[resourceId('Microsoft.Network/virtualNetworks', 'vnet')]", + "[resourceId('Microsoft.Network/networkSecurityGroups', 'nsg')]" + ] + }, + { + "type": "Microsoft.Network/networkSecurityGroups/securityRules", + "name": "nsg/nsr", + "dependsOn": [ + "[resourceId('Microsoft.Network/networkSecurityGroups', 'nsg')]" + ] + }, + { + "type": "Microsoft.Network/networkSecurityGroups", + "name": "nsg" + }, + { + "type": "Microsoft.Network/virtualNetworks", + "name": "vnet" + }, + { + "type": "Microsoft.Network/publicIPAddresses", + "name": "pip" + } + ] +} +`, + expect: armtemplate.Template{ + Resources: []armtemplate.Resource{ + { + ResourceId: armtemplate.ResourceId{ + Type: "Microsoft.Network/networkInterfaces", + Name: "nic", + }, + DependsOn: armtemplate.Dependencies{ + { + Type: "Microsoft.Network/publicIPAddresses", + Name: "pip", + }, + { + Type: "Microsoft.Network/virtualNetworks/subnets", + Name: "vnet/subnet", + }, + { + Type: "Microsoft.Network/networkSecurityGroups", + Name: "nsg", + }, + }, + }, + { + ResourceId: armtemplate.ResourceId{ + Type: "Microsoft.Network/virtualNetworks/subnets", + Name: "vnet/subnet", + }, + DependsOn: armtemplate.Dependencies{ + { + Type: "Microsoft.Network/virtualNetworks", + Name: "vnet", + }, + { + Type: "Microsoft.Network/networkSecurityGroups", + Name: "nsg", + }, + }, + }, + { + ResourceId: armtemplate.ResourceId{ + Type: "Microsoft.Network/networkSecurityGroups/securityRules", + Name: "nsg/nsr", + }, + DependsOn: armtemplate.Dependencies{ + { + Type: "Microsoft.Network/networkSecurityGroups", + Name: "nsg", + }, + }, + }, + { + ResourceId: armtemplate.ResourceId{ + Type: "Microsoft.Network/networkSecurityGroups", + Name: "nsg", + }, + }, + { + ResourceId: armtemplate.ResourceId{ + Type: "Microsoft.Network/virtualNetworks", + Name: "vnet", + }, + }, + { + ResourceId: armtemplate.ResourceId{ + Type: "Microsoft.Network/publicIPAddresses", + Name: "pip", + }, + }, + }, + }, + }, + } + + for _, c := range cases { + var out armtemplate.Template + require.NoError(t, json.Unmarshal([]byte(c.input), &out), c.name) + require.Equal(t, c.expect, out, c.name) + } +} + +func TestDependencyInfo(t *testing.T) { + cases := []struct { + name string + input armtemplate.Template + expect armtemplate.DependencyInfo + }{ + { + name: "multiple-level dependency", + input: armtemplate.Template{ + Resources: []armtemplate.Resource{ + { + ResourceId: armtemplate.ResourceId{ + Type: "Microsoft.Network/networkInterfaces", + Name: "nic", + }, + DependsOn: armtemplate.Dependencies{ + { + Type: "Microsoft.Network/publicIPAddresses", + Name: "pip", + }, + { + Type: "Microsoft.Network/virtualNetworks/subnets", + Name: "vnet/subnet", + }, + { + Type: "Microsoft.Network/networkSecurityGroups", + Name: "nsg", + }, + }, + }, + { + ResourceId: armtemplate.ResourceId{ + Type: "Microsoft.Network/virtualNetworks/subnets", + Name: "vnet/subnet", + }, + DependsOn: armtemplate.Dependencies{ + { + Type: "Microsoft.Network/virtualNetworks", + Name: "vnet", + }, + { + Type: "Microsoft.Network/networkSecurityGroups", + Name: "nsg", + }, + }, + }, + { + ResourceId: armtemplate.ResourceId{ + Type: "Microsoft.Network/networkSecurityGroups/securityRules", + Name: "nsg/nsr", + }, + DependsOn: armtemplate.Dependencies{ + { + Type: "Microsoft.Network/networkSecurityGroups", + Name: "nsg", + }, + }, + }, + { + ResourceId: armtemplate.ResourceId{ + Type: "Microsoft.Network/networkSecurityGroups", + Name: "nsg", + }, + }, + { + ResourceId: armtemplate.ResourceId{ + Type: "Microsoft.Network/virtualNetworks", + Name: "vnet", + }, + }, + { + ResourceId: armtemplate.ResourceId{ + Type: "Microsoft.Network/publicIPAddresses", + Name: "pip", + }, + }, + }, + }, + expect: map[armtemplate.ResourceId][]armtemplate.ResourceId{ + armtemplate.ResourceId{ + Type: "Microsoft.Network/networkInterfaces", + Name: "nic", + }: { + { + Type: "Microsoft.Network/publicIPAddresses", + Name: "pip", + }, + { + Type: "Microsoft.Network/virtualNetworks/subnets", + Name: "vnet/subnet", + }, + { + Type: "Microsoft.Network/networkSecurityGroups", + Name: "nsg", + }, + }, + armtemplate.ResourceId{ + Type: "Microsoft.Network/virtualNetworks/subnets", + Name: "vnet/subnet", + }: { + { + Type: "Microsoft.Network/virtualNetworks", + Name: "vnet", + }, + { + Type: "Microsoft.Network/networkSecurityGroups", + Name: "nsg", + }, + }, + armtemplate.ResourceId{ + Type: "Microsoft.Network/networkSecurityGroups/securityRules", + Name: "nsg/nsr", + }: { + { + Type: "Microsoft.Network/networkSecurityGroups", + Name: "nsg", + }, + }, + armtemplate.ResourceId{ + Type: "Microsoft.Network/networkSecurityGroups", + Name: "nsg", + }: { + armtemplate.ResourceGroupId, + }, + armtemplate.ResourceId{ + Type: "Microsoft.Network/virtualNetworks", + Name: "vnet", + }: { + armtemplate.ResourceGroupId, + }, + armtemplate.ResourceId{ + Type: "Microsoft.Network/publicIPAddresses", + Name: "pip", + }: { + armtemplate.ResourceGroupId, + }, + }, + }, + } + + for _, c := range cases { + require.Equal(t, c.expect, c.input.DependencyInfo("rg"), c.name) + } +} + +func TestNewResourceId(t *testing.T) { + cases := []struct { + name string + input string + expect armtemplate.ResourceId + error bool + }{ + { + name: "empty", + input: "", + error: true, + }, + { + name: "only subscription", + input: "/subscriptions/1234", + error: true, + }, + { + name: "only subscription and resource group", + input: "/subscriptions/1234/resourceGroups/rg1", + expect: armtemplate.ResourceGroupId, + }, + { + name: "only subscription, resource group and provider", + input: "/subscriptions/1234/resourceGroups/rg1/providers/Microsoft.Network", + error: true, + }, + { + name: "valid vnet id", + input: "/subscriptions/1234/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1", + expect: armtemplate.ResourceId{ + Type: "Microsoft.Network/virtualNetworks", + Name: "vnet1", + }, + }, + { + name: "valid vnet id (small case resourcegroups)", + input: "/subscriptions/1234/resourcegroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1", + expect: armtemplate.ResourceId{ + Type: "Microsoft.Network/virtualNetworks", + Name: "vnet1", + }, + }, + { + name: "invalid subnet id", + input: "/subscriptions/1234/resourceGroups/rg1/providers/Microsoft.Network/virtualNetworks/vnet1/subnets", + error: true, + }, + } + + for _, c := range cases { + output, err := armtemplate.NewResourceId(c.input) + if c.error { + require.Error(t, err, c.name) + continue + } + require.NoError(t, err, c.name) + require.Equal(t, c.expect, *output, c.name) + } +} diff --git a/internal/meta.go b/internal/meta.go index 04a4233..99bafa3 100644 --- a/internal/meta.go +++ b/internal/meta.go @@ -3,25 +3,28 @@ package internal import ( "bufio" "context" + "encoding/json" "fmt" + "io" "os" "path/filepath" "strings" + "github.com/magodo/tfy/internal/armtemplate" + + "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2020-06-01/resources" "github.com/hashicorp/go-version" "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/hashicorp/hcl/v2/hclwrite" "github.com/hashicorp/terraform-exec/tfexec" - "github.com/hashicorp/terraform-schema/earlydecoder" - "github.com/hashicorp/terraform-schema/module" ) type Meta struct { - resourceGroup string - workspace string - tf *tfexec.Terraform - auth *Authorizer + subscriptionId string + resourceGroup string + workspace string + tf *tfexec.Terraform + auth *Authorizer } func NewMeta(ctx context.Context, rg string) (*Meta, error) { @@ -68,10 +71,11 @@ func NewMeta(ctx context.Context, rg string) (*Meta, error) { } return &Meta{ - resourceGroup: rg, - workspace: wsp, - tf: tf, - auth: auth, + subscriptionId: auth.Config.SubscriptionID, + resourceGroup: rg, + workspace: wsp, + tf: tf, + auth: auth, }, nil } @@ -248,6 +252,10 @@ type ConfigInfo struct { hcl *hclwrite.File } +func (cfg ConfigInfo) DumpHCL(w io.Writer) (int, error) { + return w.Write(hclwrite.Format(cfg.hcl.Bytes())) +} + func (meta *Meta) StateToConfig(ctx context.Context, list ImportList) (ConfigInfos, error) { out := ConfigInfos{} for _, item := range list { @@ -276,27 +284,96 @@ func (meta *Meta) StateToConfig(ctx context.Context, list ImportList) (ConfigInf }) } - // var buf strings.Builder - // for _, cfg := range out { - // if _, err := cfg.hcl.WriteTo(&buf); err != nil { - // return nil, err - // } - // } - // fmt.Println(buf.String()) + return out, nil +} + +func (meta *Meta) ResolveDependency(ctx context.Context, configs ConfigInfos) (ConfigInfos, error) { + client := meta.auth.NewResourceGroupClient() + + exportOpt := "SkipAllParameterization" + future, err := client.ExportTemplate(ctx, meta.resourceGroup, resources.ExportTemplateRequest{ + ResourcesProperty: &[]string{"*"}, + Options: &exportOpt, + }) + if err != nil { + return nil, fmt.Errorf("exporting arm template of resource group %s: %w", meta.resourceGroup, err) + } + + if err := future.WaitForCompletionRef(ctx, client.Client); err != nil { + return nil, fmt.Errorf("waiting for exporting arm template of resource group %s: %w", meta.resourceGroup, err) + } + + result, err := future.Result(client) + if err != nil { + return nil, fmt.Errorf("getting the arm template of resource group %s: %w", meta.resourceGroup, err) + } + + // The response has been read into the ".Template" field as an interface, and the reader has been drained. + // As we have defined some (useful) types for the arm template, so we will do a json marshal then unmarshal here + // to convert the ".Template" (interface{}) into that artificial type. + raw, err := json.Marshal(result.Template) + if err != nil { + return nil, fmt.Errorf("marshalling the template: %w", err) + } + var tpl armtemplate.Template + if err := json.Unmarshal(raw, &tpl); err != nil { + return nil, fmt.Errorf("unmarshalling the template: %w", err) + } + + depInfo := tpl.DependencyInfo(meta.resourceGroup) + configSet := map[armtemplate.ResourceId]ConfigInfo{} + for _, cfg := range configs { + armId, err := armtemplate.NewResourceId(cfg.ResourceID) + if err != nil { + return nil, fmt.Errorf("new arm tempalte resource id from azure resource id: %w", err) + } + configSet[*armId] = cfg + } + + // Iterate each config to add dependency by querying the dependency info from arm template. + var out ConfigInfos + for armId, cfg := range configSet { + if armId == armtemplate.ResourceGroupId { + out = append(out, cfg) + continue + } + // This should never happen + if _, ok := depInfo[armId]; !ok { + return nil, fmt.Errorf("resource %q appeared in the list result of resource group %q, but didn't show up in the arm template", armId.ID(meta.subscriptionId, meta.resourceGroup), meta.resourceGroup) + } + + meta.hclBlockAppendDependency(cfg.hcl.Body().Blocks()[0].Body(), depInfo[armId], configSet) + out = append(out, cfg) + } + + for _, cfg := range out { + cfg.DumpHCL(os.Stdout) + } return out, nil } -func tfschemaModule(filename string, b []byte) (*module.Meta, error) { - f, diags := hclsyntax.ParseConfig(b, filename, hcl.InitialPos) - if diags.HasErrors() { - return nil, fmt.Errorf("parsing HCL for %q: %s", filename, diags.Error()) +func (meta *Meta) hclBlockAppendDependency(body *hclwrite.Body, armIds []armtemplate.ResourceId, cfgset map[armtemplate.ResourceId]ConfigInfo) error { + blk := hclwrite.NewBlock("lifecycle", nil) + dependencies := []string{} + for _, armid := range armIds { + cfg, ok := cfgset[armid] + if !ok { + dependencies = append(dependencies, fmt.Sprintf("# Depending on %q, but it is not imported by Terraform. Please fix it manually.", armid.ID(meta.subscriptionId, meta.resourceGroup))) + continue + } + dependencies = append(dependencies, cfg.TFAddr()+",") } - meta, diags := earlydecoder.LoadModule("root", map[string]*hcl.File{ - filename: f, - }) - if diags.HasErrors() { - return nil, fmt.Errorf("loading module: %s", diags.Error()) + if len(dependencies) > 0 { + src := []byte("depends_on = [\n" + strings.Join(dependencies, "\n") + "\n]") + expr, err := hclwrite.ParseConfig(src, "generate_depends_on", hcl.InitialPos) + if err != nil { + return fmt.Errorf(`building "depends_on" attribute: %w`, err) + } + + blk.Body().SetAttributeRaw("depends_on", expr.BuildTokens(nil)) + body.AppendBlock(blk) } - return meta, nil + + return nil } diff --git a/internal/run.go b/internal/run.go index 75e4604..37995c7 100644 --- a/internal/run.go +++ b/internal/run.go @@ -38,11 +38,10 @@ func Run(ctx context.Context, rg string) error { return err } - _ = configs - - ////////////////////////// - // Markup the dependencies - ////////////////////////// + configs, err = meta.ResolveDependency(ctx, configs) + if err != nil { + return err + } return nil }