Merge pull request #174 from turkenh/conversion-from-tfjson

Conversion from tfjson to consume schema from cli output
crossplane · Feb 21, 2022 · 391b759 · 391b759
2 parents b8a22b6 + 9a0c15d
commit 391b759
Show file tree

Hide file tree

Showing 6 changed files with 276 additions and 97 deletions.
diff --git a/docs/generating-a-provider.md b/docs/generating-a-provider.md
@@ -27,93 +27,21 @@ be quite similar for any other Terraform provider.
        ./hack/prepare.sh
         ```
 
-4. Configure your repo for the Terraform provider binary and schema:
+4. To configure the Terraform provider to generate from, update the following
+   variables in `Makefile`:
 
-    1. Update the following variables in `Makefile` for Terraform Provider:
+    ```makefile
+    export TERRAFORM_PROVIDER_SOURCE := integrations/github
+    export TERRAFORM_PROVIDER_VERSION := 4.19.2
+    export TERRAFORM_PROVIDER_DOWNLOAD_NAME := terraform-provider-github
+    export TERRAFORM_PROVIDER_DOWNLOAD_URL_PREFIX := https://releases.hashicorp.com/terraform-provider-github/4.19.2
+    ```
 
-        ```makefile
-        export TERRAFORM_PROVIDER_SOURCE := integrations/github
-        export TERRAFORM_PROVIDER_VERSION := 4.17.0
-        export TERRAFORM_PROVIDER_DOWNLOAD_NAME := terraform-provider-github
-        export TERRAFORM_PROVIDER_DOWNLOAD_URL_PREFIX := https://releases.hashicorp.com/terraform-provider-github/4.17.0
-        ```
-
-       You could find `TERRAFORM_PROVIDER_SOURCE` and `TERRAFORM_PROVIDER_VERSION` in
-       [Terraform GitHub provider] documentation by hitting the "**USE PROVIDER**"
-       button. Check [this line in controller Dockerfile] to see how these
-       variables are used to build the provider plugin binary.
-
-    2. Update import path of the Terraform provider schema package in the following
-       **two files**: `cmd/generator/main.go` and `cmd/provider/main.go`
-
-       Provider schema package is typically under the `<provider-name>`
-       directory in the GitHub repository of the Terraform provider, e.g.
-       in [`github` directory] for the GitHub provider.
-
-       ```diff
-       import (
-               ...
-
-       -       tf "github.com/hashicorp/terraform-provider-hashicups/hashicups"
-       +       tf "github.com/turkenh/terraform-provider-github/v4/github"
-
-               ...
-       )
-
-       ```
-
-       Run:
-       ```bash
-       go mod tidy
-       ```
-
-       Please note, we are temporarily using a [fork] of
-       [terraform-provider-github] repo as a workaround to [this issue].
-
-    3. If your provider uses an old version (<v2) of [terraform-plugin-sdk],
-       convert resource map to v2 schema as follows (in `cmd/generator/main.go`,
-       uncomment related section):
-
-       ```go
-       import (
-           "github.com/crossplane/terrajet/pkg/types/conversion"
-       )
+   You can find `TERRAFORM_PROVIDER_SOURCE` and `TERRAFORM_PROVIDER_VERSION` in
+   [Terraform GitHub provider] documentation by hitting the "**USE PROVIDER**"
+   button. Check [this line in controller Dockerfile] to see how these
+   variables are used to build the provider plugin binary.
 
-       func main() {
-           ...
-
-           resourceMap := conversion.GetV2ResourceMap(tf.Provider())
-           pipeline.Run(config.GetProvider(resourceMap), absRootDir)
-       }
-       ```
-
-       In a similar way in `cmd/provider/main.go`:
-
-       ```go
-       import (
-           "github.com/crossplane/terrajet/pkg/types/conversion"
-       )
-
-       func main() {
-           ...
-
-           resourceMap := conversion.GetV2ResourceMap(tf.Provider())
-           kingpin.FatalIfError(controller.Setup(mgr, log, rl, setup, ws, pconfig.GetProvider(resourceMap), 1), "Cannot setup Template controllers")
-           kingpin.FatalIfError(mgr.Start(ctrl.SetupSignalHandler()), "Cannot start controller manager")
-       }
-       ```
-
-       And in `go.mod` file, set the following `replace directive`
-       (uncomment related section):
-
-       ```
-       github.com/hashicorp/terraform-plugin-sdk => github.com/turkenh/terraform-plugin-sdk v1.17.2-patch1
-       ```
-
-       Run:
-       ```bash
-       go mod tidy
-       ```
 
 5. Implement `ProviderConfig` logic. In `provider-jet-template`, there is already
    a boilerplate code in file `internal/clients/${ProviderNameLower}.go` which
@@ -152,14 +80,14 @@ be quite similar for any other Terraform provider.
    ```
 
 6. Before generating all resources that the provider has, let's go step by step
-   and only start with generating `github_repository` and `github_branch`
-   resources.
+   and only start with generating CRDs for [github_repository] and
+   [github_branch] Terraform resources.
 
    To limit the resources to be generated, we need to provide an include list
    option with `tjconfig.WithIncludeList` in file `config/provider.go`:
 
    ```go
-   pc := tjconfig.NewProvider(resourceMap, resourcePrefix, modulePath,
+   pc := tjconfig.NewProviderWithSchema([]byte(providerSchema), resourcePrefix, modulePath,
        tjconfig.WithDefaultResourceFn(defaultResourceFn),
        tjconfig.WithIncludeList([]string{
            "github_repository$",
@@ -228,10 +156,11 @@ be quite similar for any other Terraform provider.
 
    ```diff
    import (
+       ...
+
        tjconfig "github.com/crossplane/terrajet/pkg/config"
-       "github.com/crossplane/terrajet/pkg/types/conversion"
+       "github.com/crossplane/terrajet/pkg/types/conversion/cli"
        "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
-       tf "github.com/turkenh/terraform-provider-github/v4/github"
 
    +   "github.com/crossplane-contrib/provider-jet-github/config/branch"
    +   "github.com/crossplane-contrib/provider-jet-github/config/repository"
@@ -240,7 +169,7 @@ be quite similar for any other Terraform provider.
     func GetProvider() *tjconfig.Provider {
        ...
        for _, configure := range []func(provider *tjconfig.Provider){
-           add custom config functions
+               // add custom config functions
    +           repository.Configure,
    +           branch.Configure,
        } {
@@ -410,9 +339,7 @@ Now let's test our generated resources.
 [Terraform GitHub provider]: https://registry.terraform.io/providers/integrations/github/latest/docs
 [provider-jet-template]: https://github.com/crossplane-contrib/provider-jet-template
 [Terraform documentation for provider configuration]: https://registry.terraform.io/providers/integrations/github/latest/docs#argument-reference
-[`github` directory]: https://github.com/integrations/terraform-provider-github/tree/main/github
+[github_repository]: https://registry.terraform.io/providers/integrations/github/latest/docs/resources/repository
+[github_branch]: https://registry.terraform.io/providers/integrations/github/latest/docs/resources/branch
 [this line in controller Dockerfile]: https://github.com/crossplane-contrib/provider-jet-template/blob/d9a793dd8a304f09bb2e9694c47c1bade1b6b057/cluster/images/provider-jet-template-controller/Dockerfile#L18-L25
-[fork]: https://github.com/turkenh/terraform-provider-github
-[terraform-provider-github]: https://github.com/integrations/terraform-provider-github
 [terraform-plugin-sdk]: https://github.com/hashicorp/terraform-plugin-sdk
-[this issue]: https://github.com/integrations/terraform-provider-github/pull/961
diff --git a/go.mod b/go.mod
@@ -7,13 +7,15 @@ require (
 	github.com/fatih/camelcase v1.0.0
 	github.com/golang/mock v1.6.0
 	github.com/google/go-cmp v0.5.6
+	github.com/hashicorp/terraform-json v0.13.0
 	github.com/hashicorp/terraform-plugin-sdk v1.17.2
 	github.com/hashicorp/terraform-plugin-sdk/v2 v2.7.0
 	github.com/iancoleman/strcase v0.2.0
 	github.com/json-iterator/go v1.1.11
 	github.com/muvaf/typewriter v0.0.0-20220131201631-921e94e8e8d7
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/afero v1.6.0
+	github.com/zclconf/go-cty v1.9.1
 	golang.org/x/tools v0.1.5
 	k8s.io/api v0.21.3
 	k8s.io/apimachinery v0.21.3

diff --git a/go.sum b/go.sum
@@ -373,6 +373,8 @@ github.com/hashicorp/terraform-exec v0.13.3/go.mod h1:SSg6lbUsVB3DmFyCPjBPklqf6E
 github.com/hashicorp/terraform-exec v0.14.0/go.mod h1:qrAASDq28KZiMPDnQ02sFS9udcqEkRly002EA2izXTA=
 github.com/hashicorp/terraform-json v0.10.0/go.mod h1:3defM4kkMfttwiE7VakJDwCd4R+umhSQnvJwORXbprE=
 github.com/hashicorp/terraform-json v0.12.0/go.mod h1:pmbq9o4EuL43db5+0ogX10Yofv1nozM+wskr/bGFJpI=
+github.com/hashicorp/terraform-json v0.13.0 h1:Li9L+lKD1FO5RVFRM1mMMIBDoUHslOniyEi5CM+FWGY=
+github.com/hashicorp/terraform-json v0.13.0/go.mod h1:y5OdLBCT+rxbwnpxZs9kGL7R9ExU76+cpdY8zHwoazk=
 github.com/hashicorp/terraform-plugin-go v0.3.0 h1:AJqYzP52JFYl9NABRI7smXI1pNjgR5Q/y2WyVJ/BOZA=
 github.com/hashicorp/terraform-plugin-go v0.3.0/go.mod h1:dFHsQMaTLpON2gWhVWT96fvtlc/MF1vSy3OdMhWBzdM=
 github.com/hashicorp/terraform-plugin-sdk/v2 v2.7.0 h1:SuI59MqNjYDrL7EfqHX9V6P/24isgqYx/FdglwVs9bg=
@@ -647,8 +649,9 @@ github.com/zclconf/go-cty v1.1.0/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLE
 github.com/zclconf/go-cty v1.2.0/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8=
 github.com/zclconf/go-cty v1.2.1/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8=
 github.com/zclconf/go-cty v1.8.2/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk=
-github.com/zclconf/go-cty v1.8.4 h1:pwhhz5P+Fjxse7S7UriBrMu6AUJSZM5pKqGem1PjGAs=
 github.com/zclconf/go-cty v1.8.4/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk=
+github.com/zclconf/go-cty v1.9.1 h1:viqrgQwFl5UpSxc046qblj78wZXVDFnSOufaOTER+cc=
+github.com/zclconf/go-cty v1.9.1/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk=
 github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8=
 github.com/zclconf/go-cty-yaml v1.0.2 h1:dNyg4QLTrv2IfJpm7Wtxi55ed5gLGOlPrZ6kMd51hY0=
 github.com/zclconf/go-cty-yaml v1.0.2/go.mod h1:IP3Ylp0wQpYm50IHK8OZWKMu6sPJIUgKa8XhiVHura0=

diff --git a/pkg/config/provider.go b/pkg/config/provider.go
@@ -4,8 +4,11 @@ import (
 	"fmt"
 	"regexp"
 
+	tfjson "github.com/hashicorp/terraform-json"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 	"github.com/pkg/errors"
+
+	conversiontfjson "github.com/crossplane/terrajet/pkg/types/conversion/tfjson"
 )
 
 // ResourceConfiguratorFn is a function that implements the ResourceConfigurator
@@ -143,7 +146,28 @@ func WithDefaultResourceFn(f DefaultResourceFn) ProviderOption {
 	}
 }
 
+// NewProviderWithSchema builds and returns a new Provider from provider
+// tfjson schema, that is generated using Terraform CLI with:
+// `terraform providers schema --json`
+func NewProviderWithSchema(schema []byte, prefix string, modulePath string, opts ...ProviderOption) *Provider {
+	ps := tfjson.ProviderSchemas{}
+	if err := ps.UnmarshalJSON(schema); err != nil {
+		panic(err)
+	}
+	if len(ps.Schemas) != 1 {
+		panic(fmt.Sprintf("there should exactly be 1 provider schema but there are %d", len(ps.Schemas)))
+	}
+	var rs map[string]*tfjson.Schema
+	for _, v := range ps.Schemas {
+		rs = v.ResourceSchemas
+		break
+	}
+	return NewProvider(conversiontfjson.GetV2ResourceMap(rs), prefix, modulePath, opts...)
+}
+
 // NewProvider builds and returns a new Provider.
+// Deprecated: This function will be removed soon, please use
+// NewProviderWithSchema instead.
 func NewProvider(resourceMap map[string]*schema.Resource, prefix string, modulePath string, opts ...ProviderOption) *Provider {
 	p := &Provider{
 		ModulePath:              modulePath,

diff --git a/pkg/types/conversion/conversion.go → pkg/types/conversion/sdkv1/sdkv1.go b/pkg/types/conversion/conversion.go → pkg/types/conversion/sdkv1/sdkv1.go
@@ -1,5 +1,5 @@
 /*
- Copyright 2021 The Crossplane Authors.
+ Copyright 2022 The Crossplane Authors.
 
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
  limitations under the License.
 */
 
-package conversion
+package sdkv1
 
 import (
 	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
@@ -25,6 +25,8 @@ import (
 
 // GetV2ResourceMap returns a Terraform provider SDK v2 resource map for an
 // input SDK v1 Terraform Resource Provider.
+// Deprecated: This functionality will be removed soon, please use
+// tfjson.GetV2ResourceMap instead.
 func GetV2ResourceMap(p terraform.ResourceProvider) map[string]*schemav2.Resource {
 	v1map := p.(*schema.Provider).ResourcesMap
 	v2map := make(map[string]*schemav2.Resource, len(v1map))