feat: handle terragrunt codebases (#91)

* feat: update specs to handle terragrunt * feat: add terragrunt in runner WIP * build: regenerate CRDs * chore: add logging * chore: clean naming conventions * feat: add terragrunt implementation of runner exec * test: remove useless module * fix: use linux in download url * chore: add logs for debugging * fix(terragrunt): issue with testdata module sourcing * chore: use main branch in manifests/all.yaml * chore: clean runner implementation * docs: update doc with new spec and terragrunt * fix: use url from repo instead of config * docs: reformulate * chore: remove dead code --------- Co-authored-by: Alan <alanl@padok.fr>
padok-team · Mar 14, 2023 · 9e1f38c · 9e1f38c
1 parent 4719975
commit 9e1f38c
Show file tree

Hide file tree

Showing 21 changed files with 596 additions and 136 deletions.
diff --git a/README.md b/README.md
@@ -97,7 +97,8 @@ metadata:
   name: random-pets
   namespace: burrito
 spec:
-  terraformVersion: "1.3.1"
+  terraform: 
+    version: "1.3.1"
   path: "internal/e2e/testdata/random-pets"
   branch: "main"
   repository:

diff --git a/api/v1alpha1/common.go b/api/v1alpha1/common.go
@@ -18,3 +18,40 @@ const (
 	DryRemediationStrategy       RemediationStrategy = "dry"
 	AutoApplyRemediationStrategy RemediationStrategy = "autoApply"
 )
+
+type TerraformConfig struct {
+	Version          string           `json:"version,omitempty"`
+	TerragruntConfig TerragruntConfig `json:"terragrunt,omitempty"`
+}
+
+type TerragruntConfig struct {
+	Enabled *bool  `json:"enabled,omitempty"`
+	Version string `json:"version,omitempty"`
+}
+
+func GetTerraformVersion(repository *TerraformRepository, layer *TerraformLayer) string {
+	version := repository.Spec.TerraformConfig.Version
+	if len(layer.Spec.TerraformConfig.Version) > 0 {
+		version = layer.Spec.TerraformConfig.Version
+	}
+	return version
+}
+
+func GetTerragruntVersion(repository *TerraformRepository, layer *TerraformLayer) string {
+	version := repository.Spec.TerraformConfig.TerragruntConfig.Version
+	if len(layer.Spec.TerraformConfig.TerragruntConfig.Version) > 0 {
+		version = layer.Spec.TerraformConfig.TerragruntConfig.Version
+	}
+	return version
+}
+
+func GetTerragruntEnabled(repository *TerraformRepository, layer *TerraformLayer) bool {
+	enabled := false
+	if repository.Spec.TerraformConfig.TerragruntConfig.Enabled != nil {
+		enabled = *repository.Spec.TerraformConfig.TerragruntConfig.Enabled
+	}
+	if layer.Spec.TerraformConfig.TerragruntConfig.Enabled != nil {
+		enabled = *layer.Spec.TerraformConfig.TerragruntConfig.Enabled
+	}
+	return enabled
+}
diff --git a/api/v1alpha1/terraformlayer_types.go b/api/v1alpha1/terraformlayer_types.go
@@ -30,7 +30,7 @@ type TerraformLayerSpec struct {
 
 	Path                string                   `json:"path,omitempty"`
 	Branch              string                   `json:"branch,omitempty"`
-	TerraformVersion    string                   `json:"terraformVersion,omitempty"`
+	TerraformConfig     TerraformConfig          `json:"terraform,omitempty"`
 	Repository          TerraformLayerRepository `json:"repository,omitempty"`
 	RemediationStrategy RemediationStrategy      `json:"remediationStrategy,omitempty"`
 	PlanOnPullRequest   bool                     `json:"planOnPullRequest,omitempty"`

diff --git a/api/v1alpha1/terraformrepository_types.go b/api/v1alpha1/terraformrepository_types.go
@@ -29,6 +29,7 @@ type TerraformRepositorySpec struct {
 	// Important: Run "make" to regenerate code after modifying this file
 
 	Repository          TerraformRepositoryRepository `json:"repository,omitempty"`
+	TerraformConfig     TerraformConfig               `json:"terraform,omitempty"`
 	RemediationStrategy RemediationStrategy           `json:"remediationStrategy,omitempty"`
 	OverrideRunnerSpec  OverrideRunnerSpec            `json:"overrideRunnerSpec,omitempty"`
 }

diff --git a/cmd/controllers/start.go b/cmd/controllers/start.go
@@ -30,7 +30,7 @@ func buildControllersStartCmd(app *burrito.App) *cobra.Command {
 	cmd.Flags().StringSliceVar(&app.Config.Controller.Types, "types", []string{"layer", "repository"}, "list of controllers to start")
 
 	cmd.Flags().DurationVar(&app.Config.Controller.Timers.DriftDetection, "drift-detection-period", defaultDriftDetectionTimer, "period between two plans. Must end with s, m or h.")
-	cmd.Flags().DurationVar(&app.Config.Controller.Timers.OnError, "on-error-period", defaultOnErrorTimer, "period between two runners launch when an error occured. Must end with s, m or h.")
+	cmd.Flags().DurationVar(&app.Config.Controller.Timers.OnError, "on-error-period", defaultOnErrorTimer, "period between two runners launch when an error occurred. Must end with s, m or h.")
 	cmd.Flags().DurationVar(&app.Config.Controller.Timers.WaitAction, "wait-action-period", defaultWaitActionTimer, "period between two runners when a layer is locked. Must end with s, m or h.")
 	cmd.Flags().BoolVar(&app.Config.Controller.LeaderElection.Enabled, "leader-election", true, "whether leader election is enabled or not, default to true")
 	cmd.Flags().StringVar(&app.Config.Controller.LeaderElection.ID, "leader-election-id", "6d185457.terraform.padok.cloud", "lease id used for leader election")

diff --git a/config/crd/bases/config.terraform.padok.cloud_terraformlayers.yaml b/config/crd/bases/config.terraform.padok.cloud_terraformlayers.yaml
@@ -132,8 +132,18 @@ spec:
                   namespace:
                     type: string
                 type: object
-              terraformVersion:
-                type: string
+              terraform:
+                properties:
+                  terragrunt:
+                    properties:
+                      enabled:
+                        type: boolean
+                      version:
+                        type: string
+                    type: object
+                  version:
+                    type: string
+                type: object
             type: object
           status:
             description: TerraformLayerStatus defines the observed state of TerraformLayer

diff --git a/config/crd/bases/config.terraform.padok.cloud_terraformrepositories.yaml b/config/crd/bases/config.terraform.padok.cloud_terraformrepositories.yaml
@@ -113,6 +113,18 @@ spec:
                   url:
                     type: string
                 type: object
+              terraform:
+                properties:
+                  terragrunt:
+                    properties:
+                      enabled:
+                        type: boolean
+                      version:
+                        type: string
+                    type: object
+                  version:
+                    type: string
+                type: object
             type: object
           status:
             description: TerraformRepositoryStatus defines the observed state of TerraformRepository

diff --git a/docs/contents/usage/README.md b/docs/contents/usage/README.md
@@ -3,6 +3,8 @@
 - [User guide](#user-guide)
   - [Override the runner pod spec](#override-the-runner-pod-spec)
   - [Choose your remediation strategy](#choose-your-remediation-strategy)
+  - [Choose your terraform version](#choose-your-terraform-version)
+  - [Use Terragrunt](#use-terragrunt)
 - [Operator guide](#operator-guide)
   - [Setup a git webhook](#setup-a-git-webhook)
   - [Configuration](#configuration)
@@ -48,7 +50,8 @@ metadata:
   name: random-pets
   namespace: burrito
 spec:
-  terraformVersion: "1.3.1"
+  terraform:
+    version: "1.3.1"
   path: "internal/e2e/testdata/random-pets"
   branch: "main"
   repository:
@@ -85,7 +88,8 @@ metadata:
   name: random-pets
   namespace: burrito
 spec:
-  terraformVersion: "1.3.1"
+  terraform:
+    version: "1.3.1"
   path: "internal/e2e/testdata/random-pets"
   branch: "main"
   repository:
@@ -112,6 +116,40 @@ The configuration of the `TerraformLayer` will take precedence.
 
 > :warning: This operator is still experimental. Use `spec.remediationStrategy: "autoApply"` at your own risk.
 
+### Choose your terraform version
+
+Both `TerraformRepository` and `TerraformLayer` expose a `spec.terrafrom.version` map field.
+
+If the field is specified for a given `TerraformRepository` it will be applied by default to all `TerraformLayer` linked to it.
+
+If the field is specified for a given `TerraformLayer` it will take precedence over the `TerraformRepository` configuration.
+
+### Use Terragrunt
+
+You can specify usage of terragrunt as follow:
+
+```yaml
+apiVersion: config.terraform.padok.cloud/v1alpha1
+kind: TerraformLayer
+metadata:
+  name: random-pets-terragrunt
+spec:
+  terraform:
+    version: "1.3.1"
+    terragrunt:
+      enabled: true
+      version: "0.44.5"
+  remediationStrategy: dry
+  path: "internal/e2e/testdata/terragrunt/random-pets/prod"
+  branch: "feat/handle-terragrunt"
+  repository:
+    kind: TerraformRepository
+    name: burrito
+    namespace: burrito
+```
+
+> This configuration can be specified at the `TerraformRepository` level to be enabled by default in each of its layers.
+
 ## Operator guide
 
 ### Setup a git webhook
@@ -151,7 +189,7 @@ You can configure `burrito` with environment variables.
 | :-----------------------------------------: | :--------------------------------------------------------------------: | :------------------------------: |
 |         `BURRITO_CONTROLLER_TYPES`          |                      list of controllers to start                      |        `layer,repository`        |
 | `BURRITO_CONTROLLER_TIMERS_DRIFTDETECTION`  |              period between two plans for drift detection              |              `20m`               |
-|     `BURRITO_CONTROLLER_TIMERS_ONERROR`     |        period between two runners launch when an error occured         |               `1m`               |
+|     `BURRITO_CONTROLLER_TIMERS_ONERROR`     |        period between two runners launch when an error occurred         |               `1m`               |
 |   `BURRITO_CONTROLLER_TIMERS_WAITACTION`    |        period between two runners launch when a layer is locked        |               `1m`               |
 | `BURRITO_CONTROLLER_LEADERELECTION_ENABLED` |               whether leader election is enabled or not                |              `true`              |
 |   `BURRITO_CONTROLLER_LEADERELECTION_ID`    |                   lease id used for leader election                    | `6d185457.terraform.padok.cloud` |

diff --git a/internal/burrito/config/config.go b/internal/burrito/config/config.go
@@ -54,22 +54,23 @@ type ControllerTimers struct {
 }
 
 type RepositoryConfig struct {
-	URL           string `yaml:"url"`
 	SSHPrivateKey string `yaml:"sshPrivateKey"`
 	Username      string `yaml:"username"`
 	Password      string `yaml:"password"`
 }
 
 type RunnerConfig struct {
-	Path                       string           `yaml:"path"`
-	Branch                     string           `yaml:"branch"`
-	Version                    string           `yaml:"version"`
 	Action                     string           `yaml:"action"`
-	Repository                 RepositoryConfig `yaml:"repository"`
 	Layer                      Layer            `yaml:"layer"`
+	Repository                 RepositoryConfig `yaml:"repository"`
 	SSHKnownHostsConfigMapName string           `yaml:"sshKnowHostsConfigMapName"`
 }
 
+type TerragruntConfig struct {
+	Enabled bool   `yaml:"enabled"`
+	Version string `yaml:"version"`
+}
+
 type Layer struct {
 	Name      string `yaml:"name"`
 	Namespace string `yaml:"namespace"`

diff --git a/internal/controllers/terraformlayer/pod.go b/internal/controllers/terraformlayer/pod.go
@@ -129,22 +129,6 @@ func defaultPodSpec(config *config.Config, layer *configv1alpha1.TerraformLayer,
 						Name:  "BURRITO_REDIS_DATABASE",
 						Value: fmt.Sprintf("%d", config.Redis.Database),
 					},
-					{
-						Name:  "BURRITO_RUNNER_REPOSITORY_URL",
-						Value: repository.Spec.Repository.Url,
-					},
-					{
-						Name:  "BURRITO_RUNNER_PATH",
-						Value: layer.Spec.Path,
-					},
-					{
-						Name:  "BURRITO_RUNNER_BRANCH",
-						Value: layer.Spec.Branch,
-					},
-					{
-						Name:  "BURRITO_RUNNER_VERSION",
-						Value: layer.Spec.TerraformVersion,
-					},
 					{
 						Name:  "BURRITO_RUNNER_LAYER_NAME",
 						Value: layer.GetObjectMeta().GetName(),

diff --git a/internal/e2e/testdata/random-pets/main.tf → ...2e/testdata/terraform/random-pets/main.tf b/internal/e2e/testdata/random-pets/main.tf → ...2e/testdata/terraform/random-pets/main.tf
diff --git a/internal/e2e/testdata/terragrunt/modules/random-pets/main.tf b/internal/e2e/testdata/terragrunt/modules/random-pets/main.tf
@@ -0,0 +1,11 @@
+resource "random_pet" "first" {
+  length = 1
+}
+
+resource "random_pet" "second" {
+  length = 2
+}
+
+resource "random_pet" "third" {
+  length = 3
+}
diff --git a/internal/e2e/testdata/terragrunt/random-pets/module.hcl b/internal/e2e/testdata/terragrunt/random-pets/module.hcl
@@ -0,0 +1,3 @@
+terraform {
+  source = "../../modules//random-pets"
+}
diff --git a/internal/e2e/testdata/terragrunt/random-pets/prod/inputs.hcl b/internal/e2e/testdata/terragrunt/random-pets/prod/inputs.hcl
@@ -0,0 +1 @@
+inputs = {}
diff --git a/internal/e2e/testdata/terragrunt/random-pets/prod/terragrunt.hcl b/internal/e2e/testdata/terragrunt/random-pets/prod/terragrunt.hcl
@@ -0,0 +1,14 @@
+include "root" {
+  path           = find_in_parent_folders()
+  merge_strategy = "deep"
+}
+
+include "module" {
+  path           = find_in_parent_folders("module.hcl")
+  merge_strategy = "deep"
+}
+
+include "inputs" {
+  path           = "inputs.hcl"
+  merge_strategy = "deep"
+}
diff --git a/internal/e2e/testdata/terragrunt/terragrunt.hcl b/internal/e2e/testdata/terragrunt/terragrunt.hcl
diff --git a/internal/runner/clone.go b/internal/runner/clone.go
@@ -0,0 +1,57 @@
+package runner
+
+import (
+	"strings"
+
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/transport/http"
+	"github.com/go-git/go-git/v5/plumbing/transport/ssh"
+	"github.com/padok-team/burrito/internal/burrito/config"
+	log "github.com/sirupsen/logrus"
+)
+
+func clone(repository config.RepositoryConfig, URL, branch, path string) (*git.Repository, error) {
+	cloneOptions, err := getCloneOptions(repository, URL, branch, path)
+	if err != nil {
+		return &git.Repository{}, err
+	}
+	return git.PlainClone(WorkingDir, false, cloneOptions)
+}
+
+func getCloneOptions(repository config.RepositoryConfig, URL, branch, path string) (*git.CloneOptions, error) {
+	authMethod := "ssh"
+	cloneOptions := &git.CloneOptions{
+		ReferenceName: plumbing.NewBranchReferenceName(branch),
+		URL:           URL,
+	}
+	if strings.Contains(URL, "https://") {
+		authMethod = "https"
+	}
+	log.Infof("clone method is %s", authMethod)
+	switch authMethod {
+	case "ssh":
+		if repository.SSHPrivateKey == "" {
+			log.Infof("detected keyless authentication")
+			return cloneOptions, nil
+		}
+		log.Infof("private key found")
+		publicKeys, err := ssh.NewPublicKeys("git", []byte(repository.SSHPrivateKey), "")
+		if err != nil {
+			return cloneOptions, err
+		}
+		cloneOptions.Auth = publicKeys
+
+	case "https":
+		if repository.Username != "" && repository.Password != "" {
+			log.Infof("username and password found")
+			cloneOptions.Auth = &http.BasicAuth{
+				Username: repository.Username,
+				Password: repository.Password,
+			}
+		} else {
+			log.Infof("passwordless authentication detected")
+		}
+	}
+	return cloneOptions, nil
+}