From 9e1f38cf2a02458ad0eb86aebcc8e58ac4c0fee6 Mon Sep 17 00:00:00 2001 From: spoukke <32708678+spoukke@users.noreply.github.com> Date: Tue, 14 Mar 2023 12:35:21 +0100 Subject: [PATCH] feat: handle terragrunt codebases (#91) * feat: update specs to handle terragrunt * feat: add terragrunt in runner WIP * build: regenerate CRDs * chore: add logging * chore: clean naming conventions * feat: add terragrunt implementation of runner exec * test: remove useless module * fix: use linux in download url * chore: add logs for debugging * fix(terragrunt): issue with testdata module sourcing * chore: use main branch in manifests/all.yaml * chore: clean runner implementation * docs: update doc with new spec and terragrunt * fix: use url from repo instead of config * docs: reformulate * chore: remove dead code --------- Co-authored-by: Alan --- README.md | 3 +- api/v1alpha1/common.go | 37 +++ api/v1alpha1/terraformlayer_types.go | 2 +- api/v1alpha1/terraformrepository_types.go | 1 + cmd/controllers/start.go | 2 +- ...terraform.padok.cloud_terraformlayers.yaml | 14 +- ...orm.padok.cloud_terraformrepositories.yaml | 12 + docs/contents/usage/README.md | 44 +++- internal/burrito/config/config.go | 11 +- internal/controllers/terraformlayer/pod.go | 16 -- .../{ => terraform}/random-pets/main.tf | 0 .../terragrunt/modules/random-pets/main.tf | 11 + .../terragrunt/random-pets/module.hcl | 3 + .../terragrunt/random-pets/prod/inputs.hcl | 1 + .../random-pets/prod/terragrunt.hcl | 14 ++ .../e2e/testdata/terragrunt/terragrunt.hcl | 0 internal/runner/clone.go | 57 +++++ internal/runner/runner.go | 213 ++++++++++-------- internal/runner/terraform/terraform.go | 98 ++++++++ internal/runner/terragrunt/terragrunt.go | 136 +++++++++++ manifests/install.yaml | 57 ++++- 21 files changed, 596 insertions(+), 136 deletions(-) rename internal/e2e/testdata/{ => terraform}/random-pets/main.tf (100%) create mode 100644 internal/e2e/testdata/terragrunt/modules/random-pets/main.tf create mode 100644 internal/e2e/testdata/terragrunt/random-pets/module.hcl create mode 100644 internal/e2e/testdata/terragrunt/random-pets/prod/inputs.hcl create mode 100644 internal/e2e/testdata/terragrunt/random-pets/prod/terragrunt.hcl create mode 100644 internal/e2e/testdata/terragrunt/terragrunt.hcl create mode 100644 internal/runner/clone.go create mode 100644 internal/runner/terraform/terraform.go create mode 100644 internal/runner/terragrunt/terragrunt.go diff --git a/README.md b/README.md index e3c8955d..a3202fde 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,8 @@ metadata: name: random-pets namespace: burrito spec: - terraformVersion: "1.3.1" + terraform: + version: "1.3.1" path: "internal/e2e/testdata/random-pets" branch: "main" repository: diff --git a/api/v1alpha1/common.go b/api/v1alpha1/common.go index bee80707..032d7105 100644 --- a/api/v1alpha1/common.go +++ b/api/v1alpha1/common.go @@ -18,3 +18,40 @@ const ( DryRemediationStrategy RemediationStrategy = "dry" AutoApplyRemediationStrategy RemediationStrategy = "autoApply" ) + +type TerraformConfig struct { + Version string `json:"version,omitempty"` + TerragruntConfig TerragruntConfig `json:"terragrunt,omitempty"` +} + +type TerragruntConfig struct { + Enabled *bool `json:"enabled,omitempty"` + Version string `json:"version,omitempty"` +} + +func GetTerraformVersion(repository *TerraformRepository, layer *TerraformLayer) string { + version := repository.Spec.TerraformConfig.Version + if len(layer.Spec.TerraformConfig.Version) > 0 { + version = layer.Spec.TerraformConfig.Version + } + return version +} + +func GetTerragruntVersion(repository *TerraformRepository, layer *TerraformLayer) string { + version := repository.Spec.TerraformConfig.TerragruntConfig.Version + if len(layer.Spec.TerraformConfig.TerragruntConfig.Version) > 0 { + version = layer.Spec.TerraformConfig.TerragruntConfig.Version + } + return version +} + +func GetTerragruntEnabled(repository *TerraformRepository, layer *TerraformLayer) bool { + enabled := false + if repository.Spec.TerraformConfig.TerragruntConfig.Enabled != nil { + enabled = *repository.Spec.TerraformConfig.TerragruntConfig.Enabled + } + if layer.Spec.TerraformConfig.TerragruntConfig.Enabled != nil { + enabled = *layer.Spec.TerraformConfig.TerragruntConfig.Enabled + } + return enabled +} diff --git a/api/v1alpha1/terraformlayer_types.go b/api/v1alpha1/terraformlayer_types.go index 4b010a37..21290d02 100644 --- a/api/v1alpha1/terraformlayer_types.go +++ b/api/v1alpha1/terraformlayer_types.go @@ -30,7 +30,7 @@ type TerraformLayerSpec struct { Path string `json:"path,omitempty"` Branch string `json:"branch,omitempty"` - TerraformVersion string `json:"terraformVersion,omitempty"` + TerraformConfig TerraformConfig `json:"terraform,omitempty"` Repository TerraformLayerRepository `json:"repository,omitempty"` RemediationStrategy RemediationStrategy `json:"remediationStrategy,omitempty"` PlanOnPullRequest bool `json:"planOnPullRequest,omitempty"` diff --git a/api/v1alpha1/terraformrepository_types.go b/api/v1alpha1/terraformrepository_types.go index 58f6bf1c..89ae2d70 100644 --- a/api/v1alpha1/terraformrepository_types.go +++ b/api/v1alpha1/terraformrepository_types.go @@ -29,6 +29,7 @@ type TerraformRepositorySpec struct { // Important: Run "make" to regenerate code after modifying this file Repository TerraformRepositoryRepository `json:"repository,omitempty"` + TerraformConfig TerraformConfig `json:"terraform,omitempty"` RemediationStrategy RemediationStrategy `json:"remediationStrategy,omitempty"` OverrideRunnerSpec OverrideRunnerSpec `json:"overrideRunnerSpec,omitempty"` } diff --git a/cmd/controllers/start.go b/cmd/controllers/start.go index fe3bee0b..846eb56c 100644 --- a/cmd/controllers/start.go +++ b/cmd/controllers/start.go @@ -30,7 +30,7 @@ func buildControllersStartCmd(app *burrito.App) *cobra.Command { cmd.Flags().StringSliceVar(&app.Config.Controller.Types, "types", []string{"layer", "repository"}, "list of controllers to start") cmd.Flags().DurationVar(&app.Config.Controller.Timers.DriftDetection, "drift-detection-period", defaultDriftDetectionTimer, "period between two plans. Must end with s, m or h.") - cmd.Flags().DurationVar(&app.Config.Controller.Timers.OnError, "on-error-period", defaultOnErrorTimer, "period between two runners launch when an error occured. Must end with s, m or h.") + cmd.Flags().DurationVar(&app.Config.Controller.Timers.OnError, "on-error-period", defaultOnErrorTimer, "period between two runners launch when an error occurred. Must end with s, m or h.") cmd.Flags().DurationVar(&app.Config.Controller.Timers.WaitAction, "wait-action-period", defaultWaitActionTimer, "period between two runners when a layer is locked. Must end with s, m or h.") cmd.Flags().BoolVar(&app.Config.Controller.LeaderElection.Enabled, "leader-election", true, "whether leader election is enabled or not, default to true") cmd.Flags().StringVar(&app.Config.Controller.LeaderElection.ID, "leader-election-id", "6d185457.terraform.padok.cloud", "lease id used for leader election") diff --git a/config/crd/bases/config.terraform.padok.cloud_terraformlayers.yaml b/config/crd/bases/config.terraform.padok.cloud_terraformlayers.yaml index 3078d0d0..19e67ffa 100644 --- a/config/crd/bases/config.terraform.padok.cloud_terraformlayers.yaml +++ b/config/crd/bases/config.terraform.padok.cloud_terraformlayers.yaml @@ -132,8 +132,18 @@ spec: namespace: type: string type: object - terraformVersion: - type: string + terraform: + properties: + terragrunt: + properties: + enabled: + type: boolean + version: + type: string + type: object + version: + type: string + type: object type: object status: description: TerraformLayerStatus defines the observed state of TerraformLayer diff --git a/config/crd/bases/config.terraform.padok.cloud_terraformrepositories.yaml b/config/crd/bases/config.terraform.padok.cloud_terraformrepositories.yaml index 63aa5240..fb12ab43 100644 --- a/config/crd/bases/config.terraform.padok.cloud_terraformrepositories.yaml +++ b/config/crd/bases/config.terraform.padok.cloud_terraformrepositories.yaml @@ -113,6 +113,18 @@ spec: url: type: string type: object + terraform: + properties: + terragrunt: + properties: + enabled: + type: boolean + version: + type: string + type: object + version: + type: string + type: object type: object status: description: TerraformRepositoryStatus defines the observed state of TerraformRepository diff --git a/docs/contents/usage/README.md b/docs/contents/usage/README.md index 90b04014..b73e466c 100644 --- a/docs/contents/usage/README.md +++ b/docs/contents/usage/README.md @@ -3,6 +3,8 @@ - [User guide](#user-guide) - [Override the runner pod spec](#override-the-runner-pod-spec) - [Choose your remediation strategy](#choose-your-remediation-strategy) + - [Choose your terraform version](#choose-your-terraform-version) + - [Use Terragrunt](#use-terragrunt) - [Operator guide](#operator-guide) - [Setup a git webhook](#setup-a-git-webhook) - [Configuration](#configuration) @@ -48,7 +50,8 @@ metadata: name: random-pets namespace: burrito spec: - terraformVersion: "1.3.1" + terraform: + version: "1.3.1" path: "internal/e2e/testdata/random-pets" branch: "main" repository: @@ -85,7 +88,8 @@ metadata: name: random-pets namespace: burrito spec: - terraformVersion: "1.3.1" + terraform: + version: "1.3.1" path: "internal/e2e/testdata/random-pets" branch: "main" repository: @@ -112,6 +116,40 @@ The configuration of the `TerraformLayer` will take precedence. > :warning: This operator is still experimental. Use `spec.remediationStrategy: "autoApply"` at your own risk. +### Choose your terraform version + +Both `TerraformRepository` and `TerraformLayer` expose a `spec.terrafrom.version` map field. + +If the field is specified for a given `TerraformRepository` it will be applied by default to all `TerraformLayer` linked to it. + +If the field is specified for a given `TerraformLayer` it will take precedence over the `TerraformRepository` configuration. + +### Use Terragrunt + +You can specify usage of terragrunt as follow: + +```yaml +apiVersion: config.terraform.padok.cloud/v1alpha1 +kind: TerraformLayer +metadata: + name: random-pets-terragrunt +spec: + terraform: + version: "1.3.1" + terragrunt: + enabled: true + version: "0.44.5" + remediationStrategy: dry + path: "internal/e2e/testdata/terragrunt/random-pets/prod" + branch: "feat/handle-terragrunt" + repository: + kind: TerraformRepository + name: burrito + namespace: burrito +``` + +> This configuration can be specified at the `TerraformRepository` level to be enabled by default in each of its layers. + ## Operator guide ### Setup a git webhook @@ -151,7 +189,7 @@ You can configure `burrito` with environment variables. | :-----------------------------------------: | :--------------------------------------------------------------------: | :------------------------------: | | `BURRITO_CONTROLLER_TYPES` | list of controllers to start | `layer,repository` | | `BURRITO_CONTROLLER_TIMERS_DRIFTDETECTION` | period between two plans for drift detection | `20m` | -| `BURRITO_CONTROLLER_TIMERS_ONERROR` | period between two runners launch when an error occured | `1m` | +| `BURRITO_CONTROLLER_TIMERS_ONERROR` | period between two runners launch when an error occurred | `1m` | | `BURRITO_CONTROLLER_TIMERS_WAITACTION` | period between two runners launch when a layer is locked | `1m` | | `BURRITO_CONTROLLER_LEADERELECTION_ENABLED` | whether leader election is enabled or not | `true` | | `BURRITO_CONTROLLER_LEADERELECTION_ID` | lease id used for leader election | `6d185457.terraform.padok.cloud` | diff --git a/internal/burrito/config/config.go b/internal/burrito/config/config.go index 32ad7d97..d715b1bc 100644 --- a/internal/burrito/config/config.go +++ b/internal/burrito/config/config.go @@ -54,22 +54,23 @@ type ControllerTimers struct { } type RepositoryConfig struct { - URL string `yaml:"url"` SSHPrivateKey string `yaml:"sshPrivateKey"` Username string `yaml:"username"` Password string `yaml:"password"` } type RunnerConfig struct { - Path string `yaml:"path"` - Branch string `yaml:"branch"` - Version string `yaml:"version"` Action string `yaml:"action"` - Repository RepositoryConfig `yaml:"repository"` Layer Layer `yaml:"layer"` + Repository RepositoryConfig `yaml:"repository"` SSHKnownHostsConfigMapName string `yaml:"sshKnowHostsConfigMapName"` } +type TerragruntConfig struct { + Enabled bool `yaml:"enabled"` + Version string `yaml:"version"` +} + type Layer struct { Name string `yaml:"name"` Namespace string `yaml:"namespace"` diff --git a/internal/controllers/terraformlayer/pod.go b/internal/controllers/terraformlayer/pod.go index 776b781b..395877a3 100644 --- a/internal/controllers/terraformlayer/pod.go +++ b/internal/controllers/terraformlayer/pod.go @@ -129,22 +129,6 @@ func defaultPodSpec(config *config.Config, layer *configv1alpha1.TerraformLayer, Name: "BURRITO_REDIS_DATABASE", Value: fmt.Sprintf("%d", config.Redis.Database), }, - { - Name: "BURRITO_RUNNER_REPOSITORY_URL", - Value: repository.Spec.Repository.Url, - }, - { - Name: "BURRITO_RUNNER_PATH", - Value: layer.Spec.Path, - }, - { - Name: "BURRITO_RUNNER_BRANCH", - Value: layer.Spec.Branch, - }, - { - Name: "BURRITO_RUNNER_VERSION", - Value: layer.Spec.TerraformVersion, - }, { Name: "BURRITO_RUNNER_LAYER_NAME", Value: layer.GetObjectMeta().GetName(), diff --git a/internal/e2e/testdata/random-pets/main.tf b/internal/e2e/testdata/terraform/random-pets/main.tf similarity index 100% rename from internal/e2e/testdata/random-pets/main.tf rename to internal/e2e/testdata/terraform/random-pets/main.tf diff --git a/internal/e2e/testdata/terragrunt/modules/random-pets/main.tf b/internal/e2e/testdata/terragrunt/modules/random-pets/main.tf new file mode 100644 index 00000000..f832fe76 --- /dev/null +++ b/internal/e2e/testdata/terragrunt/modules/random-pets/main.tf @@ -0,0 +1,11 @@ +resource "random_pet" "first" { + length = 1 +} + +resource "random_pet" "second" { + length = 2 +} + +resource "random_pet" "third" { + length = 3 +} diff --git a/internal/e2e/testdata/terragrunt/random-pets/module.hcl b/internal/e2e/testdata/terragrunt/random-pets/module.hcl new file mode 100644 index 00000000..32d03711 --- /dev/null +++ b/internal/e2e/testdata/terragrunt/random-pets/module.hcl @@ -0,0 +1,3 @@ +terraform { + source = "../../modules//random-pets" +} diff --git a/internal/e2e/testdata/terragrunt/random-pets/prod/inputs.hcl b/internal/e2e/testdata/terragrunt/random-pets/prod/inputs.hcl new file mode 100644 index 00000000..7f7867c7 --- /dev/null +++ b/internal/e2e/testdata/terragrunt/random-pets/prod/inputs.hcl @@ -0,0 +1 @@ +inputs = {} \ No newline at end of file diff --git a/internal/e2e/testdata/terragrunt/random-pets/prod/terragrunt.hcl b/internal/e2e/testdata/terragrunt/random-pets/prod/terragrunt.hcl new file mode 100644 index 00000000..11fc6f66 --- /dev/null +++ b/internal/e2e/testdata/terragrunt/random-pets/prod/terragrunt.hcl @@ -0,0 +1,14 @@ +include "root" { + path = find_in_parent_folders() + merge_strategy = "deep" +} + +include "module" { + path = find_in_parent_folders("module.hcl") + merge_strategy = "deep" +} + +include "inputs" { + path = "inputs.hcl" + merge_strategy = "deep" +} diff --git a/internal/e2e/testdata/terragrunt/terragrunt.hcl b/internal/e2e/testdata/terragrunt/terragrunt.hcl new file mode 100644 index 00000000..e69de29b diff --git a/internal/runner/clone.go b/internal/runner/clone.go new file mode 100644 index 00000000..6bff808d --- /dev/null +++ b/internal/runner/clone.go @@ -0,0 +1,57 @@ +package runner + +import ( + "strings" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/transport/http" + "github.com/go-git/go-git/v5/plumbing/transport/ssh" + "github.com/padok-team/burrito/internal/burrito/config" + log "github.com/sirupsen/logrus" +) + +func clone(repository config.RepositoryConfig, URL, branch, path string) (*git.Repository, error) { + cloneOptions, err := getCloneOptions(repository, URL, branch, path) + if err != nil { + return &git.Repository{}, err + } + return git.PlainClone(WorkingDir, false, cloneOptions) +} + +func getCloneOptions(repository config.RepositoryConfig, URL, branch, path string) (*git.CloneOptions, error) { + authMethod := "ssh" + cloneOptions := &git.CloneOptions{ + ReferenceName: plumbing.NewBranchReferenceName(branch), + URL: URL, + } + if strings.Contains(URL, "https://") { + authMethod = "https" + } + log.Infof("clone method is %s", authMethod) + switch authMethod { + case "ssh": + if repository.SSHPrivateKey == "" { + log.Infof("detected keyless authentication") + return cloneOptions, nil + } + log.Infof("private key found") + publicKeys, err := ssh.NewPublicKeys("git", []byte(repository.SSHPrivateKey), "") + if err != nil { + return cloneOptions, err + } + cloneOptions.Auth = publicKeys + + case "https": + if repository.Username != "" && repository.Password != "" { + log.Infof("username and password found") + cloneOptions.Auth = &http.BasicAuth{ + Username: repository.Username, + Password: repository.Password, + } + } else { + log.Infof("passwordless authentication detected") + } + } + return cloneOptions, nil +} diff --git a/internal/runner/runner.go b/internal/runner/runner.go index e1bb0354..f9e4e0a9 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -5,10 +5,8 @@ import ( "crypto/sha256" "errors" "fmt" - "io" "os" "strconv" - "strings" "time" log "github.com/sirupsen/logrus" @@ -17,13 +15,6 @@ import ( "encoding/json" "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/transport/http" - "github.com/go-git/go-git/v5/plumbing/transport/ssh" - "github.com/hashicorp/go-version" - "github.com/hashicorp/hc-install/product" - "github.com/hashicorp/hc-install/releases" - "github.com/hashicorp/terraform-exec/tfexec" tfjson "github.com/hashicorp/terraform-json" configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1" "github.com/padok-team/burrito/internal/annotations" @@ -38,18 +29,30 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/padok-team/burrito/internal/runner/terraform" + "github.com/padok-team/burrito/internal/runner/terragrunt" ) -const PlanArtifact string = "plan.out" +const PlanArtifact string = "/tmp/plan.out" const WorkingDir string = "/repository" type Runner struct { - config *config.Config - terraform *tfexec.Terraform - storage storage.Storage - client client.Client - layer *configv1alpha1.TerraformLayer - repository *git.Repository + config *config.Config + exec TerraformExec + storage storage.Storage + client client.Client + layer *configv1alpha1.TerraformLayer + repository *configv1alpha1.TerraformRepository + gitRepository *git.Repository +} + +type TerraformExec interface { + Install() error + Init(string) error + Plan() error + Apply() error + Show() ([]byte, error) } func New(c *config.Config) *Runner { @@ -71,8 +74,9 @@ func (r *Runner) Exec() { if err != nil { log.Errorf("error initializing runner: %s", err) } - ref, _ := r.repository.Head() + ref, _ := r.gitRepository.Head() commit := ref.Hash().String() + switch r.config.Runner.Action { case "plan": sum, err = r.plan() @@ -90,7 +94,7 @@ func (r *Runner) Exec() { ann[annotations.LastApplySum] = sum } default: - err = errors.New("Unrecognized runner action, If this is happening there might be a version mismatch between the controller and runner") + err = errors.New("unrecognized runner action, If this is happening there might be a version mismatch between the controller and runner") } if err != nil { log.Errorf("error during runner execution: %s", err) @@ -108,20 +112,9 @@ func (r *Runner) Exec() { } } -func (r *Runner) init() error { - r.storage = redis.New(r.config.Redis.URL, r.config.Redis.Password, r.config.Redis.Database) - scheme := runtime.NewScheme() - utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - utilruntime.Must(configv1alpha1.AddToScheme(scheme)) - cl, err := client.New(ctrl.GetConfigOrDie(), client.Options{ - Scheme: scheme, - }) - if err != nil { - return err - } - r.client = cl +func (r *Runner) getLayerAndRepository() error { layer := &configv1alpha1.TerraformLayer{} - err = r.client.Get(context.TODO(), types.NamespacedName{ + err := r.client.Get(context.TODO(), types.NamespacedName{ Namespace: r.config.Runner.Layer.Namespace, Name: r.config.Runner.Layer.Name, }, layer) @@ -129,38 +122,89 @@ func (r *Runner) init() error { return err } r.layer = layer - log.Infof("initializing runner with Terraform version: %s", r.config.Runner.Version) - terraformVersion, err := version.NewVersion(r.config.Runner.Version) + repository := &configv1alpha1.TerraformRepository{} + err = r.client.Get(context.TODO(), types.NamespacedName{ + Namespace: layer.Spec.Repository.Namespace, + Name: layer.Spec.Repository.Name, + }, repository) if err != nil { return err } - installer := &releases.ExactVersion{ - Product: product.Terraform, - Version: version.Must(terraformVersion, nil), + r.repository = repository + return nil +} + +func newK8SClient() (client.Client, error) { + scheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(configv1alpha1.AddToScheme(scheme)) + cl, err := client.New(ctrl.GetConfigOrDie(), client.Options{ + Scheme: scheme, + }) + if err != nil { + return nil, err + } + return cl, err +} + +func (r *Runner) install() error { + terraformVersion := configv1alpha1.GetTerraformVersion(r.repository, r.layer) + terraformExec := terraform.NewTerraform(terraformVersion, PlanArtifact) + terraformRuntime := "terraform" + if configv1alpha1.GetTerragruntEnabled(r.repository, r.layer) { + terraformRuntime = "terragrunt" } - execPath, err := installer.Install(context.Background()) + switch terraformRuntime { + case "terraform": + log.Infof("using terraform") + r.exec = terraformExec + case "terragrunt": + log.Infof("using terragrunt") + r.exec = terragrunt.NewTerragrunt(terraformExec, configv1alpha1.GetTerragruntVersion(r.repository, r.layer), PlanArtifact) + } + err := r.exec.Install() if err != nil { return err } - log.Infof("cloning repository %s %s branch", r.config.Runner.Repository.URL, r.config.Runner.Branch) - cloneOptions, err := r.getCloneOptions() + return nil +} + +func (r *Runner) init() error { + r.storage = redis.New(r.config.Redis.URL, r.config.Redis.Password, r.config.Redis.Database) + log.Infof("retrieving linked TerraformLayer and TerraformRepository") + cl, err := newK8SClient() if err != nil { + log.Errorf("error creating kubernetes client: %s", err) return err } - r.repository, err = git.PlainClone(WorkingDir, false, cloneOptions) + r.client = cl + err = r.getLayerAndRepository() if err != nil { + log.Errorf("error getting kubernetes resources: %s", err) return err } - workingDir := fmt.Sprintf("%s/%s", WorkingDir, r.config.Runner.Path) - r.terraform, err = tfexec.NewTerraform(workingDir, execPath) + log.Infof("kubernetes resources successfully retrieved") + + log.Infof("cloning repository %s %s branch", r.repository.Spec.Repository.Url, r.layer.Spec.Branch) + r.gitRepository, err = clone(r.config.Runner.Repository, r.repository.Spec.Repository.Url, r.layer.Spec.Branch, r.layer.Spec.Path) if err != nil { return err } - r.terraform.SetStdout(os.Stdout) - r.terraform.SetStderr(os.Stderr) + log.Infof("repository cloned successfully") + + log.Infof("installing binaries...") + err = r.install() + if err != nil { + log.Errorf("error installing binaries: %s", err) + return err + } + log.Infof("binaries successfully installed") + + workingDir := fmt.Sprintf("%s/%s", WorkingDir, r.layer.Spec.Path) log.Infof("Launching terraform init in %s", workingDir) - err = r.terraform.Init(context.Background(), tfexec.Upgrade(true)) + err = r.exec.Init(workingDir) if err != nil { + log.Errorf("") return err } return nil @@ -168,28 +212,30 @@ func (r *Runner) init() error { func (r *Runner) plan() (string, error) { log.Infof("starting terraform plan") - diff, err := r.terraform.Plan(context.Background(), tfexec.Out(PlanArtifact)) + err := r.exec.Plan() if err != nil { - log.Errorf("an error occured during terraform plan: %s", err) + log.Errorf("error executing terraform plan: %s", err) return "", err } - r.terraform.SetStdout(io.Discard) - r.terraform.SetStderr(io.Discard) - planJson, err := r.terraform.ShowPlanFile(context.TODO(), PlanArtifact) + planJsonBytes, err := r.exec.Show() if err != nil { - log.Errorf("an error occured during terraform show: %s", err) + log.Errorf("error getting terraform plan json: %s", err) + return "", err } - planJsonBytes, err := json.Marshal(planJson) + plan := &tfjson.Plan{} + err = json.Unmarshal(planJsonBytes, plan) if err != nil { - log.Errorf("an error occured during json plan parsing: %s", err) + log.Errorf("error parsing terraform json plan: %s", err) + return "", err } + diff, shortDiff := getDiff(plan) planJsonKey := storage.GenerateKey(storage.LastPlannedArtifactJson, r.layer) log.Infof("setting plan json into storage at key %s", planJsonKey) err = r.storage.Set(planJsonKey, planJsonBytes, 3600) if err != nil { log.Errorf("could not put plan json in cache: %s", err) } - err = r.storage.Set(storage.GenerateKey(storage.LastPlanResult, r.layer), []byte(getShortPlanDiff(planJson)), 3600) + err = r.storage.Set(storage.GenerateKey(storage.LastPlanResult, r.layer), []byte(shortDiff), 3600) if err != nil { log.Errorf("could not put short plan in cache: %s", err) } @@ -197,20 +243,20 @@ func (r *Runner) plan() (string, error) { log.Infof("terraform plan diff empty, no subsequent apply should be launched") return "", nil } - plan, err := os.ReadFile(fmt.Sprintf("%s/%s", r.terraform.WorkingDir(), PlanArtifact)) + planBin, err := os.ReadFile(PlanArtifact) if err != nil { log.Errorf("could not read plan output: %s", err) return "", err } - log.Infof("terraform plan ran successfully") - sum := sha256.Sum256(plan) + sum := sha256.Sum256(planBin) planBinKey := storage.GenerateKey(storage.LastPlannedArtifactBin, r.layer) log.Infof("setting plan binary into storage at key %s", planBinKey) - err = r.storage.Set(planBinKey, plan, 3600) + err = r.storage.Set(planBinKey, planBin, 3600) if err != nil { log.Errorf("could not put plan binary in cache: %s", err) return "", err } + log.Infof("terraform plan ran successfully") return b64.StdEncoding.EncodeToString(sum[:]), nil } @@ -224,26 +270,26 @@ func (r *Runner) apply() (string, error) { return "", err } sum := sha256.Sum256(plan) - err = os.WriteFile(fmt.Sprintf("%s/%s", r.terraform.WorkingDir(), PlanArtifact), plan, 0644) + err = os.WriteFile(PlanArtifact, plan, 0644) if err != nil { log.Errorf("could not write plan artifact to disk: %s", err) return "", err } log.Print("launching terraform apply") - err = r.terraform.Apply(context.Background(), tfexec.DirOrPlan(PlanArtifact)) + err = r.exec.Apply() if err != nil { - log.Errorf("an error occured during terraform apply: %s", err) + log.Errorf("error executing terraform apply: %s", err) return "", err } err = r.storage.Set(storage.GenerateKey(storage.LastPlanResult, r.layer), []byte(fmt.Sprintf("Apply: %s", time.Now())), 3600) if err != nil { - log.Errorf("an error occured during apply result storage: %s", err) + log.Errorf("an error occurred during apply result storage: %s", err) } log.Infof("terraform apply ran successfully") return b64.StdEncoding.EncodeToString(sum[:]), nil } -func getShortPlanDiff(plan *tfjson.Plan) string { +func getDiff(plan *tfjson.Plan) (bool, string) { delete := 0 create := 0 update := 0 @@ -258,42 +304,9 @@ func getShortPlanDiff(plan *tfjson.Plan) string { update++ } } - return fmt.Sprintf("Plan: %d to create, %d to update, %d to delete", create, update, delete) -} - -func (r *Runner) getCloneOptions() (*git.CloneOptions, error) { - authMethod := "ssh" - cloneOptions := &git.CloneOptions{ - ReferenceName: plumbing.NewBranchReferenceName(r.config.Runner.Branch), - URL: r.config.Runner.Repository.URL, - } - if strings.Contains(r.config.Runner.Repository.URL, "https://") { - authMethod = "https" - } - log.Infof("clone method is %s", authMethod) - switch authMethod { - case "ssh": - if r.config.Runner.Repository.SSHPrivateKey == "" { - log.Infof("detected keyless authentication") - return cloneOptions, nil - } - log.Infof("private key found") - publicKeys, err := ssh.NewPublicKeys("git", []byte(r.config.Runner.Repository.SSHPrivateKey), "") - if err != nil { - return cloneOptions, err - } - cloneOptions.Auth = publicKeys - - case "https": - if r.config.Runner.Repository.Username != "" && r.config.Runner.Repository.Password != "" { - log.Infof("username and password found") - cloneOptions.Auth = &http.BasicAuth{ - Username: r.config.Runner.Repository.Username, - Password: r.config.Runner.Repository.Password, - } - } else { - log.Infof("passwordless authentication detected") - } + diff := false + if create+delete+update > 0 { + diff = true } - return cloneOptions, nil + return diff, fmt.Sprintf("Plan: %d to create, %d to update, %d to delete", create, update, delete) } diff --git a/internal/runner/terraform/terraform.go b/internal/runner/terraform/terraform.go new file mode 100644 index 00000000..736aac61 --- /dev/null +++ b/internal/runner/terraform/terraform.go @@ -0,0 +1,98 @@ +package terraform + +import ( + "context" + "encoding/json" + "io" + "os" + + "github.com/hashicorp/go-version" + "github.com/hashicorp/hc-install/product" + "github.com/hashicorp/hc-install/releases" + "github.com/hashicorp/terraform-exec/tfexec" +) + +type Terraform struct { + exec *tfexec.Terraform + version string + ExecPath string + planArtifactPath string +} + +func NewTerraform(version, planArtifactPath string) *Terraform { + return &Terraform{ + version: version, + planArtifactPath: planArtifactPath, + } +} + +func (t *Terraform) Install() error { + terraformVersion, err := version.NewVersion(t.version) + if err != nil { + return err + } + installer := &releases.ExactVersion{ + Product: product.Terraform, + Version: version.Must(terraformVersion, nil), + } + execPath, err := installer.Install(context.Background()) + if err != nil { + return err + } + t.ExecPath = execPath + return nil +} + +func (t *Terraform) Init(workingDir string) error { + exec, err := tfexec.NewTerraform(workingDir, t.ExecPath) + if err != nil { + return err + } + t.exec = exec + err = t.exec.Init(context.Background(), tfexec.Upgrade(true)) + if err != nil { + return err + } + return nil +} + +func (t *Terraform) Plan() error { + t.verbose() + _, err := t.exec.Plan(context.Background(), tfexec.Out(t.planArtifactPath)) + if err != nil { + return err + } + return nil +} + +func (t *Terraform) Apply() error { + t.verbose() + err := t.exec.Apply(context.Background(), tfexec.DirOrPlan(t.planArtifactPath)) + if err != nil { + return err + } + return nil +} + +func (t *Terraform) Show() ([]byte, error) { + t.silent() + planJson, err := t.exec.ShowPlanFile(context.TODO(), t.planArtifactPath) + if err != nil { + return nil, err + } + planJsonBytes, err := json.Marshal(planJson) + if err != nil { + return nil, err + } + return planJsonBytes, nil +} + +func (t *Terraform) silent() { + t.exec.SetStdout(io.Discard) + t.exec.SetStderr(io.Discard) +} + +func (t *Terraform) verbose() { + t.exec.SetStdout(os.Stdout) + t.exec.SetStderr(os.Stderr) +} diff --git a/internal/runner/terragrunt/terragrunt.go b/internal/runner/terragrunt/terragrunt.go new file mode 100644 index 00000000..a745bc49 --- /dev/null +++ b/internal/runner/terragrunt/terragrunt.go @@ -0,0 +1,136 @@ +package terragrunt + +import ( + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + + "github.com/padok-team/burrito/internal/runner/terraform" +) + +type Terragrunt struct { + execPath string + planArtifactPath string + version string + workingDir string + terraform *terraform.Terraform +} + +func NewTerragrunt(terraformExec *terraform.Terraform, terragruntVersion, planArtifactPath string) *Terragrunt { + return &Terragrunt{ + version: terragruntVersion, + terraform: terraformExec, + planArtifactPath: planArtifactPath, + } +} + +func verbose(cmd *exec.Cmd) { + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr +} + +func (t *Terragrunt) Install() error { + err := t.terraform.Install() + if err != nil { + return err + } + path, err := downloadTerragrunt(t.version) + if err != nil { + return err + } + t.execPath = path + return nil +} + +func (t *Terragrunt) getDefaultOptions(command string) []string { + return []string{ + command, + "--terragrunt-tfpath", + t.terraform.ExecPath, + "--terragrunt-working-dir", + t.workingDir, + } +} + +func (t *Terragrunt) Init(workingDir string) error { + t.workingDir = workingDir + cmd := exec.Command(t.execPath, t.getDefaultOptions("init")...) + verbose(cmd) + cmd.Dir = t.workingDir + if err := cmd.Run(); err != nil { + return err + } + return nil +} + +func (t *Terragrunt) Plan() error { + options := append(t.getDefaultOptions("plan"), "-out", t.planArtifactPath) + cmd := exec.Command(t.execPath, options...) + verbose(cmd) + cmd.Dir = t.workingDir + if err := cmd.Run(); err != nil { + return err + } + return nil +} + +func (t *Terragrunt) Apply() error { + options := append(t.getDefaultOptions("apply"), t.planArtifactPath) + cmd := exec.Command(t.execPath, options...) + verbose(cmd) + cmd.Dir = t.workingDir + if err := cmd.Run(); err != nil { + return err + } + return nil +} + +func (t *Terragrunt) Show() ([]byte, error) { + options := append(t.getDefaultOptions("show"), "-json", t.planArtifactPath) + cmd := exec.Command(t.execPath, options...) + cmd.Dir = t.workingDir + jsonBytes, err := cmd.Output() + if err != nil { + return nil, err + } + return jsonBytes, nil +} + +func downloadTerragrunt(version string) (string, error) { + cpuArch := runtime.GOARCH + + url := fmt.Sprintf("https://github.com/gruntwork-io/terragrunt/releases/download/v%s/terragrunt_linux_%s", version, cpuArch) + + response, err := http.Get(url) + if err != nil { + return "", err + } + defer response.Body.Close() + + filename := fmt.Sprintf("terragrunt_%s", cpuArch) + file, err := os.Create(filename) + if err != nil { + return "", err + } + defer file.Close() + + _, err = io.Copy(file, response.Body) + if err != nil { + return "", err + } + + err = os.Chmod(filename, 0755) + if err != nil { + return "", err + } + + filepath, err := filepath.Abs(filename) + if err != nil { + return "", err + } + return filepath, nil +} diff --git a/manifests/install.yaml b/manifests/install.yaml index c0d99f7f..0f129ce1 100644 --- a/manifests/install.yaml +++ b/manifests/install.yaml @@ -113,6 +113,18 @@ spec: url: type: string type: object + terraform: + properties: + terragrunt: + properties: + enabled: + type: boolean + version: + type: string + type: object + version: + type: string + type: object type: object status: description: TerraformRepositoryStatus defines the observed state of TerraformRepository @@ -325,8 +337,18 @@ spec: namespace: type: string type: object - terraformVersion: - type: string + terraform: + properties: + terragrunt: + properties: + enabled: + type: boolean + version: + type: string + type: object + version: + type: string + type: object type: object status: description: TerraformLayerStatus defines the observed state of TerraformLayer @@ -601,6 +623,12 @@ metadata: app.kubernetes.io/name: burrito-controllers app.kubernetes.io/part-of: burrito rules: +- apiGroups: ["events.k8s.io"] + resources: ["events"] + verbs: ["create", "update"] +- apiGroups: [""] + resources: ["events"] + verbs: ["create", "update"] - apiGroups: [""] resources: ["pods"] verbs: @@ -611,9 +639,6 @@ rules: - patch - update - watch -- apiGroups: ["events.k8s.io"] - resources: ["events"] - verbs: ["create", "update"] - apiGroups: - config.terraform.padok.cloud resources: @@ -700,6 +725,9 @@ spec: labels: app.kubernetes.io/name: burrito-controllers spec: + # only for testing purposes + imagePullSecrets: + - name: ghcr-creds securityContext: runAsNonRoot: true containers: @@ -708,9 +736,12 @@ spec: args: - controllers - start - image: ghcr.io/padok-team/burrito:v0.1.0 + image: ghcr.io/padok-team/burrito:main name: burrito imagePullPolicy: Always + env: + - name: BURRITO_CONTROLLER_TIMERS_WAITACTION + value: 5s securityContext: allowPrivilegeEscalation: false capabilities: @@ -752,6 +783,9 @@ spec: labels: app.kubernetes.io/name: burrito-server spec: + # only for testing purposes + imagePullSecrets: + - name: ghcr-creds securityContext: runAsNonRoot: true containers: @@ -763,7 +797,16 @@ spec: ports: - containerPort: 8080 name: http - image: ghcr.io/padok-team/burrito:v0.1.0 + env: + # - name: BURRITO_SERVER_PORT + # value: "8080" + # for testing purposes + - name: BURRITO_SERVER_WEBHOOK_GITHUB_SECRET + valueFrom: + secretKeyRef: + name: burrito-webhook-secret + key: burrito-webhook-secret + image: ghcr.io/padok-team/burrito:main name: burrito imagePullPolicy: Always securityContext: