From e86bd14d4c98f74661ddc4d4be1d09918a4a4bdd Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Mon, 20 Aug 2018 15:08:22 -0700 Subject: [PATCH 1/5] add the AmazonEKS template function to the amazon_eks and kubectl_apply docs --- hack/docs/mutations.json | 5 +++++ hack/docs/schema.json | 5 +++++ hack/docs/src/markdown.ts | 2 ++ 3 files changed, 12 insertions(+) diff --git a/hack/docs/mutations.json b/hack/docs/mutations.json index f2083b71d..ef37b4264 100644 --- a/hack/docs/mutations.json +++ b/hack/docs/mutations.json @@ -494,6 +494,7 @@ }, "merge": { "description": "An `amazon_eks` asset generates a terraform file that will create an Amazon EKS Cluster.", + "extended_description": "It also populates a template function `AmazonEKS` that takes the name of the cluster and returns the path to the generated kubeconfig for the cluster. This path is only valid after the `render` and `terraform` lifecycle steps have run. This is intended to be used within the [kubectl_apply](/api/ship-lifecycle/lifecycle/kubectl_apply/) lifecycle step.}}", "examples": [ { "cluster_name": "existing-vpc-cluster", @@ -825,6 +826,10 @@ { "path": "k8s/another.yml", "kubeconfig": "k8s/generated_kubeconfig" + }, + { + "path": "k8s/another.yml", + "kubeconfig": "{{repl AmazonEKS \"eks_cluster_name\" }}" } ] }, diff --git a/hack/docs/schema.json b/hack/docs/schema.json index 7a1ae375d..702772711 100644 --- a/hack/docs/schema.json +++ b/hack/docs/schema.json @@ -13,6 +13,7 @@ "properties": { "amazon_eks": { "description": "An `amazon_eks` asset generates a terraform file that will create an Amazon EKS Cluster.", + "extended_description": "It also populates a template function `AmazonEKS` that takes the name of the cluster and returns the path to the generated kubeconfig for the cluster. This path is only valid after the `render` and `terraform` lifecycle steps have run. This is intended to be used within the [kubectl_apply](/api/ship-lifecycle/lifecycle/kubectl_apply/) lifecycle step.}}", "examples": [ { "cluster_name": "existing-vpc-cluster", @@ -748,6 +749,10 @@ { "path": "k8s/another.yml", "kubeconfig": "k8s/generated_kubeconfig" + }, + { + "path": "k8s/another.yml", + "kubeconfig": "{{repl AmazonEKS \"eks_cluster_name\" }}" } ], "type": "object", diff --git a/hack/docs/src/markdown.ts b/hack/docs/src/markdown.ts index fe3de6174..e3da560c1 100644 --- a/hack/docs/src/markdown.ts +++ b/hack/docs/src/markdown.ts @@ -161,6 +161,8 @@ gradient: "purpleToPink" ${specTypes[specType].description || ""} +${specTypes[specType].extended_description || ""} + `; } From 6417ef7b3894855f079944fd3d4622e07f374013 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Mon, 20 Aug 2018 15:21:47 -0700 Subject: [PATCH 2/5] update amazon_eks extended description --- hack/docs/mutations.json | 2 +- hack/docs/schema.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/docs/mutations.json b/hack/docs/mutations.json index ef37b4264..1b71b529c 100644 --- a/hack/docs/mutations.json +++ b/hack/docs/mutations.json @@ -494,7 +494,7 @@ }, "merge": { "description": "An `amazon_eks` asset generates a terraform file that will create an Amazon EKS Cluster.", - "extended_description": "It also populates a template function `AmazonEKS` that takes the name of the cluster and returns the path to the generated kubeconfig for the cluster. This path is only valid after the `render` and `terraform` lifecycle steps have run. This is intended to be used within the [kubectl_apply](/api/ship-lifecycle/lifecycle/kubectl_apply/) lifecycle step.}}", + "extended_description": "It also populates a template function `AmazonEKS` that takes the name of the cluster and returns the path to the generated kubeconfig for the cluster. This template function is only valid after the asset has been generated as part of the `render` lifecycle step, but can be used by later assets within that step. The file itself is created when the generated terraform is applied, whether by the `terraform` lifecycle step or otherwise. This is intended to be used within the [kubectl_apply](/api/ship-lifecycle/lifecycle/kubectl_apply/) lifecycle step.", "examples": [ { "cluster_name": "existing-vpc-cluster", diff --git a/hack/docs/schema.json b/hack/docs/schema.json index 702772711..15851cdc5 100644 --- a/hack/docs/schema.json +++ b/hack/docs/schema.json @@ -13,7 +13,7 @@ "properties": { "amazon_eks": { "description": "An `amazon_eks` asset generates a terraform file that will create an Amazon EKS Cluster.", - "extended_description": "It also populates a template function `AmazonEKS` that takes the name of the cluster and returns the path to the generated kubeconfig for the cluster. This path is only valid after the `render` and `terraform` lifecycle steps have run. This is intended to be used within the [kubectl_apply](/api/ship-lifecycle/lifecycle/kubectl_apply/) lifecycle step.}}", + "extended_description": "It also populates a template function `AmazonEKS` that takes the name of the cluster and returns the path to the generated kubeconfig for the cluster. This template function is only valid after the asset has been generated as part of the `render` lifecycle step, but can be used by later assets within that step. The file itself is created when the generated terraform is applied, whether by the `terraform` lifecycle step or otherwise. This is intended to be used within the [kubectl_apply](/api/ship-lifecycle/lifecycle/kubectl_apply/) lifecycle step.", "examples": [ { "cluster_name": "existing-vpc-cluster", From f99c2a96e64c8274fe4b44734e735263c8ff2848 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Mon, 20 Aug 2018 15:51:44 -0700 Subject: [PATCH 3/5] add integration test for AmazonEKS template function also include that set of template functions in more builders --- .../expected/.ship/release.yml | 47 ++++++++ .../expected/.ship/state.json | 5 + .../expected/installer/install.sh | 4 + .../expected/installer/kube.yaml | 1 + .../expected/installer/new/new_vpc.tf | 100 ++++++++++++++++++ .../input/.ship/release.yml | 47 ++++++++ .../input/.ship/state.json | 1 + .../base/amazon-eks-template/metadata.yaml | 6 ++ pkg/lifecycle/message/cli.go | 3 + pkg/lifecycle/message/messenger.go | 3 + pkg/lifecycle/render/docker/step.go | 3 + pkg/lifecycle/render/inline/render.go | 3 + pkg/lifecycle/render/web/step.go | 3 + 13 files changed, 226 insertions(+) create mode 100644 integration/base/amazon-eks-template/expected/.ship/release.yml create mode 100644 integration/base/amazon-eks-template/expected/.ship/state.json create mode 100644 integration/base/amazon-eks-template/expected/installer/install.sh create mode 100644 integration/base/amazon-eks-template/expected/installer/kube.yaml create mode 100644 integration/base/amazon-eks-template/expected/installer/new/new_vpc.tf create mode 100644 integration/base/amazon-eks-template/input/.ship/release.yml create mode 100644 integration/base/amazon-eks-template/input/.ship/state.json create mode 100644 integration/base/amazon-eks-template/metadata.yaml diff --git a/integration/base/amazon-eks-template/expected/.ship/release.yml b/integration/base/amazon-eks-template/expected/.ship/release.yml new file mode 100644 index 000000000..f12d987bf --- /dev/null +++ b/integration/base/amazon-eks-template/expected/.ship/release.yml @@ -0,0 +1,47 @@ +assets: + v1: + - amazon_eks: + dest: new/new_vpc.tf + cluster_name: new-vpc-cluster + region: "us-west-2" + created_vpc: + vpc_cidr: "10.0.0.0/16" + zones: + - us-west-2a + - us-west-2b + public_subnets: + - "10.0.1.0/24" + - "10.0.2.0/24" + private_subnets: + - "10.0.129.0/24" + - "10.0.130.0/24" + autoscaling_groups: + - name: alpha + group_size: 3 + machine_type: m5.2xlarge + - name: bravo + group_size: 1 + machine_type: m5.4xlarge + - inline: + dest: install.sh + contents: | + #!/bin/bash + echo "run:" + echo "terraform apply -f new/new_vpc.tf" + echo "kubectl apply -f kube.yaml --kubeconfig {{repl AmazonEKS "new-vpc-cluster" }}" + mode: 0777 + - inline: + dest: kube.yaml + contents: | + this is not a valid kubernetes yaml + mode: 0777 + +config: {} + +lifecycle: + v1: + - message: + contents: "hi" + - render: {} + - message: + contents: "bye" diff --git a/integration/base/amazon-eks-template/expected/.ship/state.json b/integration/base/amazon-eks-template/expected/.ship/state.json new file mode 100644 index 000000000..c804e4b39 --- /dev/null +++ b/integration/base/amazon-eks-template/expected/.ship/state.json @@ -0,0 +1,5 @@ +{ + "v1": { + "config": {} + } +} \ No newline at end of file diff --git a/integration/base/amazon-eks-template/expected/installer/install.sh b/integration/base/amazon-eks-template/expected/installer/install.sh new file mode 100644 index 000000000..7c12c2ce7 --- /dev/null +++ b/integration/base/amazon-eks-template/expected/installer/install.sh @@ -0,0 +1,4 @@ +#!/bin/bash +echo "run:" +echo "terraform apply -f new/new_vpc.tf" +echo "kubectl apply -f kube.yaml --kubeconfig new/kubeconfig_new-vpc-cluster" diff --git a/integration/base/amazon-eks-template/expected/installer/kube.yaml b/integration/base/amazon-eks-template/expected/installer/kube.yaml new file mode 100644 index 000000000..4c99ece15 --- /dev/null +++ b/integration/base/amazon-eks-template/expected/installer/kube.yaml @@ -0,0 +1 @@ +this is not a valid kubernetes yaml diff --git a/integration/base/amazon-eks-template/expected/installer/new/new_vpc.tf b/integration/base/amazon-eks-template/expected/installer/new/new_vpc.tf new file mode 100644 index 000000000..70ce01ae0 --- /dev/null +++ b/integration/base/amazon-eks-template/expected/installer/new/new_vpc.tf @@ -0,0 +1,100 @@ + +variable "vpc_cidr" { + type = "string" + default = "10.0.0.0/16" +} + +variable "vpc_public_subnets" { + default = [ + "10.0.1.0/24", + "10.0.2.0/24", + ] +} + +variable "vpc_private_subnets" { + default = [ + "10.0.129.0/24", + "10.0.130.0/24", + ] +} + +variable "vpc_azs" { + default = [ + "us-west-2a", + "us-west-2b", + ] +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "1.37.0" + name = "eks-vpc" + cidr = "${var.vpc_cidr}" + azs = "${var.vpc_azs}" + + private_subnets = "${var.vpc_private_subnets}" + public_subnets = "${var.vpc_public_subnets}" + + map_public_ip_on_launch = true + enable_nat_gateway = true + single_nat_gateway = true + + tags = "${map("kubernetes.io/cluster/${var.eks-cluster-name}", "shared")}" +} + +locals { + "eks_vpc" = "${module.vpc.vpc_id}" + "eks_vpc_public_subnets" = "${module.vpc.public_subnets}" + "eks_vpc_private_subnets" = "${module.vpc.private_subnets}" +} + +locals { + "worker_group_count" = "2" +} + +locals { + "worker_groups" = [ + { + name = "alpha" + asg_min_size = "3" + asg_max_size = "3" + asg_desired_capacity = "3" + instance_type = "m5.2xlarge" + + subnets = "${join(",", local.eks_vpc_private_subnets)}" + }, + { + name = "bravo" + asg_min_size = "1" + asg_max_size = "1" + asg_desired_capacity = "1" + instance_type = "m5.4xlarge" + + subnets = "${join(",", local.eks_vpc_private_subnets)}" + }, + ] +} + +provider "aws" { + version = "~> 1.27" + region = "us-west-2" +} + +variable "eks-cluster-name" { + default = "new-vpc-cluster" + type = "string" +} + +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "1.4.0" + + cluster_name = "${var.eks-cluster-name}" + + subnets = ["${local.eks_vpc_private_subnets}", "${local.eks_vpc_public_subnets}"] + + vpc_id = "${local.eks_vpc}" + + worker_group_count = "${local.worker_group_count}" + worker_groups = "${local.worker_groups}" +} diff --git a/integration/base/amazon-eks-template/input/.ship/release.yml b/integration/base/amazon-eks-template/input/.ship/release.yml new file mode 100644 index 000000000..f12d987bf --- /dev/null +++ b/integration/base/amazon-eks-template/input/.ship/release.yml @@ -0,0 +1,47 @@ +assets: + v1: + - amazon_eks: + dest: new/new_vpc.tf + cluster_name: new-vpc-cluster + region: "us-west-2" + created_vpc: + vpc_cidr: "10.0.0.0/16" + zones: + - us-west-2a + - us-west-2b + public_subnets: + - "10.0.1.0/24" + - "10.0.2.0/24" + private_subnets: + - "10.0.129.0/24" + - "10.0.130.0/24" + autoscaling_groups: + - name: alpha + group_size: 3 + machine_type: m5.2xlarge + - name: bravo + group_size: 1 + machine_type: m5.4xlarge + - inline: + dest: install.sh + contents: | + #!/bin/bash + echo "run:" + echo "terraform apply -f new/new_vpc.tf" + echo "kubectl apply -f kube.yaml --kubeconfig {{repl AmazonEKS "new-vpc-cluster" }}" + mode: 0777 + - inline: + dest: kube.yaml + contents: | + this is not a valid kubernetes yaml + mode: 0777 + +config: {} + +lifecycle: + v1: + - message: + contents: "hi" + - render: {} + - message: + contents: "bye" diff --git a/integration/base/amazon-eks-template/input/.ship/state.json b/integration/base/amazon-eks-template/input/.ship/state.json new file mode 100644 index 000000000..abdf17200 --- /dev/null +++ b/integration/base/amazon-eks-template/input/.ship/state.json @@ -0,0 +1 @@ +{"v1":{"config":{}}} \ No newline at end of file diff --git a/integration/base/amazon-eks-template/metadata.yaml b/integration/base/amazon-eks-template/metadata.yaml new file mode 100644 index 000000000..05b3520e7 --- /dev/null +++ b/integration/base/amazon-eks-template/metadata.yaml @@ -0,0 +1,6 @@ +customer_id: "-Am-_6i5pw0u4AbspOwKN4lZUCn49u_G" +installation_id: "Jn0nK8OlT3k9AYqpnq6SGqzo_srKjzjQ" +release_version: "0.0.1-b" + +disable_online: true +skip_cleanup: false diff --git a/pkg/lifecycle/message/cli.go b/pkg/lifecycle/message/cli.go index 7ec89a74a..5c069cd18 100644 --- a/pkg/lifecycle/message/cli.go +++ b/pkg/lifecycle/message/cli.go @@ -58,6 +58,9 @@ func (m *CLIMessenger) getBuilder(release *api.Release) templates.Builder { Meta: release.Metadata, Viper: m.Viper, }, + templates.ShipContext{ + Logger: m.Logger, + }, ) return builder } diff --git a/pkg/lifecycle/message/messenger.go b/pkg/lifecycle/message/messenger.go index d1dd2b643..4d0a4540d 100644 --- a/pkg/lifecycle/message/messenger.go +++ b/pkg/lifecycle/message/messenger.go @@ -17,6 +17,9 @@ func (m *DaemonMessenger) getBuilder(meta api.ReleaseMetadata) templates.Builder Meta: meta, Viper: m.Viper, }, + templates.ShipContext{ + Logger: m.Logger, + }, ) return builder } diff --git a/pkg/lifecycle/render/docker/step.go b/pkg/lifecycle/render/docker/step.go index 1b63187eb..c4f0d35b6 100644 --- a/pkg/lifecycle/render/docker/step.go +++ b/pkg/lifecycle/render/docker/step.go @@ -91,6 +91,9 @@ func (p *DefaultStep) Execute( Meta: meta, Viper: p.Viper, }, + templates.ShipContext{ + Logger: p.Logger, + }, ) builtDest, err := builder.String(dest) if err != nil { diff --git a/pkg/lifecycle/render/inline/render.go b/pkg/lifecycle/render/inline/render.go index 0c34e916e..b278e4b39 100644 --- a/pkg/lifecycle/render/inline/render.go +++ b/pkg/lifecycle/render/inline/render.go @@ -72,6 +72,9 @@ func (r *LocalRenderer) Execute( Meta: meta, Viper: r.Viper, }, + templates.ShipContext{ + Logger: r.Logger, + }, ) built, err := builder.String(asset.Contents) diff --git a/pkg/lifecycle/render/web/step.go b/pkg/lifecycle/render/web/step.go index 6c45762b3..70367e114 100644 --- a/pkg/lifecycle/render/web/step.go +++ b/pkg/lifecycle/render/web/step.go @@ -119,6 +119,9 @@ func (p *DefaultStep) buildAsset( Meta: meta, Viper: p.Viper, }, + templates.ShipContext{ + Logger: p.Logger, + }, ) builtURL, err := builder.String(asset.URL) From dfee36d282159df12d0ea5c71d9e04d042e2c540 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Mon, 20 Aug 2018 16:15:33 -0700 Subject: [PATCH 4/5] moved amazon-eks-template integration test to init_app suite --- integration/base/amazon-eks-template/input/.ship/state.json | 1 - integration/base/amazon-eks-template/metadata.yaml | 6 ------ .../amazon-eks-template/expected/.ship/release.yml | 0 .../amazon-eks-template/expected/.ship/state.json | 0 .../amazon-eks-template/expected/installer/install.sh | 0 .../amazon-eks-template/expected/installer/kube.yaml | 0 .../amazon-eks-template/expected/installer/new/new_vpc.tf | 0 .../amazon-eks-template/input/.ship/release.yml | 0 integration/init_app/amazon-eks-template/metadata.yaml | 5 +++++ 9 files changed, 5 insertions(+), 7 deletions(-) delete mode 100644 integration/base/amazon-eks-template/input/.ship/state.json delete mode 100644 integration/base/amazon-eks-template/metadata.yaml rename integration/{base => init_app}/amazon-eks-template/expected/.ship/release.yml (100%) rename integration/{base => init_app}/amazon-eks-template/expected/.ship/state.json (100%) rename integration/{base => init_app}/amazon-eks-template/expected/installer/install.sh (100%) rename integration/{base => init_app}/amazon-eks-template/expected/installer/kube.yaml (100%) rename integration/{base => init_app}/amazon-eks-template/expected/installer/new/new_vpc.tf (100%) rename integration/{base => init_app}/amazon-eks-template/input/.ship/release.yml (100%) create mode 100644 integration/init_app/amazon-eks-template/metadata.yaml diff --git a/integration/base/amazon-eks-template/input/.ship/state.json b/integration/base/amazon-eks-template/input/.ship/state.json deleted file mode 100644 index abdf17200..000000000 --- a/integration/base/amazon-eks-template/input/.ship/state.json +++ /dev/null @@ -1 +0,0 @@ -{"v1":{"config":{}}} \ No newline at end of file diff --git a/integration/base/amazon-eks-template/metadata.yaml b/integration/base/amazon-eks-template/metadata.yaml deleted file mode 100644 index 05b3520e7..000000000 --- a/integration/base/amazon-eks-template/metadata.yaml +++ /dev/null @@ -1,6 +0,0 @@ -customer_id: "-Am-_6i5pw0u4AbspOwKN4lZUCn49u_G" -installation_id: "Jn0nK8OlT3k9AYqpnq6SGqzo_srKjzjQ" -release_version: "0.0.1-b" - -disable_online: true -skip_cleanup: false diff --git a/integration/base/amazon-eks-template/expected/.ship/release.yml b/integration/init_app/amazon-eks-template/expected/.ship/release.yml similarity index 100% rename from integration/base/amazon-eks-template/expected/.ship/release.yml rename to integration/init_app/amazon-eks-template/expected/.ship/release.yml diff --git a/integration/base/amazon-eks-template/expected/.ship/state.json b/integration/init_app/amazon-eks-template/expected/.ship/state.json similarity index 100% rename from integration/base/amazon-eks-template/expected/.ship/state.json rename to integration/init_app/amazon-eks-template/expected/.ship/state.json diff --git a/integration/base/amazon-eks-template/expected/installer/install.sh b/integration/init_app/amazon-eks-template/expected/installer/install.sh similarity index 100% rename from integration/base/amazon-eks-template/expected/installer/install.sh rename to integration/init_app/amazon-eks-template/expected/installer/install.sh diff --git a/integration/base/amazon-eks-template/expected/installer/kube.yaml b/integration/init_app/amazon-eks-template/expected/installer/kube.yaml similarity index 100% rename from integration/base/amazon-eks-template/expected/installer/kube.yaml rename to integration/init_app/amazon-eks-template/expected/installer/kube.yaml diff --git a/integration/base/amazon-eks-template/expected/installer/new/new_vpc.tf b/integration/init_app/amazon-eks-template/expected/installer/new/new_vpc.tf similarity index 100% rename from integration/base/amazon-eks-template/expected/installer/new/new_vpc.tf rename to integration/init_app/amazon-eks-template/expected/installer/new/new_vpc.tf diff --git a/integration/base/amazon-eks-template/input/.ship/release.yml b/integration/init_app/amazon-eks-template/input/.ship/release.yml similarity index 100% rename from integration/base/amazon-eks-template/input/.ship/release.yml rename to integration/init_app/amazon-eks-template/input/.ship/release.yml diff --git a/integration/init_app/amazon-eks-template/metadata.yaml b/integration/init_app/amazon-eks-template/metadata.yaml new file mode 100644 index 000000000..5338afbef --- /dev/null +++ b/integration/init_app/amazon-eks-template/metadata.yaml @@ -0,0 +1,5 @@ +customer_id: "-Am-_6i5pw0u4AbspOwKN4lZUCn49u_G" +release_version: "1.0.0-amazon-eks-template" + +disable_online: false +skip_cleanup: false From 8e08f435e56f9839c7b6087d01e4dac143681a42 Mon Sep 17 00:00:00 2001 From: Andrew Lavery Date: Mon, 20 Aug 2018 16:40:43 -0700 Subject: [PATCH 5/5] fix parallel init_app integration tests --- integration/init_app/integration_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/integration/init_app/integration_test.go b/integration/init_app/integration_test.go index 23ddbb22a..4cc242322 100644 --- a/integration/init_app/integration_test.go +++ b/integration/init_app/integration_test.go @@ -8,6 +8,7 @@ import ( "net/url" "os" "path" + "path/filepath" "strings" "testing" "time" @@ -19,9 +20,9 @@ import ( "github.com/replicatedhq/ship/pkg/cli" "github.com/replicatedhq/ship/pkg/e2e" "github.com/replicatedhq/ship/pkg/logger" + "github.com/spf13/afero" "github.com/spf13/viper" "gopkg.in/yaml.v2" - "github.com/spf13/afero" ) type TestMetadata struct { @@ -88,7 +89,7 @@ var _ = Describe("ship init replicated.app/...", func() { // if a token is provided, try to ensure the release matches what we have here in the repo if vendorToken != "" { - channelName := fmt.Sprintf("integration replicated.app %s", file.Name()) + channelName := fmt.Sprintf("integration replicated.app %s", filepath.Base(testPath)) installationID = createRelease(vendorEndpoint, vendorToken, testInputPath, testMetadata, channelName) } close(done)