allow create new VPC and bastion machine

pingcap · Apr 19, 2019 · 55241c4 · 55241c4
1 parent 696d683
commit 55241c4
Show file tree

Hide file tree

Showing 5 changed files with 191 additions and 73 deletions.
diff --git a/deploy/aws/README.md b/deploy/aws/README.md
@@ -15,20 +15,54 @@ https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html
 ``` shell
 $ git clone https://github.com/pingcap/tidb-operator
 $ cd tidb-operator/cloud/aws
-
-$ # customize variables.tf according to your needs
-$ # especially you need to adjust the vpc_id and subnets id list
 $ terraform init
 $ terraform apply
 ```
 
-After `terraform apply` is executed successfully, you can access the `monitor_endpoint` using your web browser and access the TiDB cluster via `tidb_endpoint` using MySQL clinet in an ec2 instance in the VPC. If the endpoint DNS name is not resolvable, be patient and wait a few minutes.
+After `terraform apply` is executed successfully, you can access the `monitor_endpoint` using your web browser.
+
+To access TiDB cluster, use the following command to first ssh into the bastion machine, and then connect it via MySQL client:
+
+``` shell
+ssh -i credentials/k8s-prod-my-cluster.pem ec2-user@<bastion_ip>
+mysql -h <tidb_dns> -P <tidb_port> -u root
+```
+
+If the DNS name is not resolvable, be patient and wait a few minutes.
+
+You can interact with the EKS cluster using `kubectl` and `helm` with the kubeconfig file `credentials/kubeconfig_<cluster_name>`. The default `cluster_name` is `my-cluster`, you can change it in the variables.tf.
+
+``` shell
+# By specifying --kubeconfig argument
+kubectl --kubeconfig credentials/kubeconfig_<cluster_name> get po -n tidb
+helm --kubeconfig credentials/kubeconfig_<cluster_name> ls
+
+# Or setting KUBECONFIG environment variable
+export KUBECONFIG=$PWD/credentials/kubeconfig_<cluster_name>
+kubectl get po -n tidb
+helm ls
+```
+
+> **NOTE:** You have to manually delete the EBS volumes after running `terraform destroy` if you don't need the data on the volumes any more.
+
+## Upgrade TiDB cluster
+
+To upgrade TiDB cluster, modify `tidb_version` variable to a higher version in variables.tf and run `terraform apply`.
+
+## Scale TiDB cluster
+
+To scale TiDB cluster, modify `tikv_count` or `tidb_count` to your desired count, and then run `terraform apply`.
+
+## Customize
+
+By default, the terraform script will create a new VPC. You can use an existing VPC by setting `create_vpc` to `false` and specify your existing VPC id and subnet ids to `vpc_id` and `subnets` variables.
+
+An ec2 instance is also created by default as bastion machine to connect to the created TiDB cluster, because the TiDB service is exposed as an [Internal Elastic Load Balancer](https://aws.amazon.com/blogs/aws/internal-elastic-load-balancers/). The ec2 instance has MySQL and Sysbench pre-installed, so you can SSH into the ec2 instance and connect to TiDB using the ELB endpoint. You can disable the bastion instance creation by setting `create_bastion` to `false` if you already have an ec2 instance in the VPC.
 
-You can interact with the EKS cluster using `kubectl` and `helm` with the kubeconfig file `credentials/kubeconfig_<cluster_name>`.
+The TiDB version and component count are also configurable in variables.tf, you can customize these variables to suit your need.
 
-You have to manually delete the EBS volumes after running `terraform destroy` if you don't need the data on the volumes any more.
+Currently, the instance type of TiDB cluster component is not configurable because PD and TiKV relies on [NVMe SSD instance store](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ssd-instance-store.html), different instance types have different disks.
 
 ## TODO
 - [ ] auto-scaling group policy
-- [ ] Allow both creating VPC or using existing VPC
-- [ ] Delete load balancers automatically when running `terraform destroy`
+- [ ] Allow create a minimal TiDB cluster
diff --git a/deploy/aws/bastion-userdata b/deploy/aws/bastion-userdata
@@ -0,0 +1,6 @@
+#cloud-config
+packages:
+- mysql
+runcmd:
+- curl -s https://packagecloud.io/install/repositories/akopytov/sysbench/script.rpm.sh | bash
+- yum -y install sysbench
diff --git a/deploy/aws/main.tf b/deploy/aws/main.tf
@@ -2,56 +2,88 @@ provider "aws" {
   region = "${var.region}"
 }
 
-provider "kubernetes" {
-  config_path = "${path.module}/credentials/kubeconfig_my-cluster"
+module "key-pair" {
+  source  = "cloudposse/key-pair/aws"
+  version = "0.3.2"
+
+  name = "${var.cluster_name}"
+  namespace = "k8s"
+  stage = "prod"
+  ssh_public_key_path = "${path.module}/credentials/"
+  generate_ssh_key = "true"
+  private_key_extension = ".pem"
+  chmod_command = "chmod 600 %v"
 }
 
-provider "helm" {
-  insecure = true
-  kubernetes {
-    config_path = "${path.module}/credentials/kubeconfig_my-cluster"
+resource "aws_security_group" "ssh" {
+  name = "${var.cluster_name}"
+  description = "Allow SSH access for bastion instance"
+  vpc_id = "${var.create_vpc ? module.vpc.vpc_id : var.vpc_id}"
+  ingress {
+    from_port = 22
+    to_port = 22
+    protocol = "tcp"
+    cidr_blocks = "${var.ingress_cidr}"
+  }
+ egress {
+    from_port       = 0
+    to_port         = 0
+    protocol        = "-1"
+    cidr_blocks     = ["0.0.0.0/0"]
   }
 }
 
-resource "aws_key_pair" "k8s_node" {
-  key_name_prefix = "${var.cluster_name}"
-  public_key = "${file("~/.ssh/id_rsa.pub")}"
+module "vpc" {
+  source = "terraform-aws-modules/vpc/aws"
+  version = "1.60.0"
+  name = "${var.cluster_name}"
+  cidr = "${var.vpc_cidr}"
+  create_vpc = "${var.create_vpc}"
+  azs = ["${data.aws_availability_zones.available.names[0]}", "${data.aws_availability_zones.available.names[1]}", "${data.aws_availability_zones.available.names[2]}"]
+  private_subnets = "${var.private_subnets}"
+  public_subnets = "${var.public_subnets}"
+  enable_nat_gateway = true
+  single_nat_gateway = true
+
+  # The following tags are required for ELB
+  private_subnet_tags = {
+    "kubernetes.io/cluster/${var.cluster_name}" = "shared"
+  }
+  public_subnet_tags = {
+    "kubernetes.io/cluster/${var.cluster_name}" = "shared"
+  }
+  vpc_tags = {
+    "kubernetes.io/cluster/${var.cluster_name}" = "shared"
+  }
 }
 
-# module "vpc" {
-#   source = "terraform-aws-modules/vpc/aws"
-#   version = "1.60.0"
-#   name = "${var.cluster_name}"
-#   cidr = "${var.vpc_cidr}"
-#   create_vpc = "${var.create_vpc}"
-#   azs = ["${data.aws_availability_zones.available.names[0]}", "${data.aws_availability_zones.available.names[1]}", "${data.aws_availability_zones.available.names[2]}"]
-#   private_subnets = "${var.private_subnets}"
-#   public_subnets = "${var.public_subnets}"
-#   enable_nat_gateway = true
-#   single_nat_gateway = true
-
-#   # The following tags are required for ELB
-#   private_subnet_tags = {
-#     "kubernetes.io/cluster/${var.cluster_name}" = "shared"
-#   }
-#   public_subnet_tags = {
-#     "kubernetes.io/cluster/${var.cluster_name}" = "shared"
-#   }
-#   vpc_tags = {
-#     "kubernetes.io/cluster/${var.cluster_name}" = "shared"
-#   }
-# }
+module "ec2" {
+  source = "terraform-aws-modules/ec2-instance/aws"
+  version = "1.21.0"
+  name = "${var.cluster_name}-bastion"
+  instance_count = "${var.create_bastion ? 1:0}"
+  ami = "${var.bastion_ami}"
+  instance_type = "${var.bastion_instance_type}"
+  key_name = "${module.key-pair.key_name}"
+  associate_public_ip_address = true
+  monitoring = false
+  user_data = "${file("bastion-userdata")}"
+  vpc_security_group_ids = ["${aws_security_group.ssh.id}"]
+  subnet_ids = "${split(",", var.create_vpc ? join(",", module.vpc.public_subnets) : join(",", var.subnets))}"
+
+  tags = {
+    app = "tidb"
+  }
+}
 
 module "eks" {
   source = "terraform-aws-modules/eks/aws"
   version = "2.3.1"
   cluster_name = "${var.cluster_name}"
   cluster_version = "${var.k8s_version}"
   config_output_path = "credentials/"
-  # subnets = ["${module.vpc.private_subnets}"]
-  # vpc_id = "${module.vpc.vpc_id}"
-  subnets = "${var.subnets}"
-  vpc_id = "${var.vpc_id}"
+  subnets = "${split(",", var.create_vpc ? join(",", module.vpc.private_subnets) : join(",", var.subnets))}"
+  vpc_id = "${var.create_vpc ? module.vpc.vpc_id : var.vpc_id}"
 
   # instance types: https://aws.amazon.com/ec2/instance-types/
   # instance prices: https://aws.amazon.com/ec2/pricing/on-demand/
@@ -60,7 +92,7 @@ module "eks" {
     {
       # pd
       name = "pd_worker_group"
-      key_name = "${aws_key_pair.k8s_node.key_name}"
+      key_name = "${module.key-pair.key_name}"
       # WARNING: if you change instance type, you must also modify the corresponding disk mounting in pd-userdata.sh script
       # instance_type = "c5d.xlarge" # 4c, 8G, 100G NVMe SSD
       instance_type = "m5d.xlarge" # 4c, 16G, 150G NVMe SSD
@@ -73,7 +105,7 @@ module "eks" {
     },
     { # tikv
       name = "tikv_worker_group"
-      key_name = "${aws_key_pair.k8s_node.key_name}"
+      key_name = "${module.key-pair.key_name}"
       # WARNING: if you change instance type, you must also modify the corresponding disk mounting in tikv-userdata.sh script
       instance_type = "i3.2xlarge" # 8c, 61G, 1.9T NVMe SSD
       root_volume_type = "gp2"
@@ -86,7 +118,7 @@ module "eks" {
     },
     { # tidb
       name = "tidb_worker_group"
-      key_name = "${aws_key_pair.k8s_node.key_name}"
+      key_name = "${module.key-pair.key_name}"
       instance_type = "c4.4xlarge" # 16c, 30G
       root_volume_type = "gp2"
       root_volume_size = "100"
@@ -97,7 +129,7 @@ module "eks" {
     },
     { # monitor
       name = "monitor_worker_group"
-      key_name = "${aws_key_pair.k8s_node.key_name}"
+      key_name = "${module.key-pair.key_name}"
       instance_type = "c5.xlarge" # 4c, 8G
       root_volume_type = "gp2"
       root_volume_size = "100"
@@ -114,6 +146,19 @@ module "eks" {
   }
 }
 
+provider "kubernetes" {
+  config_path = "${path.module}/credentials/kubeconfig_my-cluster"
+}
+
+provider "helm" {
+  insecure = true
+  # service_account = "tiller"
+  # install_tiller = true # currently this doesn't work, so we install tiller in the local-exec provisioner. See https://github.com/terraform-providers/terraform-provider-helm/issues/148
+  kubernetes {
+    config_path = "${path.module}/credentials/kubeconfig_my-cluster"
+  }
+}
+
 resource "null_resource" "setup-env" {
   depends_on = ["module.eks"]
 
@@ -125,6 +170,9 @@ kubectl apply -f manifests/local-volume-provisioner.yaml
 kubectl apply -f manifests/gp2-storageclass.yaml
 kubectl apply -f manifests/tiller-rbac.yaml
 helm init --service-account tiller --upgrade --wait
+until helm ls; do
+  echo "Wait tiller ready"
+done
 EOS
     environment = {
       KUBECONFIG = "${path.module}/credentials/kubeconfig_my-cluster"

diff --git a/deploy/aws/outputs.tf b/deploy/aws/outputs.tf
@@ -18,12 +18,22 @@ output "eks_endpoint" {
   value = "${module.eks.cluster_endpoint}"
 }
 
-output "tidb_endpoint" {
-  description = "tidb service endpoint"
-  value = "${data.kubernetes_service.tidb.load_balancer_ingress.0.hostname}:4000"
+output "tidb_dns" {
+  description = "tidb service dns name"
+  value = "${data.kubernetes_service.tidb.load_balancer_ingress.0.hostname}"
+}
+
+output "tidb_port" {
+  description = "tidb service port"
+  value = "4000"
 }
 
 output "monitor_endpoint" {
   description = "monitor service endpoint"
   value = "http://${data.kubernetes_service.monitor.load_balancer_ingress.0.hostname}:3000"
 }
+
+output "bastion_ip" {
+  description = "Bastion IP address"
+  value = "${module.ec2.public_ip}"
+}
diff --git a/deploy/aws/variables.tf b/deploy/aws/variables.tf
@@ -3,27 +3,32 @@ variable "region" {
   default = "us-east-2"
 }
 
-# variable "create_vpc" {
-#   description = "Create a new VPC or not, if true the vpc_cidr/private_subnets/public_subnets must be set correctly, otherwise vpc_id/subnet_ids must be set correctly"
-#   default = true
-# }
-
-# variable "vpc_cidr" {
-#   description = "vpc cidr"
-#   default = "10.0.0.0/16"
-# }
-
-# variable "private_subnets" {
-#   description = "vpc private subnets"
-#   type = "list"
-#   default = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"]
-# }
-
-# variable "public_subnets" {
-#   description = "vpc public subnets"
-#   type = "list"
-#   default = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"]
-# }
+variable "ingress_cidr" {
+  description = "IP cidr that allowed to access bastion ec2 instance"
+  default = ["0.0.0.0/0"]	# Note: Please restrict your ingress to only necessary IPs. Opening to 0.0.0.0/0 can lead to security vulnerabilities.
+}
+
+variable "create_vpc" {
+  description = "Create a new VPC or not, if true the vpc_cidr/private_subnets/public_subnets must be set correctly, otherwise vpc_id/subnet_ids must be set correctly"
+  default = true
+}
+
+variable "vpc_cidr" {
+  description = "vpc cidr"
+  default = "10.0.0.0/16"
+}
+
+variable "private_subnets" {
+  description = "vpc private subnets"
+  type = "list"
+  default = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"]
+}
+
+variable "public_subnets" {
+  description = "vpc public subnets"
+  type = "list"
+  default = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"]
+}
 
 variable "vpc_id" {
   description = "VPC id"
@@ -37,6 +42,21 @@ variable "subnets" {
   default = ["subnet-899e79f3", "subnet-a72d80cf", "subnet-a76d34ea"]
 }
 
+variable "create_bastion" {
+  description = "Create bastion ec2 instance to access TiDB cluster"
+  default = true
+}
+
+variable "bastion_ami" {
+  description = "bastion ami id"
+  default = "ami-0cd3dfa4e37921605"
+}
+
+variable "bastion_instance_type" {
+  description = "bastion ec2 instance type"
+  default = "t2.micro"
+}
+
 variable "cluster_name" {
   description = "eks cluster name"
   default = "my-cluster"
@@ -49,7 +69,7 @@ variable "k8s_version" {
 
 variable "tidb_version" {
   description = "tidb cluster version"
-  default = "v2.1.0"
+  default = "v2.1.8"
 }
 
 variable "pd_count" {