Release v0.5.0 (#6)

* releaser

* no error keypar

* no keypar delete

* no keypar delete

* packer

* endline

* credentials

* fix typo

* sudo

* no packer

* dollar sign

* public and no vpc

* image

* local provider and reset machine

* SubnetId and packer regions

* v0.3.3

* clone amis

* mapping and name

* release/v0.5.0

.github/workflows/build-cml-ami.yml

@@ -0,0 +1,25 @@
+name: cml-ami
+on: 
+  push:
+    tags:
+      - 'ami*'
+
+jobs:
+  build-ami:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Packer
+        env:
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_INSTANCE_TYPE: g2.2xlarge
+        run: |
+          sudo apt install unzip
+
+          cd packer
+          packer validate ami.json
+          packer build ami.json
+          ./clone-amis.go

.gitignore

@@ -1,5 +1,5 @@
 .DS_Store
-terraform-provider-hashicups
+examples
 bin
 
 # Local .terraform directories

Makefile

@@ -13,20 +13,6 @@ default: install
 build:
 	go build -o ${BINARY}
 
-# release:
-# 	GOOS=darwin GOARCH=amd64 go build -o ./bin/${BINARY}_${VERSION}_darwin_amd64
-# 	GOOS=freebsd GOARCH=386 go build -o ./bin/${BINARY}_${VERSION}_freebsd_386
-# 	GOOS=freebsd GOARCH=amd64 go build -o ./bin/${BINARY}_${VERSION}_freebsd_amd64
-# 	GOOS=freebsd GOARCH=arm go build -o ./bin/${BINARY}_${VERSION}_freebsd_arm
-# 	GOOS=linux GOARCH=386 go build -o ./bin/${BINARY}_${VERSION}_linux_386
-# 	GOOS=linux GOARCH=amd64 go build -o ./bin/${BINARY}_${VERSION}_linux_amd64
-# 	GOOS=linux GOARCH=arm go build -o ./bin/${BINARY}_${VERSION}_linux_arm
-# 	GOOS=openbsd GOARCH=386 go build -o ./bin/${BINARY}_${VERSION}_openbsd_386
-# 	GOOS=openbsd GOARCH=amd64 go build -o ./bin/${BINARY}_${VERSION}_openbsd_amd64
-# 	GOOS=solaris GOARCH=amd64 go build -o ./bin/${BINARY}_${VERSION}_solaris_amd64
-# 	GOOS=windows GOARCH=386 go build -o ./bin/${BINARY}_${VERSION}_windows_386
-# 	GOOS=windows GOARCH=amd64 go build -o ./bin/${BINARY}_${VERSION}_windows_amd64
-
 install: build
 	mkdir -p ~/.terraform.d/plugins/${HOSTNAME}/${NAMESPACE}/${NAME}/${VERSION}/${OS_ARCH}
 	mv ${BINARY} ~/.terraform.d/plugins/${HOSTNAME}/${NAMESPACE}/${NAME}/${VERSION}/${OS_ARCH}

README.md

@@ -1,7 +1,67 @@
-# cml-terraform-provider
-Terraform provider for CML
+![Terraform Provider Iterative](https://user-images.githubusercontent.com/414967/98701372-7f60d700-2379-11eb-90d0-47b5eeb22658.png)
 
-```sh
-go mod vendor
-make install
+# Terraform Provider Iterative
+
+The Terraform Iterative provider is a plugin for Terraform that allows for the full lifecycle management of GPU or non GPU cloud resources with your favourite [vendor](#supported-vendors). The provider offers a simple and homogeneous way to deploy a GPU or a cluster of them reducing the complexity.
+
+# Usage
+
+```tf
+terraform {
+  required_providers {
+    iterative = {
+      source = "iterative/iterative"
+      version = "0.5.0"
+    }
+  }
+}
+
+provider "iterative" {}
+
+resource "iterative_machine" "machine" {
+  region = "us-west"
+  instance_name = "machine"
+  instance_hdd_size = "20"
+  instance_type = "m"
+  instance_gpu = "tesla"
+}
 ```
+
+## Argument reference
+
+| Variable | Values | Default | |
+| ------- | ------ | -------- | ------------- |
+| ```region``` | ```us-west``` ```us-east``` ```eu-west``` ```eu-north``` | ```us-west``` | Sets the collocation region |
+| ```instance_name``` |  | cml_{UID} | Sets the instance name and related resources like AWS key pair. |
+| ```instance_hdd_size``` | | 10 | Sets the instance hard disk size in gb |
+| ```instance_type``` | ```m```, ```l```, ```xl``` | ```m``` | Sets thee instance computing size. You can also specify vendor specific machines in AWS i.e. ```t2.micro``` |
+| ```instance_gpu``` | ``` ```, ```testla```, ```k80``` | ``` ``` | Sets the desired GPU  if the ```instance_type``` is one of our types. |
+| ```key_public``` | | | Set up ssh access with your OpenSSH public key. If not provided one be automatically generated and returned in terraform.tfstate  |
+| aws_security_group | | ```cml``` | AWS specific variable to setup an specific security group. If specified the instance will be launched in with that sg within the vpc managed by the specified sg. If not a new sg called ```cml``` will be created under the default vpc |
+
+
+# Supported vendors
+
+ - AWS
+
+### AWS instance equivalences.
+The instance type in AWS is calculated joining the ```instance_type``` and ```instance_gpu```
+
+| type | gpu | aws |
+| ------- | ------ | -------- |
+| m |  | m5.2xlarge |
+| l |  | m5.8xlarge |
+| xl |  | m5.16xlarge |
+| m | k80 | p2.xlarge |
+| l | k80 | p2.8xlarge |
+| xl | k80 | p2.16xlarge |
+| m | tesla | p3.xlarge |
+| l | tesla | p3.8xlarge |
+| xl | tesla | p3.16xlarge |
+
+| region | aws |
+| ------- | ------ |
+| us-west | us-west-1 |
+| us-east | us-east-1 |
+| eu-north | us-north-1 |
+| eu-west | us-west-1 |

cml/ami-test.json

@@ -0,0 +1,44 @@
+{
+    "variables" : {
+        "instance_type" : "{{env `AWS_INSTANCE_TYPE`}}"
+    },
+    "builders" : [
+        {
+            "type" : "amazon-ebs", 
+            "assume_role": {
+                "role_arn": "arn:aws:iam::260760892802:role/dvc-cml-packer",
+                "session_name": "cml-packer-session"
+            },
+            "region" : "us-west-1",
+            "ami_name" : "iterative-cml-test",
+            "ami_description" : "CML (Continous Machine Learning)",
+            "ami_groups": ["all"],
+            "force_deregister": "true",
+            "force_delete_snapshot": "true",
+            "ssh_username" : "ubuntu",
+            "instance_type" : "g2.2xlarge",
+            "source_ami_filter": {
+                "filters": {
+                  "virtualization-type": "hvm",
+                  "name": "iterative-cml",
+                  "root-device-type": "ebs"
+                },
+                "owners": ["260760892802"],
+                "most_recent": true
+            },
+            "run_tags" : {
+                "Author" : "iterative"
+            }
+        }
+    ],
+    "provisioners" : [
+        {
+            "type": "shell",
+            "inline": [
+              "nvidia-smi"
+            ],
+            "start_retry_timeout": "10m",
+            "expect_disconnect": true
+        }
+    ]
+}

cml/ami.json

@@ -0,0 +1,50 @@
+{
+    "variables" : {
+        "instance_type" : "{{env `AWS_INSTANCE_TYPE`}}"
+    },
+    "builders" : [
+        {
+            "type" : "amazon-ebs", 
+            "assume_role": {
+                "role_arn": "arn:aws:iam::260760892802:role/dvc-cml-packer",
+                "session_name": "cml-packer-session"
+            },
+            "region" : "us-west-1",
+            "ami_name" : "iterative-cml",
+            "ami_description" : "CML (Continous Machine Learning)",
+            "ami_groups": ["all"],
+            "force_deregister": "true",
+            "force_delete_snapshot": "true",
+            "ssh_username" : "ubuntu",
+            "instance_type" : "g2.2xlarge",
+            "source_ami_filter": {
+                "filters": {
+                  "virtualization-type": "hvm",
+                  "name": "ubuntu/images/*ubuntu-*-18.04-amd64-server-*",
+                  "root-device-type": "ebs"
+                },
+                "owners": ["099720109477"],
+                "most_recent": true
+            },
+            "run_tags" : {
+                "Author" : "iterative"
+            }
+        }
+    ],
+    "provisioners" : [
+        {
+            "type" : "shell",
+            "environment_vars": ["DEBIAN_FRONTEND=noninteractive"],
+            "script" : "./setup.sh"
+        },
+        {
+            "type": "shell",
+            "inline": [
+              "sudo shutdown -r now",
+              "sleep 60"
+            ],
+            "start_retry_timeout": "10m",
+            "expect_disconnect": true
+        }
+    ]
+}

cml/clone-amis.go

@@ -0,0 +1,98 @@
+//usr/bin/env go run $0 "$@"; exit
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/diag"
+)
+
+func main() {
+	region := "us-west-1"
+	amiName := "iterative-cml"
+	regions := []string{"us-east-1", "us-east-2", "us-west-2", "eu-central-1", "eu-west-1"}
+
+	sess, sessError := session.NewSession(&aws.Config{
+		Region: aws.String(region)},
+	)
+	if sessError != nil {
+		log.Printf("[ERROR] %s", sessError)
+		os.Exit(1)
+	}
+
+	svc := ec2.New(sess)
+
+	amiParams := &ec2.DescribeImagesInput{
+		Filters: []*ec2.Filter{
+			{
+				Name:   aws.String("name"),
+				Values: []*string{aws.String(amiName)},
+			},
+			{
+				Name:   aws.String("architecture"),
+				Values: []*string{aws.String("x86_64")},
+			},
+		},
+	}
+	imagesRes, imagesErr := svc.DescribeImages(amiParams)
+	if imagesErr != nil {
+		diag.FromErr(imagesErr)
+	}
+	if len(imagesRes.Images) == 0 {
+		log.Printf("[ERROR] ami %s not found", amiName)
+		os.Exit(1)
+	}
+
+	ami := imagesRes.Images[0]
+	amiID := *ami.ImageId
+	amiDesc := *ami.Description
+
+	for _, value := range regions {
+		fmt.Println("Cloning", value)
+
+		sess, _ := session.NewSession(&aws.Config{
+			Region: aws.String(value)},
+		)
+
+		svc := ec2.New(sess)
+
+		copyResult, err := svc.CopyImage(&ec2.CopyImageInput{
+			SourceImageId: aws.String(amiID),
+			SourceRegion:  aws.String(region),
+			Name:          aws.String(amiName),
+			Description:   aws.String(amiDesc),
+		})
+		if err != nil {
+			fmt.Println(err)
+		}
+
+		svc.WaitUntilImageExists(&ec2.DescribeImagesInput{
+			ImageIds: []*string{aws.String(*copyResult.ImageId)},
+			Filters: []*ec2.Filter{
+				{
+					Name:   aws.String("state"),
+					Values: []*string{aws.String("available")},
+				},
+			},
+		})
+
+		_, modifyErr := svc.ModifyImageAttribute(&ec2.ModifyImageAttributeInput{
+			ImageId: aws.String(*copyResult.ImageId),
+			LaunchPermission: &ec2.LaunchPermissionModifications{
+				Add: []*ec2.LaunchPermission{
+					{
+						Group: aws.String("all"),
+					},
+				},
+			},
+		})
+		if modifyErr != nil {
+			fmt.Println(modifyErr)
+		}
+	}
+}

cml/setup.sh

@@ -0,0 +1,13 @@
+#/bin/sh
+
+echo "APT::Get::Assume-Yes \"true\";" | sudo tee -a /etc/apt/apt.conf.d/90assumeyes
+
+curl -fsSL https://get.docker.com -o get-docker.sh && sh get-docker.sh && \
+sudo usermod -aG docker \${USER}
+sudo setfacl --modify user:\${USER}:rw /var/run/docker.sock
+
+curl -s -L https://nvidia.GitHub.io/nvidia-docker/gpgkey | sudo apt-key add - && \
+curl -s -L https://nvidia.GitHub.io/nvidia-docker/ubuntu18.04/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list && \
+sudo apt update && sudo apt install -y ubuntu-drivers-common  && \
+sudo ubuntu-drivers autoinstall  && \
+sudo apt install -y nvidia-container-toolkit

examples/main.tf

@@ -1,18 +1,18 @@
-
 terraform {
   required_providers {
     iterative = {
-      source = "DavidGOrtega/iterative"
-      version = "0.4.0"
+      #source = "DavidGOrtega/iterative"
+      #version = "0.4.0"
+      versions = ["0.3"]
+      source = "github.com/davidgortega/iterative"
     }
   }
 }
 
 provider "iterative" {}
 
 resource "iterative_machine" "machine" {
-  region = "us-east-1"
-
-
-
-}
+  region = "us-west"
+  instance_type = "t2.micro"
+  instance_gpu = "tesla"
+}