From b1bbb8a07d93236efc1591b2cfec0df67d123192 Mon Sep 17 00:00:00 2001 From: Riley Karson Date: Mon, 5 Nov 2018 14:47:40 -0800 Subject: [PATCH 1/2] Add support for accelerators to dataproc. --- .../resources/resource_dataproc_cluster.go | 57 +++++++++++++++++++ .../tests/resource_dataproc_cluster_test.go | 45 +++++++++++++++ .../docs/r/dataproc_cluster.html.markdown | 50 +++++++++++++++- 3 files changed, 151 insertions(+), 1 deletion(-) diff --git a/provider/terraform/resources/resource_dataproc_cluster.go b/provider/terraform/resources/resource_dataproc_cluster.go index d3bccffc7757..525dd7005abc 100644 --- a/provider/terraform/resources/resource_dataproc_cluster.go +++ b/provider/terraform/resources/resource_dataproc_cluster.go @@ -362,6 +362,30 @@ func instanceConfigSchema() *schema.Schema { }, }, + // Note: preemptible workers don't support accelerators + "accelerators": { + // Terraform was unhappy when this was a set, even though it probably could be one + Type: schema.TypeList, + Optional: true, + ForceNew: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "accelerator_type": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + DiffSuppressFunc: compareSelfLinkOrResourceName, + }, + + "accelerator_count": { + Type: schema.TypeInt, + Required: true, + ForceNew: true, + }, + }, + }, + }, + "instance_names": { Type: schema.TypeList, Computed: true, @@ -607,9 +631,26 @@ func expandInstanceGroupConfig(cfg map[string]interface{}) *dataproc.InstanceGro } } } + + icg.Accelerators = expandAccelerators(cfg["accelerators"].([]interface{})) return icg } +func expandAccelerators(configured []interface{}) []*dataproc.AcceleratorConfig { + accelerators := make([]*dataproc.AcceleratorConfig, 0, len(configured)) + for _, raw := range configured { + data := raw.(map[string]interface{}) + accelerator := dataproc.AcceleratorConfig{ + AcceleratorTypeUri: data["accelerator_type"].(string), + AcceleratorCount: int64(data["accelerator_count"].(int)), + } + + accelerators = append(accelerators, &accelerator) + } + + return accelerators +} + func resourceDataprocClusterUpdate(d *schema.ResourceData, meta interface{}) error { config := meta.(*Config) @@ -746,6 +787,20 @@ func flattenSoftwareConfig(d *schema.ResourceData, sc *dataproc.SoftwareConfig) return []map[string]interface{}{data} } +func flattenAccelerators(accelerators []*dataproc.AcceleratorConfig) []map[string]interface{} { + acceleratorsSchema := make([]map[string]interface{}, 0, len(accelerators)) + for _, accelerator := range accelerators { + data := map[string]interface{}{ + "accelerator_type": accelerator.AcceleratorTypeUri, + "accelerator_count": accelerator.AcceleratorCount, + } + + acceleratorsSchema = append(acceleratorsSchema, data) + } + + return acceleratorsSchema +} + func flattenInitializationActions(nia []*dataproc.NodeInitializationAction) ([]map[string]interface{}, error) { actions := []map[string]interface{}{} @@ -819,6 +874,8 @@ func flattenInstanceGroupConfig(d *schema.ResourceData, icg *dataproc.InstanceGr disk["num_local_ssds"] = icg.DiskConfig.NumLocalSsds disk["boot_disk_type"] = icg.DiskConfig.BootDiskType } + + data["accelerators"] = flattenAccelerators(icg.Accelerators) } data["disk_config"] = []map[string]interface{}{disk} diff --git a/provider/terraform/tests/resource_dataproc_cluster_test.go b/provider/terraform/tests/resource_dataproc_cluster_test.go index 089e9febe24d..44adbee6deba 100644 --- a/provider/terraform/tests/resource_dataproc_cluster_test.go +++ b/provider/terraform/tests/resource_dataproc_cluster_test.go @@ -134,6 +134,22 @@ func TestAccDataprocCluster_basic(t *testing.T) { }) } +func TestAccDataprocCluster_withAccelerators(t *testing.T) { + t.Parallel() + + rnd := acctest.RandString(10) + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckDataprocClusterDestroy(), + Steps: []resource.TestStep{ + { + Config: testAccDataprocCluster_withAccelerators(rnd), + }, + }, + }) +} + func TestAccDataprocCluster_withInternalIpOnlyTrue(t *testing.T) { t.Parallel() @@ -630,6 +646,35 @@ resource "google_dataproc_cluster" "basic" { `, rnd) } +func testAccDataprocCluster_withAccelerators(rnd string) string { + return fmt.Sprintf(` +resource "google_dataproc_cluster" "accelerated_cluster" { + name = "dproc-cluster-test-%s" + region = "us-central1" + + cluster_config { + gce_cluster_config { + zone = "us-central1-a" + } + + master_config { + accelerators { + accelerator_type = "nvidia-tesla-k80" + accelerator_count = "1" + } + } + + worker_config { + accelerators { + accelerator_type = "nvidia-tesla-k80" + accelerator_count = "1" + } + } + } +} +`, rnd) +} + func testAccDataprocCluster_withInternalIpOnlyTrue(rnd string) string { return fmt.Sprintf(` variable subnetwork_cidr { diff --git a/provider/terraform/website/docs/r/dataproc_cluster.html.markdown b/provider/terraform/website/docs/r/dataproc_cluster.html.markdown index 84d2b2b87268..955e1d47bb2a 100644 --- a/provider/terraform/website/docs/r/dataproc_cluster.html.markdown +++ b/provider/terraform/website/docs/r/dataproc_cluster.html.markdown @@ -16,14 +16,18 @@ Manages a Cloud Dataproc cluster resource within GCP. For more information see `labels`,`cluster_config.worker_config.num_instances` and `cluster_config.preemptible_worker_config.num_instances` are non-updateable. Changing others will cause recreation of the whole cluster! -## Example usage +## Example Usage - Basic ```hcl resource "google_dataproc_cluster" "simplecluster" { name = "simplecluster" region = "us-central1" } +``` + +## Example Usage - Advanced +```hcl resource "google_dataproc_cluster" "mycluster" { name = "mycluster" region = "us-central1" @@ -79,6 +83,28 @@ resource "google_dataproc_cluster" "mycluster" { } ``` +## Example Usage - Using a GPU accelerator + +```hcl +resource "google_dataproc_cluster" "accelerated_cluster" { + name = "my-cluster-with-gpu" + region = "us-central1" + + cluster_config { + gce_cluster_config { + zone = "us-central1-a" + } + + master_config { + accelerators { + accelerator_type = "nvidia-tesla-k80" + accelerator_count = "1" + } + } + } +} +``` + ## Argument Reference * `name` - (Required) The name of the cluster, unique within the project and @@ -239,6 +265,17 @@ The `cluster_config.master_config` block supports: * `disk_config.num_local_ssds` - (Optional) The amount of local SSD disks that will be attached to each master cluster node. Defaults to 0. +* `accelerators` (Optional) The Compute Engine accelerator (GPU) configuration for these instances. Can be specified multiple times. + + * `accelerators.accelerator_type` - (Required) Full URL, partial URI, or short name of the accelerator type resource to expose to this instance. + If you are using the Cloud Dataproc Auto Zone Placement feature, you must use the short name of the accelerator type resource, for example, `nvidia-tesla-k80`. + + * `accelerators.accelerator_count` - (Required) The number of the accelerator cards of this type exposed to this instance. Often restricted to one of `1`, `2`, `4`, or `8`. + +~> The Cloud Dataproc API can return unintuitive error messages when using accelerators; even when you have defined an accelerator, Auto Zone Placement does not exclusively select +zones that have that accelerator available. If you get a 400 error that the accelerator can't be found, this is a likely cause. Make sure you check [accelerator availability by zone](https://cloud.google.com/compute/docs/reference/rest/v1/acceleratorTypes/list) +if you are trying to use accelerators in a given zone. + - - - The `cluster_config.worker_config` block supports: @@ -282,6 +319,17 @@ The `cluster_config.worker_config` block supports: * `num_local_ssds` - (Optional) The amount of local SSD disks that will be attached to each worker cluster node. Defaults to 0. +* `accelerators` (Optional) The Compute Engine accelerator configuration for these instances. Can be specified multiple times. + + * `accelerators.accelerator_type` - (Required) Full URL, partial URI, or short name of the accelerator type resource to expose to this instance. + If you are using the Cloud Dataproc Auto Zone Placement feature, you must use the short name of the accelerator type resource, for example, `nvidia-tesla-k80`. + + * `accelerators.accelerator_count` - (Required) The number of the accelerator cards of this type exposed to this instance. Often restricted to one of `1`, `2`, `4`, or `8`. + +~> The Cloud Dataproc API can return unintuitive error messages when using accelerators; even when you have defined an accelerator, Auto Zone Placement does not exclusively select +zones that have that accelerator available. If you get a 400 error that the accelerator can't be found, this is a likely cause. Make sure you check [accelerator availability by zone](https://cloud.google.com/compute/docs/reference/rest/v1/acceleratorTypes/list) +if you are trying to use accelerators in a given zone. + - - - The `cluster_config.preemptible_worker_config` block supports: From c7b00ad03c4ba965bede3c7551625fa39bad5157 Mon Sep 17 00:00:00 2001 From: Riley Karson Date: Tue, 6 Nov 2018 16:07:11 -0800 Subject: [PATCH 2/2] Review comments. --- .../resources/resource_dataproc_cluster.go | 2 +- .../tests/resource_dataproc_cluster_test.go | 24 ++++++++++++----- .../docs/r/dataproc_cluster.html.markdown | 26 +++++++++---------- 3 files changed, 32 insertions(+), 20 deletions(-) diff --git a/provider/terraform/resources/resource_dataproc_cluster.go b/provider/terraform/resources/resource_dataproc_cluster.go index 525dd7005abc..0a5e3ade06a6 100644 --- a/provider/terraform/resources/resource_dataproc_cluster.go +++ b/provider/terraform/resources/resource_dataproc_cluster.go @@ -364,7 +364,7 @@ func instanceConfigSchema() *schema.Schema { // Note: preemptible workers don't support accelerators "accelerators": { - // Terraform was unhappy when this was a set, even though it probably could be one + // Terraform throws an error if you try to set this field while it is a TypeSet. Type: schema.TypeList, Optional: true, ForceNew: true, diff --git a/provider/terraform/tests/resource_dataproc_cluster_test.go b/provider/terraform/tests/resource_dataproc_cluster_test.go index 44adbee6deba..15ff69f77757 100644 --- a/provider/terraform/tests/resource_dataproc_cluster_test.go +++ b/provider/terraform/tests/resource_dataproc_cluster_test.go @@ -138,13 +138,25 @@ func TestAccDataprocCluster_withAccelerators(t *testing.T) { t.Parallel() rnd := acctest.RandString(10) + + project := getTestProjectFromEnv() + zone := "us-central1-a" + acceleratorType := "nvidia-tesla-k80" + acceleratorLink := fmt.Sprintf("https://www.googleapis.com/compute/beta/projects/%s/zones/%s/acceleratorTypes/%s", project, zone, acceleratorType) + resource.Test(t, resource.TestCase{ PreCheck: func() { testAccPreCheck(t) }, Providers: testAccProviders, CheckDestroy: testAccCheckDataprocClusterDestroy(), Steps: []resource.TestStep{ { - Config: testAccDataprocCluster_withAccelerators(rnd), + Config: testAccDataprocCluster_withAccelerators(rnd, zone, acceleratorType), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("google_dataproc_cluster.accelerated_cluster", "cluster_config.0.master_config.0.accelerators.0.accelerator_count", "1"), + resource.TestCheckResourceAttr("google_dataproc_cluster.accelerated_cluster", "cluster_config.0.master_config.0.accelerators.0.accelerator_type", acceleratorLink), + resource.TestCheckResourceAttr("google_dataproc_cluster.accelerated_cluster", "cluster_config.0.worker_config.0.accelerators.0.accelerator_count", "1"), + resource.TestCheckResourceAttr("google_dataproc_cluster.accelerated_cluster", "cluster_config.0.worker_config.0.accelerators.0.accelerator_type", acceleratorLink), + ), }, }, }) @@ -646,7 +658,7 @@ resource "google_dataproc_cluster" "basic" { `, rnd) } -func testAccDataprocCluster_withAccelerators(rnd string) string { +func testAccDataprocCluster_withAccelerators(rnd, zone, acceleratorType string) string { return fmt.Sprintf(` resource "google_dataproc_cluster" "accelerated_cluster" { name = "dproc-cluster-test-%s" @@ -654,25 +666,25 @@ resource "google_dataproc_cluster" "accelerated_cluster" { cluster_config { gce_cluster_config { - zone = "us-central1-a" + zone = "%s" } master_config { accelerators { - accelerator_type = "nvidia-tesla-k80" + accelerator_type = "%s" accelerator_count = "1" } } worker_config { accelerators { - accelerator_type = "nvidia-tesla-k80" + accelerator_type = "%s" accelerator_count = "1" } } } } -`, rnd) +`, rnd, zone, acceleratorType, acceleratorType) } func testAccDataprocCluster_withInternalIpOnlyTrue(rnd string) string { diff --git a/provider/terraform/website/docs/r/dataproc_cluster.html.markdown b/provider/terraform/website/docs/r/dataproc_cluster.html.markdown index 955e1d47bb2a..fc0f7e52c6f9 100644 --- a/provider/terraform/website/docs/r/dataproc_cluster.html.markdown +++ b/provider/terraform/website/docs/r/dataproc_cluster.html.markdown @@ -87,8 +87,8 @@ resource "google_dataproc_cluster" "mycluster" { ```hcl resource "google_dataproc_cluster" "accelerated_cluster" { - name = "my-cluster-with-gpu" - region = "us-central1" + name = "my-cluster-with-gpu" + region = "us-central1" cluster_config { gce_cluster_config { @@ -99,9 +99,9 @@ resource "google_dataproc_cluster" "accelerated_cluster" { accelerators { accelerator_type = "nvidia-tesla-k80" accelerator_count = "1" - } - } - } + } + } + } } ``` @@ -253,24 +253,24 @@ The `cluster_config.master_config` block supports: * `disk_config` (Optional) Disk Config - * `disk_config.boot_disk_type` - (Optional) The disk type of the primary disk attached to each node. + * `boot_disk_type` - (Optional) The disk type of the primary disk attached to each node. One of `"pd-ssd"` or `"pd-standard"`. Defaults to `"pd-standard"`. - * `disk_config.boot_disk_size_gb` - (Optional, Computed) Size of the primary disk attached to each node, specified + * `boot_disk_size_gb` - (Optional, Computed) Size of the primary disk attached to each node, specified in GB. The primary disk contains the boot volume and system libraries, and the smallest allowed disk size is 10GB. GCP will default to a predetermined computed value if not set (currently 500GB). Note: If SSDs are not attached, it also contains the HDFS data blocks and Hadoop working directories. - * `disk_config.num_local_ssds` - (Optional) The amount of local SSD disks that will be + * `num_local_ssds` - (Optional) The amount of local SSD disks that will be attached to each master cluster node. Defaults to 0. * `accelerators` (Optional) The Compute Engine accelerator (GPU) configuration for these instances. Can be specified multiple times. - * `accelerators.accelerator_type` - (Required) Full URL, partial URI, or short name of the accelerator type resource to expose to this instance. + * `accelerator_type` - (Required) Full URL, partial URI, or short name of the accelerator type resource to expose to this instance. If you are using the Cloud Dataproc Auto Zone Placement feature, you must use the short name of the accelerator type resource, for example, `nvidia-tesla-k80`. - * `accelerators.accelerator_count` - (Required) The number of the accelerator cards of this type exposed to this instance. Often restricted to one of `1`, `2`, `4`, or `8`. + * `accelerator_count` - (Required) The number of the accelerator cards of this type exposed to this instance. Often restricted to one of `1`, `2`, `4`, or `8`. ~> The Cloud Dataproc API can return unintuitive error messages when using accelerators; even when you have defined an accelerator, Auto Zone Placement does not exclusively select zones that have that accelerator available. If you get a 400 error that the accelerator can't be found, this is a likely cause. Make sure you check [accelerator availability by zone](https://cloud.google.com/compute/docs/reference/rest/v1/acceleratorTypes/list) @@ -308,7 +308,7 @@ The `cluster_config.worker_config` block supports: * `disk_config` (Optional) Disk Config - * `disk_config.boot_disk_type` - (Optional) The disk type of the primary disk attached to each node. + * `boot_disk_type` - (Optional) The disk type of the primary disk attached to each node. One of `"pd-ssd"` or `"pd-standard"`. Defaults to `"pd-standard"`. * `boot_disk_size_gb` - (Optional, Computed) Size of the primary disk attached to each worker node, specified @@ -321,10 +321,10 @@ The `cluster_config.worker_config` block supports: * `accelerators` (Optional) The Compute Engine accelerator configuration for these instances. Can be specified multiple times. - * `accelerators.accelerator_type` - (Required) Full URL, partial URI, or short name of the accelerator type resource to expose to this instance. + * `accelerator_type` - (Required) Full URL, partial URI, or short name of the accelerator type resource to expose to this instance. If you are using the Cloud Dataproc Auto Zone Placement feature, you must use the short name of the accelerator type resource, for example, `nvidia-tesla-k80`. - * `accelerators.accelerator_count` - (Required) The number of the accelerator cards of this type exposed to this instance. Often restricted to one of `1`, `2`, `4`, or `8`. + * `accelerator_count` - (Required) The number of the accelerator cards of this type exposed to this instance. Often restricted to one of `1`, `2`, `4`, or `8`. ~> The Cloud Dataproc API can return unintuitive error messages when using accelerators; even when you have defined an accelerator, Auto Zone Placement does not exclusively select zones that have that accelerator available. If you get a 400 error that the accelerator can't be found, this is a likely cause. Make sure you check [accelerator availability by zone](https://cloud.google.com/compute/docs/reference/rest/v1/acceleratorTypes/list)