ec_deployment: Avoid unsetting Elasticsearch topology configuration

.changelog/361.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+resource/ec_deployment: Fix bug where some of the settings that were set by the UI were unset by the Terraform provider. See #214 for more details on the bug report.
+```

ec/ecresource/deploymentresource/elasticsearch_expanders.go

@@ -246,6 +246,12 @@ func expandEsTopology(raw interface{}, topologies []*models.ElasticsearchCluster
 				}
 			}
 		}
+
+		if cfg, ok := topology["config"]; ok {
+			if err := expandEsConfig(cfg, elem.Elasticsearch); err != nil {
+				return nil, err
+			}
+		}
 	}
 
 	return res, nil

ec/ecresource/deploymentresource/elasticsearch_flatteners.go

@@ -185,6 +185,9 @@ func flattenEsTopology(plan *models.ElasticsearchClusterPlan) ([]interface{}, er
 			m["autoscaling"] = []interface{}{autoscaling}
 		}
 
+		// Computed config object to avoid unsetting legacy topology config settings.
+		m["config"] = flattenEsConfig(topology.Elasticsearch)
+
 		result = append(result, m)
 	}
 

ec/ecresource/deploymentresource/elasticsearch_flatteners_test.go

@@ -158,6 +158,7 @@ func Test_flattenEsResource(t *testing.T) {
 					"config":         func() []interface{} { return nil }(),
 					"topology": []interface{}{
 						map[string]interface{}{
+							"config":                    func() []interface{} { return nil }(),
 							"id":                        "hot_content",
 							"instance_configuration_id": "aws.data.highio.i3",
 							"size":                      "2g",
@@ -238,6 +239,7 @@ func Test_flattenEsResource(t *testing.T) {
 					"user_settings_override_json": "{\"some.setting\":\"value2\"}",
 				}},
 				"topology": []interface{}{map[string]interface{}{
+					"config":                    func() []interface{} { return nil }(),
 					"id":                        "hot_content",
 					"instance_configuration_id": "aws.data.highio.i3",
 					"size":                      "2g",
@@ -251,7 +253,7 @@ func Test_flattenEsResource(t *testing.T) {
 			}},
 		},
 		{
-			name: "resource with a remote clusters set",
+			name: "resource with remote clusters set",
 			args: args{
 				in: []*models.ElasticsearchResourceInfo{{
 					Region: ec.String("some-region"),
@@ -327,6 +329,7 @@ func Test_flattenEsResource(t *testing.T) {
 					},
 				},
 				"topology": []interface{}{map[string]interface{}{
+					"config":                    func() []interface{} { return nil }(),
 					"id":                        "hot_content",
 					"instance_configuration_id": "aws.data.highio.i3",
 					"size":                      "2g",
@@ -388,18 +391,17 @@ func Test_flattenEsTopology(t *testing.T) {
 					},
 				},
 			}},
-			want: []interface{}{
-				map[string]interface{}{
-					"id":                        "hot_content",
-					"instance_configuration_id": "aws.data.highio.i3",
-					"size":                      "4g",
-					"size_resource":             "memory",
-					"zone_count":                int32(1),
-					"node_type_data":            "true",
-					"node_type_ingest":          "true",
-					"node_type_master":          "true",
-				},
-			},
+			want: []interface{}{map[string]interface{}{
+				"config":                    func() []interface{} { return nil }(),
+				"id":                        "hot_content",
+				"instance_configuration_id": "aws.data.highio.i3",
+				"size":                      "4g",
+				"size_resource":             "memory",
+				"zone_count":                int32(1),
+				"node_type_data":            "true",
+				"node_type_ingest":          "true",
+				"node_type_master":          "true",
+			}},
 		},
 	}
 	for _, tt := range tests {

ec/ecresource/deploymentresource/expanders_test.go

@@ -4576,6 +4576,90 @@ func Test_updateResourceToModel(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "handles Elasticsearch with topology.config block",
+			args: args{
+				d: util.NewResourceData(t, util.ResDataParams{
+					ID: mock.ValidClusterID,
+					State: map[string]interface{}{
+						"name":                   "my_deployment_name",
+						"deployment_template_id": "aws-io-optimized-v2",
+						"region":                 "us-east-1",
+						"version":                "7.10.1",
+						"elasticsearch": []interface{}{map[string]interface{}{
+							"version": "7.10.1",
+							"config":  []interface{}{},
+							"topology": []interface{}{map[string]interface{}{
+								"id":   "hot_content",
+								"size": "8g",
+								"config": []interface{}{map[string]interface{}{
+									"user_settings_yaml": "setting: true",
+								}},
+							}},
+						}},
+					},
+					Schema: newSchema(),
+				}),
+				client: api.NewMock(mock.New200Response(ioOptimizedTpl())),
+			},
+			want: &models.DeploymentUpdateRequest{
+				Name:         "my_deployment_name",
+				PruneOrphans: ec.Bool(true),
+				Settings:     &models.DeploymentUpdateSettings{},
+				Metadata: &models.DeploymentUpdateMetadata{
+					Tags: []*models.MetadataItem{},
+				},
+				Resources: &models.DeploymentUpdateResources{
+					Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, ioOptimizedTpl(), true), &models.ElasticsearchPayload{
+						Region: ec.String("us-east-1"),
+						RefID:  ec.String("main-elasticsearch"),
+						Settings: &models.ElasticsearchClusterSettings{
+							DedicatedMastersThreshold: 6,
+						},
+						Plan: &models.ElasticsearchClusterPlan{
+							AutoscalingEnabled: ec.Bool(false),
+							Elasticsearch: &models.ElasticsearchConfiguration{
+								Version: "7.10.1",
+							},
+							DeploymentTemplate: &models.DeploymentTemplateReference{
+								ID: ec.String("aws-io-optimized-v2"),
+							},
+							ClusterTopology: []*models.ElasticsearchClusterTopologyElement{{
+								ID: "hot_content",
+								Elasticsearch: &models.ElasticsearchConfiguration{
+									NodeAttributes:   map[string]string{"data": "hot"},
+									UserSettingsYaml: "setting: true",
+								},
+								ZoneCount:               2,
+								InstanceConfigurationID: "aws.data.highio.i3",
+								Size: &models.TopologySize{
+									Resource: ec.String("memory"),
+									Value:    ec.Int32(8192),
+								},
+								NodeRoles: []string{
+									"master",
+									"ingest",
+									"remote_cluster_client",
+									"data_hot",
+									"transform",
+									"data_content",
+								},
+								TopologyElementControl: &models.TopologyElementControl{
+									Min: &models.TopologySize{
+										Resource: ec.String("memory"),
+										Value:    ec.Int32(1024),
+									},
+								},
+								AutoscalingMax: &models.TopologySize{
+									Value:    ec.Int32(118784),
+									Resource: ec.String("memory"),
+								},
+							}},
+						},
+					}),
+				},
+			},
+		},
 		{
 			name: "topology change with invalid resources returns an error",
 			args: args{

ec/ecresource/deploymentresource/flatteners_test.go

@@ -1065,6 +1065,73 @@ func Test_modelToState(t *testing.T) {
 				Schema: newSchema(),
 			}),
 		},
+		{
+			name: "flattens an aws plan with topology.config set",
+			args: args{
+				d: newDeploymentRD(t, "123b7b540dfc967a7a649c18e2fce4ed", nil),
+				res: &models.DeploymentGetResponse{
+					ID:    ec.String("123b7b540dfc967a7a649c18e2fce4ed"),
+					Alias: "OH",
+					Name:  ec.String("up2d"),
+					Resources: &models.DeploymentResources{
+						Elasticsearch: []*models.ElasticsearchResourceInfo{{
+							RefID:  ec.String("main-elasticsearch"),
+							Region: ec.String("aws-eu-central-1"),
+							Info: &models.ElasticsearchClusterInfo{
+								Status: ec.String("running"),
+								PlanInfo: &models.ElasticsearchClusterPlansInfo{
+									Current: &models.ElasticsearchClusterPlanInfo{
+										Plan: &models.ElasticsearchClusterPlan{
+											DeploymentTemplate: &models.DeploymentTemplateReference{
+												ID: ec.String("aws-io-optimized-v2"),
+											},
+											Elasticsearch: &models.ElasticsearchConfiguration{
+												Version: "7.13.1",
+											},
+											ClusterTopology: []*models.ElasticsearchClusterTopologyElement{{
+												ID: "hot_content",
+												Size: &models.TopologySize{
+													Value:    ec.Int32(4096),
+													Resource: ec.String("memory"),
+												},
+												Elasticsearch: &models.ElasticsearchConfiguration{
+													UserSettingsYaml: "a.setting: true",
+												},
+											}},
+										},
+									},
+								},
+								Settings: &models.ElasticsearchClusterSettings{},
+							},
+						}},
+					},
+				},
+			},
+			want: util.NewResourceData(t, util.ResDataParams{
+				ID: "123b7b540dfc967a7a649c18e2fce4ed",
+				State: map[string]interface{}{
+					"alias":                  "OH",
+					"deployment_template_id": "aws-io-optimized-v2",
+					"id":                     "123b7b540dfc967a7a649c18e2fce4ed",
+					"name":                   "up2d",
+					"region":                 "aws-eu-central-1",
+					"version":                "7.13.1",
+					"elasticsearch": []interface{}{map[string]interface{}{
+						"region": "aws-eu-central-1",
+						"ref_id": "main-elasticsearch",
+						"topology": []interface{}{map[string]interface{}{
+							"id":            "hot_content",
+							"size":          "4g",
+							"size_resource": "memory",
+							"config": []interface{}{map[string]interface{}{
+								"user_settings_yaml": "a.setting: true",
+							}},
+						}},
+					}},
+				},
+				Schema: newSchema(),
+			}),
+		},
 		{
 			name: "flattens an aws plan (io-optimized) with tags",
 			args: args{d: awsIOOptimizedTagsRD, res: awsIOOptimizedTagsRes},

ec/ecresource/deploymentresource/schema_elasticsearch.go

@@ -211,6 +211,53 @@ func elasticsearchTopologySchema() *schema.Schema {
 						},
 					},
 				},
+
+				// Read only config block that is present in the provider to
+				// avoid unsetting already set 'topology.elasticsearch' in the
+				// deployment plan.
+				"config": {
+					Type:        schema.TypeList,
+					Computed:    true,
+					Description: `Computed read-only configuration to avoid unsetting plan settings from 'topology.elasticsearch'`,
+					Elem: &schema.Resource{
+						Schema: map[string]*schema.Schema{
+							// Settings
+
+							// plugins maps to the `enabled_built_in_plugins` API setting.
+							"plugins": {
+								Type:        schema.TypeSet,
+								Set:         schema.HashString,
+								Description: "List of Elasticsearch supported plugins, which vary from version to version. Check the Stack Pack version to see which plugins are supported for each version. This is currently only available from the UI and [ecctl](https://www.elastic.co/guide/en/ecctl/master/ecctl_stack_list.html)",
+								Computed:    true,
+								Elem: &schema.Schema{
+									Type: schema.TypeString,
+								},
+							},
+
+							// User settings
+							"user_settings_json": {
+								Type:        schema.TypeString,
+								Description: `JSON-formatted user level "elasticsearch.yml" setting overrides`,
+								Computed:    true,
+							},
+							"user_settings_override_json": {
+								Type:        schema.TypeString,
+								Description: `JSON-formatted admin (ECE) level "elasticsearch.yml" setting overrides`,
+								Computed:    true,
+							},
+							"user_settings_yaml": {
+								Type:        schema.TypeString,
+								Description: `YAML-formatted user level "elasticsearch.yml" setting overrides`,
+								Computed:    true,
+							},
+							"user_settings_override_yaml": {
+								Type:        schema.TypeString,
+								Description: `YAML-formatted admin (ECE) level "elasticsearch.yml" setting overrides`,
+								Computed:    true,
+							},
+						},
+					},
+				},
 			},
 		},
 	}