Skip to content

Commit

Permalink
Upgrade TF azurerm v2 and NatGateway pubIP migration
Browse files Browse the repository at this point in the history
  • Loading branch information
dkistner committed Nov 23, 2020
1 parent b4c3648 commit 32e50a8
Show file tree
Hide file tree
Showing 11 changed files with 135 additions and 26 deletions.
2 changes: 1 addition & 1 deletion charts/images.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ images:
- name: terraformer
sourceRepository: github.com/gardener/terraformer
repository: eu.gcr.io/gardener-project/gardener/terraformer
tag: "v1.4.0"
tag: "v1.5.0"

- name: cloud-controller-manager
sourceRepository: github.com/kubernetes/kubernetes
Expand Down
23 changes: 16 additions & 7 deletions charts/internal/azure-infra/templates/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ provider "azurerm" {
tenant_id = "{{ required "azure.tenantID is required" .Values.azure.tenantID }}"
client_id = var.CLIENT_ID
client_secret = var.CLIENT_SECRET

features {}
}

{{ if .Values.create.resourceGroup -}}
Expand Down Expand Up @@ -47,10 +49,8 @@ resource "azurerm_subnet" "workers" {
virtual_network_name = data.azurerm_virtual_network.vnet.name
resource_group_name = data.azurerm_virtual_network.vnet.resource_group_name
{{- end }}
address_prefix = "{{ required "networks.worker is required" .Values.networks.worker }}"
address_prefixes = ["{{ required "networks.worker is required" .Values.networks.worker }}"]
service_endpoints = [{{range $index, $serviceEndpoint := .Values.resourceGroup.subnet.serviceEndpoints}}{{if $index}},{{end}}"{{$serviceEndpoint}}"{{end}}]
route_table_id = azurerm_route_table.workers.id
network_security_group_id = azurerm_network_security_group.workers.id
}

resource "azurerm_route_table" "workers" {
Expand Down Expand Up @@ -109,14 +109,23 @@ resource "azurerm_nat_gateway" "nat" {
resource_group_name = data.azurerm_resource_group.rg.name
{{- end }}
sku_name = "Standard"
public_ip_address_ids = [azurerm_public_ip.natip.id]
{{- if .Values.natGateway }}
{{- if .Values.natGateway.idleConnectionTimeoutMinutes }}
{{ if .Values.natGateway -}}
{{ if .Values.natGateway.idleConnectionTimeoutMinutes -}}
idle_timeout_in_minutes = {{ .Values.natGateway.idleConnectionTimeoutMinutes }}
{{- end }}

# TODO(natipmigration) This can be removed in future versions when the ip migration has been completed.
{{ if .Values.natGateway.migrateNatGatewayToIPAssociation -}}
public_ip_address_ids = []
{{- end }}
{{- end }}
}

resource "azurerm_nat_gateway_public_ip_association" "natip-association" {
nat_gateway_id = azurerm_nat_gateway.nat.id
public_ip_address_id = azurerm_public_ip.natip.id
}

resource "azurerm_subnet_nat_gateway_association" "nat-worker-subnet-association" {
subnet_id = azurerm_subnet.workers.id
nat_gateway_id = azurerm_nat_gateway.nat.id
Expand Down Expand Up @@ -216,4 +225,4 @@ output "{{ .Values.outputKeys.identityID }}" {
output "{{ .Values.outputKeys.identityClientID }}" {
value = data.azurerm_user_assigned_identity.identity.client_id
}
{{- end }}
{{- end }}
9 changes: 5 additions & 4 deletions charts/internal/azure-infra/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,11 @@ create:
# name: identity-name
# resourceGroup: identity-resource-group

natGateway:
idleConnectionTimeoutMinutes:
# TODO(natipmigration) This can be removed in future versions when the ip migration has been completed.
migrateNatGatewayToIPAssociation: false

resourceGroup:
name: my-resource-group
vnet:
Expand Down Expand Up @@ -42,7 +47,3 @@ outputKeys:
securityGroupName: securityGroupName
# identityID: managedIdentityID
# identityClientID: managedIdentityClientID

natGateway:
idleConnectionTimeoutMinutes:

13 changes: 13 additions & 0 deletions hack/api-reference/api.md
Original file line number Diff line number Diff line change
Expand Up @@ -645,6 +645,19 @@ bool
<p>Zoned indicates whether the cluster uses zones</p>
</td>
</tr>
<tr>
<td>
<code>natGatewayPublicIpMigrated</code></br>
<em>
bool
</em>
</td>
<td>
<em>(Optional)</em>
<p>NatGatewayPublicIPMigrated is an indicator if the Gardener managed public ip address is already migrated.
TODO(natipmigration) This can be removed in future versions when the ip migration has been completed.</p>
</td>
</tr>
</tbody>
</table>
<h3 id="azure.provider.extensions.gardener.cloud/v1alpha1.MachineImage">MachineImage
Expand Down
3 changes: 3 additions & 0 deletions pkg/apis/azure/types_infrastructure.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,9 @@ type InfrastructureStatus struct {
Identity *IdentityStatus
// Zoned indicates whether the cluster uses zones
Zoned bool
// NatGatewayPublicIPMigrated is an indicator if the Gardener managed public ip address is already migrated.
// TODO(natipmigration) This can be removed in future versions when the ip migration has been completed.
NatGatewayPublicIPMigrated bool
}

// NetworkStatus is the current status of the infrastructure networks.
Expand Down
4 changes: 4 additions & 0 deletions pkg/apis/azure/v1alpha1/types_infrastructure.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,10 @@ type InfrastructureStatus struct {
// Zoned indicates whether the cluster uses zones
// +optional
Zoned bool `json:"zoned,omitempty"`
// NatGatewayPublicIPMigrated is an indicator if the Gardener managed public ip address is already migrated.
// TODO(natipmigration) This can be removed in future versions when the ip migration has been completed.
// +optional
NatGatewayPublicIPMigrated bool `json:"natGatewayPublicIpMigrated,omitempty"`
}

// NetworkStatus is the current status of the infrastructure networks.
Expand Down
2 changes: 2 additions & 0 deletions pkg/apis/azure/v1alpha1/zz_generated.conversion.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 1 addition & 6 deletions pkg/controller/infrastructure/actuator.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,12 +45,7 @@ func NewActuator() infrastructure.Actuator {
}
}

func (a *actuator) updateProviderStatus(
ctx context.Context,
tf terraformer.Terraformer,
infra *extensionsv1alpha1.Infrastructure,
config *api.InfrastructureConfig,
) error {
func (a *actuator) updateProviderStatus(ctx context.Context, tf terraformer.Terraformer, infra *extensionsv1alpha1.Infrastructure, config *api.InfrastructureConfig) error {
status, err := infrainternal.ComputeStatus(tf, config)
if err != nil {
return err
Expand Down
42 changes: 42 additions & 0 deletions pkg/internal/infrastructure/terraform.go
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,14 @@ func ComputeTerraformerChartValues(infra *extensionsv1alpha1.Infrastructure, cli
}
}

// Checks if the Gardener managed NatGateway public ip needs to be migrated.
// TODO(natipmigration) This can be removed in future versions when the ip migration has been completed.
natGatewayIPMigrationRequired, err := isNatGatewayIPMigrationRequired(infra, config)
if err != nil {
return nil, err
}
natGatewayConfig["migrateNatGatewayToIPAssociation"] = natGatewayIPMigrationRequired

if config.Identity != nil && config.Identity.Name != "" && config.Identity.ResourceGroup != "" {
identityConfig = map[string]interface{}{
"name": config.Identity.Name,
Expand Down Expand Up @@ -227,6 +235,9 @@ type TerraformState struct {
IdentityID string
// IdentityClientID is the client id of the identity.
IdentityClientID string
// NatGatewayIPMigrated is the indicator if the nat gateway ip is migrated.
// TODO(natipmigration) This can be removed in future versions when the ip migration has been completed.
NatGatewayIPMigrated string
}

// ExtractTerraformState extracts the TerraformState from the given Terraformer.
Expand Down Expand Up @@ -289,6 +300,10 @@ func ExtractTerraformState(tf terraformer.Terraformer, config *api.Infrastructur
tfState.IdentityClientID = vars[TerraformerOutputKeyIdentityClientID]
}

if config.Networks.NatGateway != nil && config.Networks.NatGateway.Enabled {
tfState.NatGatewayIPMigrated = "true"
}

return &tfState, nil
}

Expand Down Expand Up @@ -344,6 +359,11 @@ func StatusFromTerraformState(state *TerraformState) *apiv1alpha1.Infrastructure
})
}

// TODO(natipmigration) This can be removed in future versions when the ip migration has been completed.
if state.NatGatewayIPMigrated == "true" {
tfState.NatGatewayPublicIPMigrated = true
}

return &tfState
}

Expand Down Expand Up @@ -418,3 +438,25 @@ func findDomainCounts(cluster *controller.Cluster, infra *extensionsv1alpha1.Inf
updateDomains: *updateDomainCount,
}, nil
}

// isNatGatewayIPMigrationRequired checks if the Gardener managed NatGateway public ip needs to be migrated.
// TODO(natipmigration) This can be removed in future versions when the ip migration has been completed.
func isNatGatewayIPMigrationRequired(infra *extensionsv1alpha1.Infrastructure, config *api.InfrastructureConfig) (bool, error) {
if config.Networks.NatGateway == nil || !config.Networks.NatGateway.Enabled {
return false, nil
}

if infra.Status.ProviderStatus == nil {
return false, nil
}

infrastructureStatus, err := helper.InfrastructureStatusFromInfrastructure(infra)
if err != nil {
return false, err
}

if infrastructureStatus.NatGatewayPublicIPMigrated {
return false, nil
}
return true, nil
}
49 changes: 47 additions & 2 deletions pkg/internal/infrastructure/terraform_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (
apiv1alpha1 "github.com/gardener/gardener-extension-provider-azure/pkg/apis/azure/v1alpha1"
"github.com/gardener/gardener-extension-provider-azure/pkg/internal"
"github.com/gardener/gardener/extensions/pkg/controller"

gardencorev1beta1 "github.com/gardener/gardener/pkg/apis/core/v1beta1"
extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1"
. "github.com/onsi/ginkgo"
Expand Down Expand Up @@ -178,7 +177,9 @@ var _ = Describe("Terraform", func() {
"securityGroupName": TerraformerOutputKeySecurityGroupName,
}

expectedNatGatewayValues = map[string]interface{}{}
expectedNatGatewayValues = map[string]interface{}{
"migrateNatGatewayToIPAssociation": false,
}

expectedValues = map[string]interface{}{
"azure": expectedAzureValues,
Expand Down Expand Up @@ -330,6 +331,50 @@ var _ = Describe("Terraform", func() {
Expect(err).To(Not(HaveOccurred()))
Expect(values).To(BeEquivalentTo(expectedValues))
})

// TODO(natipmigration) This can be removed in future versions when the ip migration has been completed.
Context("NatGateway Gardener managed IP migration", func() {
BeforeEach(func() {
config.Networks.NatGateway = &api.NatGatewayConfig{
Enabled: true,
}
expectedCreateValues["natGateway"] = true
})

It("should migrate the NatGateway IP as it is not yet migrated", func() {
infrastructureStatus := api.InfrastructureStatus{
NatGatewayPublicIPMigrated: false,
}
infrastructureStatusMarshalled, err := json.Marshal(infrastructureStatus)
Expect(err).NotTo(HaveOccurred())

infra.Status.ProviderStatus = &runtime.RawExtension{
Raw: infrastructureStatusMarshalled,
}

expectedNatGatewayValues["migrateNatGatewayToIPAssociation"] = true
values, err := ComputeTerraformerChartValues(infra, clientAuth, config, cluster)
Expect(err).To(Not(HaveOccurred()))
Expect(values).To(BeEquivalentTo(expectedValues))
})

It("should not migrate the NatGateway IP as it is already migrated", func() {
infrastructureStatus := api.InfrastructureStatus{
NatGatewayPublicIPMigrated: true,
}
infrastructureStatusMarshalled, err := json.Marshal(infrastructureStatus)
Expect(err).NotTo(HaveOccurred())

infra.Status.ProviderStatus = &runtime.RawExtension{
Raw: infrastructureStatusMarshalled,
}

expectedNatGatewayValues["migrateNatGatewayToIPAssociation"] = false
values, err := ComputeTerraformerChartValues(infra, clientAuth, config, cluster)
Expect(err).To(Not(HaveOccurred()))
Expect(values).To(BeEquivalentTo(expectedValues))
})
})
})
})

Expand Down
7 changes: 1 addition & 6 deletions pkg/internal/terraform.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,7 @@ func TerraformVariablesEnvironmentFromClientAuth(auth *ClientAuth) map[string]st
}

// NewTerraformer initializes a new Terraformer.
func NewTerraformer(
restConfig *rest.Config,
purpose,
namespace,
name string,
) (terraformer.Terraformer, error) {
func NewTerraformer(restConfig *rest.Config, purpose, namespace, name string) (terraformer.Terraformer, error) {
tf, err := terraformer.NewForConfig(logger.NewLogger("info"), restConfig, purpose, namespace, name, imagevector.TerraformerImage())
if err != nil {
return nil, err
Expand Down

0 comments on commit 32e50a8

Please sign in to comment.