From 4d4e6e884b56671d4ba9bf97e294f2b5c8a7ea48 Mon Sep 17 00:00:00 2001 From: upodroid Date: Mon, 19 Feb 2024 21:45:26 +0300 Subject: [PATCH] refactor regional/global GCE API methods --- cloudmock/gce/mockcompute/api.go | 4 +- cloudmock/gce/mockcompute/backend_services.go | 2 +- cloudmock/gce/mockcompute/health_checks.go | 2 +- pkg/model/gcemodel/api_loadbalancer.go | 29 +++- pkg/resources/gce/gce.go | 58 ++++++- .../minimal_gce_dns-none/kubernetes.tf | 44 +++-- .../minimal_gce_ilb/kubernetes.tf | 41 +++-- .../kubernetes.tf | 41 +++-- .../minimal_gce_plb/kubernetes.tf | 4 +- upup/pkg/fi/cloudup/gce/compute.go | 163 ++++++++++++++---- upup/pkg/fi/cloudup/gce/gce_cloud.go | 3 +- upup/pkg/fi/cloudup/gcetasks/address.go | 28 +-- .../fi/cloudup/gcetasks/backend_service.go | 40 +++-- .../pkg/fi/cloudup/gcetasks/forwardingrule.go | 30 ++-- upup/pkg/fi/cloudup/gcetasks/healthcheck.go | 64 ++++--- 15 files changed, 387 insertions(+), 166 deletions(-) diff --git a/cloudmock/gce/mockcompute/api.go b/cloudmock/gce/mockcompute/api.go index 8f5803d3ca14c..00876539d5c17 100644 --- a/cloudmock/gce/mockcompute/api.go +++ b/cloudmock/gce/mockcompute/api.go @@ -137,11 +137,11 @@ func (c *MockClient) HTTPHealthChecks() gce.HttpHealthChecksClient { return c.httpHealthChecksClient } -func (c *MockClient) RegionHealthChecks() gce.RegionHealthChecksClient { +func (c *MockClient) HealthChecks() gce.HealthChecksClient { return c.healthCheckClient } -func (c *MockClient) RegionBackendServices() gce.RegionBackendServiceClient { +func (c *MockClient) BackendServices() gce.BackendServiceClient { return c.backendServiceClient } diff --git a/cloudmock/gce/mockcompute/backend_services.go b/cloudmock/gce/mockcompute/backend_services.go index 4e75ff5a6ca93..ca6e8ad6f07a2 100644 --- a/cloudmock/gce/mockcompute/backend_services.go +++ b/cloudmock/gce/mockcompute/backend_services.go @@ -31,7 +31,7 @@ type backendServiceClient struct { sync.Mutex } -var _ gce.RegionBackendServiceClient = &backendServiceClient{} +var _ gce.BackendServiceClient = &backendServiceClient{} func newBackendServiceClient() *backendServiceClient { return &backendServiceClient{ diff --git a/cloudmock/gce/mockcompute/health_checks.go b/cloudmock/gce/mockcompute/health_checks.go index 0aee37594b7cd..de4938c06f275 100644 --- a/cloudmock/gce/mockcompute/health_checks.go +++ b/cloudmock/gce/mockcompute/health_checks.go @@ -31,7 +31,7 @@ type healthCheckClient struct { sync.Mutex } -var _ gce.RegionHealthChecksClient = &healthCheckClient{} +var _ gce.HealthChecksClient = &healthCheckClient{} func newHealthCheckClient() *healthCheckClient { return &healthCheckClient{ diff --git a/pkg/model/gcemodel/api_loadbalancer.go b/pkg/model/gcemodel/api_loadbalancer.go index 5fecf77a1774f..823e138154ea1 100644 --- a/pkg/model/gcemodel/api_loadbalancer.go +++ b/pkg/model/gcemodel/api_loadbalancer.go @@ -40,6 +40,12 @@ var _ fi.CloudupModelBuilder = &APILoadBalancerBuilder{} // createPublicLB validates the existence of a target pool with the given name, // and creates an IP address and forwarding rule pointing to that target pool. func (b *APILoadBalancerBuilder) createPublicLB(c *fi.CloudupModelBuilderContext) error { + region := "" + for _, subnet := range b.Cluster.Spec.Networking.Subnets { + if subnet.Region != "" { + region = subnet.Region + } + } healthCheck := &gcetasks.HTTPHealthcheck{ Name: s(b.NameForHealthcheck("api")), Port: i64(wellknownports.KubeAPIServerHealthCheck), @@ -68,6 +74,7 @@ func (b *APILoadBalancerBuilder) createPublicLB(c *fi.CloudupModelBuilderContext Name: s(b.NameForIPAddress("api")), Lifecycle: b.Lifecycle, + Region: region, WellKnownServices: []wellknownservices.WellKnownService{wellknownservices.KubeAPIServer}, } c.AddTask(ipAddress) @@ -80,6 +87,7 @@ func (b *APILoadBalancerBuilder) createPublicLB(c *fi.CloudupModelBuilderContext PortRange: s(strconv.Itoa(wellknownports.KubeAPIServer) + "-" + strconv.Itoa(wellknownports.KubeAPIServer)), TargetPool: targetPool, IPAddress: ipAddress, + Region: region, IPProtocol: "TCP", LoadBalancingScheme: s("EXTERNAL"), Labels: map[string]string{ @@ -97,6 +105,7 @@ func (b *APILoadBalancerBuilder) createPublicLB(c *fi.CloudupModelBuilderContext TargetPool: targetPool, IPAddress: ipAddress, IPProtocol: "TCP", + Region: region, LoadBalancingScheme: s("EXTERNAL"), Labels: map[string]string{ clusterLabel.Key: clusterLabel.Value, @@ -155,10 +164,20 @@ func (b *APILoadBalancerBuilder) addFirewallRules(c *fi.CloudupModelBuilderConte func (b *APILoadBalancerBuilder) createInternalLB(c *fi.CloudupModelBuilderContext) error { clusterLabel := gce.LabelForCluster(b.ClusterName()) + // internal Loadbalancers are always regional + region := "" + for _, subnet := range b.Cluster.Spec.Networking.Subnets { + if subnet.Region != "" { + region = subnet.Region + } + } + hc := &gcetasks.HealthCheck{ - Name: s(b.NameForHealthCheck("api")), - Port: wellknownports.KubeAPIServer, - Lifecycle: b.Lifecycle, + Name: s(b.NameForHealthCheck("api")), + Port: i64(wellknownports.KubeAPIServer), + RequestPath: s("/healthz"), + Region: region, + Lifecycle: b.Lifecycle, } c.AddTask(hc) var igms []*gcetasks.InstanceGroupManager @@ -181,6 +200,7 @@ func (b *APILoadBalancerBuilder) createInternalLB(c *fi.CloudupModelBuilderConte HealthChecks: []*gcetasks.HealthCheck{hc}, Lifecycle: b.Lifecycle, LoadBalancingScheme: s("INTERNAL"), + Region: region, InstanceGroupManagers: igms, } c.AddTask(bs) @@ -209,6 +229,7 @@ func (b *APILoadBalancerBuilder) createInternalLB(c *fi.CloudupModelBuilderConte Subnetwork: subnet, WellKnownServices: []wellknownservices.WellKnownService{wellknownservices.KubeAPIServer}, + Region: region, Lifecycle: b.Lifecycle, } c.AddTask(ipAddress) @@ -222,6 +243,7 @@ func (b *APILoadBalancerBuilder) createInternalLB(c *fi.CloudupModelBuilderConte IPProtocol: "TCP", LoadBalancingScheme: s("INTERNAL"), Network: network, + Region: region, Subnetwork: subnet, Labels: map[string]string{ clusterLabel.Key: clusterLabel.Value, @@ -240,6 +262,7 @@ func (b *APILoadBalancerBuilder) createInternalLB(c *fi.CloudupModelBuilderConte IPProtocol: "TCP", LoadBalancingScheme: s("INTERNAL"), Network: network, + Region: region, Subnetwork: subnet, Labels: map[string]string{ clusterLabel.Key: clusterLabel.Value, diff --git a/pkg/resources/gce/gce.go b/pkg/resources/gce/gce.go index 5d3d5c9296bdc..a96534ff81c71 100644 --- a/pkg/resources/gce/gce.go +++ b/pkg/resources/gce/gce.go @@ -511,7 +511,11 @@ func deleteForwardingRule(cloud fi.Cloud, r *resources.Resource) error { return err } - op, err := c.Compute().ForwardingRules().Delete(ctx, u.Project, u.Region, u.Name) + region := "" + if !u.Global { + region = u.Region + } + op, err := c.Compute().ForwardingRules().Delete(ctx, u.Project, region, u.Name) if err != nil { if gce.IsNotFound(err) { klog.Infof("ForwardingRule not found, assuming deleted: %q", t.SelfLink) @@ -828,9 +832,14 @@ func (d *clusterDiscoveryGCE) listAddresses() ([]*resources.Resource, error) { addrs, err := c.Compute().Addresses().List(ctx, c.Project(), c.Region()) if err != nil { - return nil, fmt.Errorf("error listing Addresses: %v", err) + return nil, fmt.Errorf("error listing regional Addresses: %v", err) + } + globalAddrs, err := c.Compute().Addresses().List(ctx, c.Project(), "") + if err != nil { + return nil, fmt.Errorf("error listing global Addresses: %v", err) } + addrs = append(addrs, globalAddrs...) for _, a := range addrs { if !d.matchesClusterName(a.Name) { klog.V(8).Infof("Skipping Address with name %q", a.Name) @@ -861,8 +870,12 @@ func deleteAddress(cloud fi.Cloud, r *resources.Resource) error { if err != nil { return err } + region := "" + if !u.Global { + region = u.Region + } - op, err := c.Compute().Addresses().Delete(u.Project, u.Region, u.Name) + op, err := c.Compute().Addresses().Delete(u.Project, region, u.Name) if err != nil { if gce.IsNotFound(err) { klog.Infof("Address not found, assuming deleted: %q", t.SelfLink) @@ -1079,15 +1092,26 @@ func containsOnlyListedIGMs(svc *compute.BackendService, igms []*resources.Resou func (d *clusterDiscoveryGCE) listBackendServices() ([]*resources.Resource, error) { c := d.gceCloud + // list global backendservices first + svcs, err := c.Compute().BackendServices().List(context.Background(), c.Project(), "") + if err != nil { + if gce.IsNotFound(err) { + klog.Infof("backend services not found, assuming none exist in project: %q region: %q", c.Project(), c.Region()) + return nil, nil + } + return nil, fmt.Errorf("failed to list global backend services: %w", err) + } - svcs, err := c.Compute().RegionBackendServices().List(context.Background(), c.Project(), c.Region()) + // list regional backendservices as well + regionalsvcs, err := c.Compute().BackendServices().List(context.Background(), c.Project(), c.Region()) if err != nil { if gce.IsNotFound(err) { klog.Infof("backend services not found, assuming none exist in project: %q region: %q", c.Project(), c.Region()) return nil, nil } - return nil, fmt.Errorf("Failed to list backend services: %w", err) + return nil, fmt.Errorf("failed to list regional backend services: %w", err) } + svcs = append(svcs, regionalsvcs...) // TODO: cache, for efficiency, if needed. // Find all relevant backend services by finding all the cluster's IGMs, and then // listing all backend services in the project / region, then selecting @@ -1104,7 +1128,15 @@ func (d *clusterDiscoveryGCE) listBackendServices() ([]*resources.Resource, erro ID: svc.Name, Type: typeBackendService, Deleter: func(cloud fi.Cloud, r *resources.Resource) error { - op, err := c.Compute().RegionBackendServices().Delete(c.Project(), c.Region(), svc.Name) + u, err := gce.ParseGoogleCloudURL(svc.SelfLink) + if err != nil { + return err + } + region := "" + if !u.Global { + region = u.Region + } + op, err := c.Compute().BackendServices().Delete(c.Project(), region, svc.Name) if err != nil { return err } @@ -1139,12 +1171,20 @@ func (d *clusterDiscoveryGCE) listHealthchecks() ([]*resources.Resource, error) } var hcResources []*resources.Resource for hc := range hcs { + u, err := gce.ParseGoogleCloudURL(hc) + if err != nil { + return nil, err + } + region := "" + if !u.Global { + region = u.Region + } hcResources = append(hcResources, &resources.Resource{ - Name: gce.LastComponent(hc), - ID: gce.LastComponent(hc), + Name: u.Name, + ID: u.Name, Type: typeHealthcheck, Deleter: func(cloud fi.Cloud, r *resources.Resource) error { - op, err := c.Compute().RegionHealthChecks().Delete(c.Project(), c.Region(), gce.LastComponent(hc)) + op, err := c.Compute().HealthChecks().Delete(u.Project, region, u.Name) if err != nil { return err } diff --git a/tests/integration/update_cluster/minimal_gce_dns-none/kubernetes.tf b/tests/integration/update_cluster/minimal_gce_dns-none/kubernetes.tf index cc3d4c8dd06f7..83085168f584c 100644 --- a/tests/integration/update_cluster/minimal_gce_dns-none/kubernetes.tf +++ b/tests/integration/update_cluster/minimal_gce_dns-none/kubernetes.tf @@ -174,19 +174,10 @@ resource "google_compute_address" "api-us-test1-minimal-gce-example-com" { address_type = "INTERNAL" name = "api-us-test1-minimal-gce-example-com" purpose = "SHARED_LOADBALANCER_VIP" + region = "us-test1" subnetwork = google_compute_subnetwork.us-test1-minimal-gce-example-com.name } -resource "google_compute_backend_service" "api-minimal-gce-example-com" { - backend { - group = google_compute_instance_group_manager.a-master-us-test1-a-minimal-gce-example-com.instance_group - } - health_checks = [google_compute_health_check.api-minimal-gce-example-com.id] - load_balancing_scheme = "INTERNAL_SELF_MANAGED" - name = "api-minimal-gce-example-com" - protocol = "TCP" -} - resource "google_compute_disk" "a-etcd-events-minimal-gce-example-com" { labels = { "k8s-io-cluster-name" = "minimal-gce-example-com" @@ -449,7 +440,7 @@ resource "google_compute_firewall" "ssh-external-to-node-minimal-gce-example-com } resource "google_compute_forwarding_rule" "api-us-test1-minimal-gce-example-com" { - backend_service = google_compute_backend_service.api-minimal-gce-example-com.id + backend_service = google_compute_region_backend_service.api-minimal-gce-example-com.id ip_address = google_compute_address.api-us-test1-minimal-gce-example-com.address ip_protocol = "TCP" labels = { @@ -460,11 +451,12 @@ resource "google_compute_forwarding_rule" "api-us-test1-minimal-gce-example-com" name = "api-us-test1-minimal-gce-example-com" network = google_compute_network.minimal-gce-example-com.name ports = ["443"] + region = "us-test1" subnetwork = google_compute_subnetwork.us-test1-minimal-gce-example-com.name } resource "google_compute_forwarding_rule" "kops-controller-us-test1-minimal-gce-example-com" { - backend_service = google_compute_backend_service.api-minimal-gce-example-com.id + backend_service = google_compute_region_backend_service.api-minimal-gce-example-com.id ip_address = google_compute_address.api-us-test1-minimal-gce-example-com.address ip_protocol = "TCP" labels = { @@ -475,16 +467,10 @@ resource "google_compute_forwarding_rule" "kops-controller-us-test1-minimal-gce- name = "kops-controller-us-test1-minimal-gce-example-com" network = google_compute_network.minimal-gce-example-com.name ports = ["3988"] + region = "us-test1" subnetwork = google_compute_subnetwork.us-test1-minimal-gce-example-com.name } -resource "google_compute_health_check" "api-minimal-gce-example-com" { - name = "api-minimal-gce-example-com" - tcp_health_check { - port = 443 - } -} - resource "google_compute_instance_group_manager" "a-master-us-test1-a-minimal-gce-example-com" { base_instance_name = "master-us-test1-a" list_managed_instances_results = "PAGINATED" @@ -612,6 +598,26 @@ resource "google_compute_network" "minimal-gce-example-com" { name = "minimal-gce-example-com" } +resource "google_compute_region_backend_service" "api-minimal-gce-example-com" { + backend { + group = google_compute_instance_group_manager.a-master-us-test1-a-minimal-gce-example-com.instance_group + } + health_checks = [google_compute_region_health_check.api-minimal-gce-example-com.id] + load_balancing_scheme = "INTERNAL_SELF_MANAGED" + name = "api-minimal-gce-example-com" + protocol = "TCP" + region = "us-test1" +} + +resource "google_compute_region_health_check" "api-minimal-gce-example-com" { + http_health_check { + port = 443 + request_path = "/healthz" + } + name = "api-minimal-gce-example-com" + region = "us-test1" +} + resource "google_compute_router" "nat-minimal-gce-example-com" { name = "nat-minimal-gce-example-com" network = google_compute_network.minimal-gce-example-com.name diff --git a/tests/integration/update_cluster/minimal_gce_ilb/kubernetes.tf b/tests/integration/update_cluster/minimal_gce_ilb/kubernetes.tf index ae3d43a0e61a9..378140bb7e22b 100644 --- a/tests/integration/update_cluster/minimal_gce_ilb/kubernetes.tf +++ b/tests/integration/update_cluster/minimal_gce_ilb/kubernetes.tf @@ -182,19 +182,10 @@ resource "google_compute_address" "api-us-test1-minimal-gce-ilb-example-com" { address_type = "INTERNAL" name = "api-us-test1-minimal-gce-ilb-example-com" purpose = "SHARED_LOADBALANCER_VIP" + region = "us-test1" subnetwork = google_compute_subnetwork.us-test1-minimal-gce-ilb-example-com.name } -resource "google_compute_backend_service" "api-minimal-gce-ilb-example-com" { - backend { - group = google_compute_instance_group_manager.a-master-us-test1-a-minimal-gce-ilb-example-com.instance_group - } - health_checks = [google_compute_health_check.api-minimal-gce-ilb-example-com.id] - load_balancing_scheme = "INTERNAL_SELF_MANAGED" - name = "api-minimal-gce-ilb-example-com" - protocol = "TCP" -} - resource "google_compute_disk" "a-etcd-events-minimal-gce-ilb-example-com" { labels = { "k8s-io-cluster-name" = "minimal-gce-ilb-example-com" @@ -433,7 +424,7 @@ resource "google_compute_firewall" "ssh-external-to-node-minimal-gce-ilb-example } resource "google_compute_forwarding_rule" "api-us-test1-minimal-gce-ilb-example-com" { - backend_service = google_compute_backend_service.api-minimal-gce-ilb-example-com.id + backend_service = google_compute_region_backend_service.api-minimal-gce-ilb-example-com.id ip_address = google_compute_address.api-us-test1-minimal-gce-ilb-example-com.address ip_protocol = "TCP" labels = { @@ -444,16 +435,10 @@ resource "google_compute_forwarding_rule" "api-us-test1-minimal-gce-ilb-example- name = "api-us-test1-minimal-gce-ilb-example-com" network = google_compute_network.minimal-gce-ilb-example-com.name ports = ["443"] + region = "us-test1" subnetwork = google_compute_subnetwork.us-test1-minimal-gce-ilb-example-com.name } -resource "google_compute_health_check" "api-minimal-gce-ilb-example-com" { - name = "api-minimal-gce-ilb-example-com" - tcp_health_check { - port = 443 - } -} - resource "google_compute_instance_group_manager" "a-master-us-test1-a-minimal-gce-ilb-example-com" { base_instance_name = "master-us-test1-a" list_managed_instances_results = "PAGINATED" @@ -581,6 +566,26 @@ resource "google_compute_network" "minimal-gce-ilb-example-com" { name = "minimal-gce-ilb-example-com" } +resource "google_compute_region_backend_service" "api-minimal-gce-ilb-example-com" { + backend { + group = google_compute_instance_group_manager.a-master-us-test1-a-minimal-gce-ilb-example-com.instance_group + } + health_checks = [google_compute_region_health_check.api-minimal-gce-ilb-example-com.id] + load_balancing_scheme = "INTERNAL_SELF_MANAGED" + name = "api-minimal-gce-ilb-example-com" + protocol = "TCP" + region = "us-test1" +} + +resource "google_compute_region_health_check" "api-minimal-gce-ilb-example-com" { + http_health_check { + port = 443 + request_path = "/healthz" + } + name = "api-minimal-gce-ilb-example-com" + region = "us-test1" +} + resource "google_compute_router" "nat-minimal-gce-ilb-example-com" { name = "nat-minimal-gce-ilb-example-com" network = google_compute_network.minimal-gce-ilb-example-com.name diff --git a/tests/integration/update_cluster/minimal_gce_ilb_longclustername/kubernetes.tf b/tests/integration/update_cluster/minimal_gce_ilb_longclustername/kubernetes.tf index 45b568d9be9d4..3046ba7e1e824 100644 --- a/tests/integration/update_cluster/minimal_gce_ilb_longclustername/kubernetes.tf +++ b/tests/integration/update_cluster/minimal_gce_ilb_longclustername/kubernetes.tf @@ -182,19 +182,10 @@ resource "google_compute_address" "api-us-test1-minimal-gce-with-a-very-very-ver address_type = "INTERNAL" name = "api-us-test1-minimal-gce-with-a-very-very-very-very-very-96dqvi" purpose = "SHARED_LOADBALANCER_VIP" + region = "us-test1" subnetwork = google_compute_subnetwork.us-test1-minimal-gce-with-a-very-very-very-very-very-lon-96dqvi.name } -resource "google_compute_backend_service" "api-minimal-gce-with-a-very-very-very-very-very-long-name-example-com" { - backend { - group = google_compute_instance_group_manager.a-master-us-test1-a-minimal-gce-with-a-very-very-very-ve-j0fh8f.instance_group - } - health_checks = [google_compute_health_check.api-minimal-gce-with-a-very-very-very-very-very-long-name-example-com.id] - load_balancing_scheme = "INTERNAL_SELF_MANAGED" - name = "api-minimal-gce-with-a-very-very-very-very-very-long-name-example-com" - protocol = "TCP" -} - resource "google_compute_disk" "a-etcd-events-minimal-gce-with-a-very-very-very-very-ver-96dqvi" { labels = { "k8s-io-cluster-name" = "minimal-gce-with-a-very-very-very-very-very-long-name-example-com" @@ -433,7 +424,7 @@ resource "google_compute_firewall" "ssh-external-to-node-minimal-gce-with-a-very } resource "google_compute_forwarding_rule" "api-us-test1-minimal-gce-with-a-very-very-very-very-very-96dqvi" { - backend_service = google_compute_backend_service.api-minimal-gce-with-a-very-very-very-very-very-long-name-example-com.id + backend_service = google_compute_region_backend_service.api-minimal-gce-with-a-very-very-very-very-very-long-name-example-com.id ip_address = google_compute_address.api-us-test1-minimal-gce-with-a-very-very-very-very-very-96dqvi.address ip_protocol = "TCP" labels = { @@ -444,16 +435,10 @@ resource "google_compute_forwarding_rule" "api-us-test1-minimal-gce-with-a-very- name = "api-us-test1-minimal-gce-with-a-very-very-very-very-very-96dqvi" network = google_compute_network.minimal-gce-with-a-very-very-very-very-very-long-name-ex-96dqvi.name ports = ["443"] + region = "us-test1" subnetwork = google_compute_subnetwork.us-test1-minimal-gce-with-a-very-very-very-very-very-lon-96dqvi.name } -resource "google_compute_health_check" "api-minimal-gce-with-a-very-very-very-very-very-long-name-example-com" { - name = "api-minimal-gce-with-a-very-very-very-very-very-long-name-example-com" - tcp_health_check { - port = 443 - } -} - resource "google_compute_instance_group_manager" "a-master-us-test1-a-minimal-gce-with-a-very-very-very-ve-j0fh8f" { base_instance_name = "master-us-test1-a" list_managed_instances_results = "PAGINATED" @@ -581,6 +566,26 @@ resource "google_compute_network" "minimal-gce-with-a-very-very-very-very-very-l name = "minimal-gce-with-a-very-very-very-very-very-long-name-ex-96dqvi" } +resource "google_compute_region_backend_service" "api-minimal-gce-with-a-very-very-very-very-very-long-name-example-com" { + backend { + group = google_compute_instance_group_manager.a-master-us-test1-a-minimal-gce-with-a-very-very-very-ve-j0fh8f.instance_group + } + health_checks = [google_compute_region_health_check.api-minimal-gce-with-a-very-very-very-very-very-long-name-example-com.id] + load_balancing_scheme = "INTERNAL_SELF_MANAGED" + name = "api-minimal-gce-with-a-very-very-very-very-very-long-name-example-com" + protocol = "TCP" + region = "us-test1" +} + +resource "google_compute_region_health_check" "api-minimal-gce-with-a-very-very-very-very-very-long-name-example-com" { + http_health_check { + port = 443 + request_path = "/healthz" + } + name = "api-minimal-gce-with-a-very-very-very-very-very-long-name-example-com" + region = "us-test1" +} + resource "google_compute_router" "nat-minimal-gce-with-a-very-very-very-very-very-long-nam-96dqvi" { name = "nat-minimal-gce-with-a-very-very-very-very-very-long-nam-96dqvi" network = google_compute_network.minimal-gce-with-a-very-very-very-very-very-long-name-ex-96dqvi.name diff --git a/tests/integration/update_cluster/minimal_gce_plb/kubernetes.tf b/tests/integration/update_cluster/minimal_gce_plb/kubernetes.tf index 890532af7d0f3..bac0fc4c82ce3 100644 --- a/tests/integration/update_cluster/minimal_gce_plb/kubernetes.tf +++ b/tests/integration/update_cluster/minimal_gce_plb/kubernetes.tf @@ -179,7 +179,8 @@ resource "aws_s3_object" "nodeupconfig-nodes" { } resource "google_compute_address" "api-minimal-gce-plb-example-com" { - name = "api-minimal-gce-plb-example-com" + name = "api-minimal-gce-plb-example-com" + region = "us-test1" } resource "google_compute_disk" "a-etcd-events-minimal-gce-plb-example-com" { @@ -429,6 +430,7 @@ resource "google_compute_forwarding_rule" "api-minimal-gce-plb-example-com" { load_balancing_scheme = "EXTERNAL" name = "api-minimal-gce-plb-example-com" port_range = "443-443" + region = "us-test1" target = google_compute_target_pool.api-minimal-gce-plb-example-com.self_link } diff --git a/upup/pkg/fi/cloudup/gce/compute.go b/upup/pkg/fi/cloudup/gce/compute.go index bde7c2d757ccc..555aa0961a0e1 100644 --- a/upup/pkg/fi/cloudup/gce/compute.go +++ b/upup/pkg/fi/cloudup/gce/compute.go @@ -32,7 +32,7 @@ type ComputeClient interface { Routes() RouteClient ForwardingRules() ForwardingRuleClient HTTPHealthChecks() HttpHealthChecksClient - RegionHealthChecks() RegionHealthChecksClient + HealthChecks() HealthChecksClient Addresses() AddressClient Firewalls() FirewallClient Routers() RouterClient @@ -41,7 +41,7 @@ type ComputeClient interface { InstanceGroupManagers() InstanceGroupManagerClient TargetPools() TargetPoolClient Disks() DiskClient - RegionBackendServices() RegionBackendServiceClient + BackendServices() BackendServiceClient } type computeClientImpl struct { @@ -98,13 +98,15 @@ func (c *computeClientImpl) Routes() RouteClient { func (c *computeClientImpl) ForwardingRules() ForwardingRuleClient { return &forwardingRuleClientImpl{ - srv: c.srv.ForwardingRules, + srv: c.srv.ForwardingRules, + gsrv: c.srv.GlobalForwardingRules, } } -func (c *computeClientImpl) RegionBackendServices() RegionBackendServiceClient { - return ®ionBackendServiceClientImpl{ - srv: c.srv.RegionBackendServices, +func (c *computeClientImpl) BackendServices() BackendServiceClient { + return &backendServiceClientImpl{ + srv: c.srv.BackendServices, + rsrv: c.srv.RegionBackendServices, } } @@ -114,15 +116,17 @@ func (c *computeClientImpl) HTTPHealthChecks() HttpHealthChecksClient { } } -func (c *computeClientImpl) RegionHealthChecks() RegionHealthChecksClient { +func (c *computeClientImpl) HealthChecks() HealthChecksClient { return &healthCheckClientImpl{ - srv: c.srv.RegionHealthChecks, + srv: c.srv.HealthChecks, + rsrv: c.srv.RegionHealthChecks, } } func (c *computeClientImpl) Addresses() AddressClient { return &addressClientImpl{ - srv: c.srv.Addresses, + srv: c.srv.Addresses, + gsrv: c.srv.GlobalAddresses, } } @@ -297,40 +301,59 @@ func (c *subnetworkClientImpl) List(ctx context.Context, project, region string) } // === -type RegionBackendServiceClient interface { +type BackendServiceClient interface { Insert(project, region string, fr *compute.BackendService) (*compute.Operation, error) Delete(project, region, name string) (*compute.Operation, error) Get(project, region, name string) (*compute.BackendService, error) List(ctx context.Context, project, region string) ([]*compute.BackendService, error) } -type regionBackendServiceClientImpl struct { - srv *compute.RegionBackendServicesService +type backendServiceClientImpl struct { + srv *compute.BackendServicesService + rsrv *compute.RegionBackendServicesService } -var _ RegionBackendServiceClient = ®ionBackendServiceClientImpl{} +var _ BackendServiceClient = &backendServiceClientImpl{} -func (c *regionBackendServiceClientImpl) Insert(project, region string, fr *compute.BackendService) (*compute.Operation, error) { - return c.srv.Insert(project, region, fr).Do() +func (c *backendServiceClientImpl) Insert(project, region string, fr *compute.BackendService) (*compute.Operation, error) { + if region == "" { + return c.srv.Insert(project, fr).Do() + } + return c.rsrv.Insert(project, region, fr).Do() } -func (c *regionBackendServiceClientImpl) Delete(project, region, name string) (*compute.Operation, error) { - return c.srv.Delete(project, region, name).Do() +func (c *backendServiceClientImpl) Delete(project, region, name string) (*compute.Operation, error) { + if region == "" { + return c.srv.Delete(project, name).Do() + } + return c.rsrv.Delete(project, region, name).Do() } -func (c *regionBackendServiceClientImpl) Get(project, region, name string) (*compute.BackendService, error) { - return c.srv.Get(project, region, name).Do() +func (c *backendServiceClientImpl) Get(project, region, name string) (*compute.BackendService, error) { + if region == "" { + return c.srv.Get(project, name).Do() + } + return c.rsrv.Get(project, region, name).Do() } -func (c *regionBackendServiceClientImpl) List(ctx context.Context, project, region string) ([]*compute.BackendService, error) { - var hcs []*compute.BackendService - if err := c.srv.List(project, region).Pages(ctx, func(p *compute.BackendServiceList) error { - hcs = append(hcs, p.Items...) +func (c *backendServiceClientImpl) List(ctx context.Context, project, region string) ([]*compute.BackendService, error) { + var bs []*compute.BackendService + if region == "" { + if err := c.srv.List(project).Pages(ctx, func(p *compute.BackendServiceList) error { + bs = append(bs, p.Items...) + return nil + }); err != nil { + return nil, err + } + return bs, nil + } + if err := c.rsrv.List(project, region).Pages(ctx, func(p *compute.BackendServiceList) error { + bs = append(bs, p.Items...) return nil }); err != nil { return nil, err } - return hcs, nil + return bs, nil } // === @@ -370,29 +393,55 @@ type ForwardingRuleClient interface { } type forwardingRuleClientImpl struct { - srv *compute.ForwardingRulesService + srv *compute.ForwardingRulesService + gsrv *compute.GlobalForwardingRulesService } var _ ForwardingRuleClient = &forwardingRuleClientImpl{} func (c *forwardingRuleClientImpl) Insert(ctx context.Context, project, region string, fr *compute.ForwardingRule) (*compute.Operation, error) { + if region == "" { + return c.gsrv.Insert(project, fr).Context(ctx).Do() + } return c.srv.Insert(project, region, fr).Context(ctx).Do() } func (c *forwardingRuleClientImpl) Delete(ctx context.Context, project, region, name string) (*compute.Operation, error) { + if region == "" { + return c.gsrv.Delete(project, name).Context(ctx).Do() + } return c.srv.Delete(project, region, name).Context(ctx).Do() } func (c *forwardingRuleClientImpl) Get(ctx context.Context, project, region, name string) (*compute.ForwardingRule, error) { - return c.srv.Get(project, region, name).Context(ctx).Do() + if region == "" { + return c.gsrv.Get(project, name).Do() + } + return c.srv.Get(project, region, name).Do() } func (c *forwardingRuleClientImpl) SetLabels(ctx context.Context, project string, region string, resource string, request *compute.RegionSetLabelsRequest) (*compute.Operation, error) { + if region == "" { + grequest := &compute.GlobalSetLabelsRequest{ + LabelFingerprint: request.LabelFingerprint, + Labels: request.Labels, + } + return c.gsrv.SetLabels(project, resource, grequest).Context(ctx).Do() + } return c.srv.SetLabels(project, region, resource, request).Context(ctx).Do() } func (c *forwardingRuleClientImpl) List(ctx context.Context, project, region string) ([]*compute.ForwardingRule, error) { var frs []*compute.ForwardingRule + if region == "" { + if err := c.gsrv.List(project).Pages(ctx, func(p *compute.ForwardingRuleList) error { + frs = append(frs, p.Items...) + return nil + }); err != nil { + return nil, err + } + return frs, nil + } if err := c.srv.List(project, region).Pages(ctx, func(p *compute.ForwardingRuleList) error { frs = append(frs, p.Items...) return nil @@ -402,7 +451,7 @@ func (c *forwardingRuleClientImpl) List(ctx context.Context, project, region str return frs, nil } -type RegionHealthChecksClient interface { +type HealthChecksClient interface { Insert(project, region string, fr *compute.HealthCheck) (*compute.Operation, error) Delete(project, region, name string) (*compute.Operation, error) Get(project, region, name string) (*compute.HealthCheck, error) @@ -410,26 +459,44 @@ type RegionHealthChecksClient interface { } type healthCheckClientImpl struct { - srv *compute.RegionHealthChecksService + srv *compute.HealthChecksService + rsrv *compute.RegionHealthChecksService } -var _ RegionHealthChecksClient = &healthCheckClientImpl{} +var _ HealthChecksClient = &healthCheckClientImpl{} -func (c *healthCheckClientImpl) Insert(project, region string, fr *compute.HealthCheck) (*compute.Operation, error) { - return c.srv.Insert(project, region, fr).Do() +func (c *healthCheckClientImpl) Insert(project, region string, hc *compute.HealthCheck) (*compute.Operation, error) { + if region == "" { + return c.srv.Insert(project, hc).Do() + } + return c.rsrv.Insert(project, region, hc).Do() } func (c *healthCheckClientImpl) Delete(project, region, name string) (*compute.Operation, error) { - return c.srv.Delete(project, region, name).Do() + if region == "" { + return c.srv.Delete(project, name).Do() + } + return c.rsrv.Delete(project, region, name).Do() } func (c *healthCheckClientImpl) Get(project, region, name string) (*compute.HealthCheck, error) { - return c.srv.Get(project, region, name).Do() + if region == "" { + return c.srv.Get(project, name).Do() + } + return c.rsrv.Get(project, region, name).Do() } func (c *healthCheckClientImpl) List(ctx context.Context, project, region string) ([]*compute.HealthCheck, error) { var hcs []*compute.HealthCheck - if err := c.srv.List(project, region).Pages(ctx, func(p *compute.HealthCheckList) error { + if region == "" { + if err := c.srv.List(project).Pages(ctx, func(p *compute.HealthCheckList) error { + hcs = append(hcs, p.Items...) + return nil + }); err != nil { + return nil, err + } + } + if err := c.rsrv.List(project, region).Pages(ctx, func(p *compute.HealthCheckList) error { hcs = append(hcs, p.Items...) return nil }); err != nil { @@ -483,25 +550,44 @@ type AddressClient interface { } type addressClientImpl struct { - srv *compute.AddressesService + srv *compute.AddressesService + gsrv *compute.GlobalAddressesService } var _ AddressClient = &addressClientImpl{} func (c *addressClientImpl) Insert(project, region string, addr *compute.Address) (*compute.Operation, error) { + if region == "" { + return c.gsrv.Insert(project, addr).Do() + } return c.srv.Insert(project, region, addr).Do() } func (c *addressClientImpl) Delete(project, region, name string) (*compute.Operation, error) { + if region == "" { + return c.gsrv.Delete(project, name).Do() + } return c.srv.Delete(project, region, name).Do() } func (c *addressClientImpl) Get(project, region, name string) (*compute.Address, error) { + if region == "" { + return c.gsrv.Get(project, name).Do() + } return c.srv.Get(project, region, name).Do() } func (c *addressClientImpl) List(ctx context.Context, project, region string) ([]*compute.Address, error) { var addrs []*compute.Address + if region == "" { + if err := c.gsrv.List(project).Pages(ctx, func(p *compute.AddressList) error { + addrs = append(addrs, p.Items...) + return nil + }); err != nil { + return nil, err + } + return addrs, nil + } if err := c.srv.List(project, region).Pages(ctx, func(p *compute.AddressList) error { addrs = append(addrs, p.Items...) return nil @@ -512,6 +598,13 @@ func (c *addressClientImpl) List(ctx context.Context, project, region string) ([ } func (c *addressClientImpl) ListWithFilter(project, region, filter string) ([]*compute.Address, error) { + if region == "" { + addrs, err := c.gsrv.List(project).Filter(filter).Do() + if err != nil { + return nil, err + } + return addrs.Items, nil + } addrs, err := c.srv.List(project, region).Filter(filter).Do() if err != nil { return nil, err diff --git a/upup/pkg/fi/cloudup/gce/gce_cloud.go b/upup/pkg/fi/cloudup/gce/gce_cloud.go index cf013d19fef30..4327564c49538 100644 --- a/upup/pkg/fi/cloudup/gce/gce_cloud.go +++ b/upup/pkg/fi/cloudup/gce/gce_cloud.go @@ -318,6 +318,7 @@ func (c *gceCloudImplementation) GetApiIngressStatus(cluster *kops.Cluster) ([]f var ingresses []fi.ApiIngressStatus klog.V(2).Infof("Querying GCE to find forwardingRules for API") + region := c.region // TODO: tweak this when LoadBalancerClass for GCE is global // These are the ingress rules, so we search for them in the network project. _, project, err := ParseNameAndProjectFromNetworkID(cluster.Spec.Networking.NetworkID) if err != nil { @@ -326,7 +327,7 @@ func (c *gceCloudImplementation) GetApiIngressStatus(cluster *kops.Cluster) ([]f project = c.Project() } - forwardingRules, err := c.compute.ForwardingRules().List(context.Background(), project, c.region) + forwardingRules, err := c.compute.ForwardingRules().List(context.Background(), project, region) if err != nil { if !IsNotFound(err) { forwardingRules = nil diff --git a/upup/pkg/fi/cloudup/gcetasks/address.go b/upup/pkg/fi/cloudup/gcetasks/address.go index 47306800410d2..c759374f7dd0e 100644 --- a/upup/pkg/fi/cloudup/gcetasks/address.go +++ b/upup/pkg/fi/cloudup/gcetasks/address.go @@ -36,6 +36,7 @@ type Address struct { IPAddress *string IPAddressType *string Purpose *string + Region string Subnetwork *Subnet @@ -51,7 +52,7 @@ func (e *Address) CompareWithID() *string { } func (e *Address) Find(c *fi.CloudupContext) (*Address, error) { - actual, err := e.find(c.T.Cloud.(gce.GCECloud)) + actual, err := e.find(c.T.Cloud.(gce.GCECloud), e.Region) if actual != nil && err == nil { if e.IPAddress == nil { e.IPAddress = actual.IPAddress @@ -64,9 +65,9 @@ func (e *Address) Find(c *fi.CloudupContext) (*Address, error) { return actual, err } -func findAddressByIP(cloud gce.GCECloud, ip string) (*Address, error) { +func findAddressByIP(cloud gce.GCECloud, ip, region string) (*Address, error) { // Technically this is a regex, but it doesn't matter... - addrs, err := cloud.Compute().Addresses().ListWithFilter(cloud.Project(), cloud.Region(), "address eq "+ip) + addrs, err := cloud.Compute().Addresses().ListWithFilter(cloud.Project(), region, "address eq "+ip) if err != nil { return nil, fmt.Errorf("error listing IP Addresses: %v", err) } @@ -87,8 +88,8 @@ func findAddressByIP(cloud gce.GCECloud, ip string) (*Address, error) { return actual, nil } -func (e *Address) find(cloud gce.GCECloud) (*Address, error) { - r, err := cloud.Compute().Addresses().Get(cloud.Project(), cloud.Region(), *e.Name) +func (e *Address) find(cloud gce.GCECloud, region string) (*Address, error) { + r, err := cloud.Compute().Addresses().Get(cloud.Project(), region, *e.Name) if err != nil { if gce.IsNotFound(err) { return nil, nil @@ -120,7 +121,7 @@ func (e *Address) GetWellKnownServices() []wellknownservices.WellKnownService { } func (e *Address) FindAddresses(context *fi.CloudupContext) ([]string, error) { - actual, err := e.find(context.T.Cloud.(gce.GCECloud)) + actual, err := e.find(context.T.Cloud.(gce.GCECloud), e.Region) if err != nil { return nil, fmt.Errorf("error querying for IP Address: %v", err) } @@ -153,7 +154,7 @@ func (_ *Address) RenderGCE(t *gce.GCEAPITarget, a, e, changes *Address) error { Address: fi.ValueOf(e.IPAddress), AddressType: fi.ValueOf(e.IPAddressType), Purpose: fi.ValueOf(e.Purpose), - Region: cloud.Region(), + Region: e.Region, } if e.Subnetwork != nil { @@ -163,7 +164,7 @@ func (_ *Address) RenderGCE(t *gce.GCEAPITarget, a, e, changes *Address) error { if a == nil { klog.V(2).Infof("Creating Address: %q", addr.Name) - op, err := cloud.Compute().Addresses().Insert(cloud.Project(), cloud.Region(), addr) + op, err := cloud.Compute().Addresses().Insert(cloud.Project(), e.Region, addr) if err != nil { return fmt.Errorf("error creating IP Address: %v", err) } @@ -183,9 +184,10 @@ type terraformAddress struct { AddressType *string `cty:"address_type"` Purpose *string `cty:"purpose"` Subnetwork *terraformWriter.Literal `cty:"subnetwork"` + Region string `cty:"region"` } -func (_ *Address) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *Address) error { +func (*Address) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *Address) error { tf := &terraformAddress{ Name: e.Name, AddressType: e.IPAddressType, @@ -194,11 +196,17 @@ func (_ *Address) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *A if e.Subnetwork != nil { tf.Subnetwork = e.Subnetwork.TerraformLink() } + if e.Region == "" { + return t.RenderResource("google_compute_global_address", *e.Name, tf) + } + tf.Region = e.Region return t.RenderResource("google_compute_address", *e.Name, tf) } func (e *Address) TerraformAddress() *terraformWriter.Literal { name := fi.ValueOf(e.Name) - + if e.Region == "" { + return terraformWriter.LiteralProperty("google_compute_global_address", name, "address") + } return terraformWriter.LiteralProperty("google_compute_address", name, "address") } diff --git a/upup/pkg/fi/cloudup/gcetasks/backend_service.go b/upup/pkg/fi/cloudup/gcetasks/backend_service.go index afeee3e990ccb..c9e8fac3b0ea1 100644 --- a/upup/pkg/fi/cloudup/gcetasks/backend_service.go +++ b/upup/pkg/fi/cloudup/gcetasks/backend_service.go @@ -35,6 +35,7 @@ type BackendService struct { HealthChecks []*HealthCheck LoadBalancingScheme *string Protocol *string + Region string InstanceGroupManagers []*InstanceGroupManager Lifecycle fi.Lifecycle @@ -48,7 +49,7 @@ func (e *BackendService) CompareWithID() *string { } func (e *BackendService) Find(c *fi.CloudupContext) (*BackendService, error) { - actual, err := e.find(c.T.Cloud.(gce.GCECloud)) + actual, err := e.find(c.T.Cloud.(gce.GCECloud), e.Region) if actual != nil && err == nil { // Ignore system fields actual.Lifecycle = e.Lifecycle @@ -57,8 +58,8 @@ func (e *BackendService) Find(c *fi.CloudupContext) (*BackendService, error) { return actual, err } -func (e *BackendService) find(cloud gce.GCECloud) (*BackendService, error) { - r, err := cloud.Compute().RegionBackendServices().Get(cloud.Project(), cloud.Region(), *e.Name) +func (e *BackendService) find(cloud gce.GCECloud, region string) (*BackendService, error) { + r, err := cloud.Compute().BackendServices().Get(cloud.Project(), e.Region, *e.Name) if err != nil { if gce.IsNotFound(err) { return nil, nil @@ -107,7 +108,7 @@ func (_ *BackendService) RenderGCE(t *gce.GCEAPITarget, a, e, changes *BackendSe cloud := t.Cloud var hcs []string for _, hc := range e.HealthChecks { - hcs = append(hcs, hc.URL(cloud)) + hcs = append(hcs, hc.URL(cloud, e.Region)) } var backends []*compute.Backend for _, igm := range e.InstanceGroupManagers { @@ -121,12 +122,13 @@ func (_ *BackendService) RenderGCE(t *gce.GCEAPITarget, a, e, changes *BackendSe HealthChecks: hcs, LoadBalancingScheme: *e.LoadBalancingScheme, Backends: backends, + PortName: "http", } if a == nil { klog.V(2).Infof("Creating BackendService: %q", bs.Name) - op, err := cloud.Compute().RegionBackendServices().Insert(cloud.Project(), cloud.Region(), bs) + op, err := cloud.Compute().BackendServices().Insert(cloud.Project(), e.Region, bs) if err != nil { return fmt.Errorf("error creating backend service: %v", err) } @@ -141,7 +143,12 @@ func (_ *BackendService) RenderGCE(t *gce.GCEAPITarget, a, e, changes *BackendSe return nil } -func (a *BackendService) URL(cloud gce.GCECloud) string { +func (a *BackendService) URL(cloud gce.GCECloud, region string) string { + if region == "" { + return fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/global/backendServices/%s", + cloud.Project(), + *a.Name) + } return fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/regions/%s/backendServices/%s", cloud.Project(), cloud.Region(), @@ -158,6 +165,7 @@ type terraformBackendService struct { LoadBalancingScheme *string `cty:"load_balancing_scheme"` Protocol *string `cty:"protocol"` Backend []terraformBackend `cty:"backend"` + Region string `cty:"region"` } func (_ *BackendService) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *BackendService) error { @@ -181,15 +189,25 @@ func (_ *BackendService) RenderTerraform(t *terraform.TerraformTarget, a, e, cha var hcs []*terraformWriter.Literal for _, hc := range e.HealthChecks { - hcs = append(hcs, terraformWriter.LiteralProperty("google_compute_health_check", *hc.Name, "id")) + if e.Region == "" { + hcs = append(hcs, terraformWriter.LiteralProperty("google_compute_health_check", *hc.Name, "id")) + + } else { + hcs = append(hcs, terraformWriter.LiteralProperty("google_compute_region_health_check", *hc.Name, "id")) + } } tf.HealthChecks = hcs - - return t.RenderResource("google_compute_backend_service", *e.Name, tf) + if e.Region == "" { + return t.RenderResource("google_compute_backend_service", *e.Name, tf) + } + tf.Region = e.Region + return t.RenderResource("google_compute_region_backend_service", *e.Name, tf) } func (e *BackendService) TerraformAddress() *terraformWriter.Literal { name := fi.ValueOf(e.Name) - - return terraformWriter.LiteralProperty("google_compute_backend_service", name, "id") + if e.Region == "" { + return terraformWriter.LiteralProperty("google_compute_backend_service", name, "id") + } + return terraformWriter.LiteralProperty("google_compute_region_backend_service", name, "id") } diff --git a/upup/pkg/fi/cloudup/gcetasks/forwardingrule.go b/upup/pkg/fi/cloudup/gcetasks/forwardingrule.go index dbc79b44c257a..dbfba4b693ffe 100644 --- a/upup/pkg/fi/cloudup/gcetasks/forwardingrule.go +++ b/upup/pkg/fi/cloudup/gcetasks/forwardingrule.go @@ -49,6 +49,7 @@ type ForwardingRule struct { Network *Network Subnetwork *Subnet BackendService *BackendService + Region string // Labels to set on the resource. Labels map[string]string @@ -70,7 +71,7 @@ func (e *ForwardingRule) Find(c *fi.CloudupContext) (*ForwardingRule, error) { cloud := c.T.Cloud.(gce.GCECloud) name := fi.ValueOf(e.Name) - r, err := cloud.Compute().ForwardingRules().Get(ctx, cloud.Project(), cloud.Region(), name) + r, err := cloud.Compute().ForwardingRules().Get(ctx, cloud.Project(), e.Region, name) if err != nil { if gce.IsNotFound(err) { return nil, nil @@ -95,7 +96,7 @@ func (e *ForwardingRule) Find(c *fi.CloudupContext) (*ForwardingRule, error) { } } if r.IPAddress != "" { - address, err := findAddressByIP(cloud, r.IPAddress) + address, err := findAddressByIP(cloud, r.IPAddress, e.Region) if err != nil { return nil, fmt.Errorf("error finding Address with IP=%q: %v", r.IPAddress, err) } @@ -133,14 +134,14 @@ func (e *ForwardingRule) Run(c *fi.CloudupContext) error { return fi.CloudupDefaultDeltaRunMethod(e, c) } -func (_ *ForwardingRule) CheckChanges(a, e, changes *ForwardingRule) error { +func (*ForwardingRule) CheckChanges(a, e, changes *ForwardingRule) error { if fi.ValueOf(e.Name) == "" { return fi.RequiredField("Name") } return nil } -func (_ *ForwardingRule) RenderGCE(t *gce.GCEAPITarget, a, e, changes *ForwardingRule) error { +func (*ForwardingRule) RenderGCE(t *gce.GCEAPITarget, a, e, changes *ForwardingRule) error { ctx := context.TODO() name := fi.ValueOf(e.Name) @@ -166,15 +167,15 @@ func (_ *ForwardingRule) RenderGCE(t *gce.GCEAPITarget, a, e, changes *Forwardin if e.BackendService != nil { if o.Target != "" { - return fmt.Errorf("cannot specify both %q and %q for forwarding rule target.", o.Target, e.BackendService) + return fmt.Errorf("cannot specify both %q and %q for forwarding rule target", o.Target, e.BackendService) } - o.BackendService = e.BackendService.URL(t.Cloud) + o.BackendService = e.BackendService.URL(t.Cloud, e.Region) } if e.IPAddress != nil { o.IPAddress = fi.ValueOf(e.IPAddress.IPAddress) if o.IPAddress == "" { - addr, err := e.IPAddress.find(t.Cloud) + addr, err := e.IPAddress.find(t.Cloud, e.Region) if err != nil { return fmt.Errorf("error finding Address %q: %v", e.IPAddress, err) } @@ -189,7 +190,7 @@ func (_ *ForwardingRule) RenderGCE(t *gce.GCEAPITarget, a, e, changes *Forwardin } } if o.IPAddress != "" && e.RuleIPAddress != nil { - return fmt.Errorf("Specified both IP Address and rule-managed IP address: %v, %v", e.IPAddress, *e.RuleIPAddress) + return fmt.Errorf("specified both IP Address and rule-managed IP address: %v, %v", e.IPAddress, *e.RuleIPAddress) } if e.RuleIPAddress != nil { o.IPAddress = *e.RuleIPAddress @@ -214,7 +215,7 @@ func (_ *ForwardingRule) RenderGCE(t *gce.GCEAPITarget, a, e, changes *Forwardin if a == nil { klog.V(4).Infof("Creating ForwardingRule %q", o.Name) - op, err := t.Cloud.Compute().ForwardingRules().Insert(ctx, t.Cloud.Project(), t.Cloud.Region(), o) + op, err := t.Cloud.Compute().ForwardingRules().Insert(ctx, t.Cloud.Project(), e.Region, o) if err != nil { return fmt.Errorf("error creating ForwardingRule %q: %v", o.Name, err) } @@ -225,7 +226,7 @@ func (_ *ForwardingRule) RenderGCE(t *gce.GCEAPITarget, a, e, changes *Forwardin if e.Labels != nil { // We can't set labels on creation; we have to read the object to get the fingerprint - r, err := t.Cloud.Compute().ForwardingRules().Get(ctx, t.Cloud.Project(), t.Cloud.Region(), name) + r, err := t.Cloud.Compute().ForwardingRules().Get(ctx, t.Cloud.Project(), e.Region, name) if err != nil { return fmt.Errorf("reading created ForwardingRule %q: %v", name, err) } @@ -234,7 +235,7 @@ func (_ *ForwardingRule) RenderGCE(t *gce.GCEAPITarget, a, e, changes *Forwardin LabelFingerprint: r.LabelFingerprint, Labels: e.Labels, } - op, err := t.Cloud.Compute().ForwardingRules().SetLabels(ctx, t.Cloud.Project(), t.Cloud.Region(), o.Name, &req) + op, err := t.Cloud.Compute().ForwardingRules().SetLabels(ctx, t.Cloud.Project(), e.Region, o.Name, &req) if err != nil { return fmt.Errorf("setting ForwardingRule labels: %w", err) } @@ -249,7 +250,7 @@ func (_ *ForwardingRule) RenderGCE(t *gce.GCEAPITarget, a, e, changes *Forwardin LabelFingerprint: a.labelFingerprint, Labels: e.Labels, } - op, err := t.Cloud.Compute().ForwardingRules().SetLabels(ctx, t.Cloud.Project(), t.Cloud.Region(), o.Name, &req) + op, err := t.Cloud.Compute().ForwardingRules().SetLabels(ctx, t.Cloud.Project(), e.Region, o.Name, &req) if err != nil { return fmt.Errorf("setting ForwardingRule labels: %w", err) } @@ -281,6 +282,7 @@ type terraformForwardingRule struct { Subnetwork *terraformWriter.Literal `cty:"subnetwork"` BackendService *terraformWriter.Literal `cty:"backend_service"` Labels map[string]string `cty:"labels"` + Region string `cty:"region"` } func (_ *ForwardingRule) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *ForwardingRule) error { @@ -293,8 +295,12 @@ func (_ *ForwardingRule) RenderTerraform(t *terraform.TerraformTarget, a, e, cha Ports: e.Ports, PortRange: e.PortRange, Labels: e.Labels, + Region: e.Region, } + if e.Region != "" { + tf.Region = e.Region + } if e.TargetPool != nil { tf.Target = e.TargetPool.TerraformLink() } diff --git a/upup/pkg/fi/cloudup/gcetasks/healthcheck.go b/upup/pkg/fi/cloudup/gcetasks/healthcheck.go index f6bdd7a932bec..8f1f9458eade8 100644 --- a/upup/pkg/fi/cloudup/gcetasks/healthcheck.go +++ b/upup/pkg/fi/cloudup/gcetasks/healthcheck.go @@ -33,9 +33,11 @@ import ( // and HTTPSHealthCheck. Those HCs are still needed for some types, so both // are implemented in kops, but this one should be preferred when possible. type HealthCheck struct { - Name *string - Port int64 - Lifecycle fi.Lifecycle + Name *string + Port *int64 + Lifecycle fi.Lifecycle + Region string + RequestPath *string } var _ fi.CompareWithID = &HealthCheck{} @@ -53,15 +55,21 @@ func (e *HealthCheck) Find(c *fi.CloudupContext) (*HealthCheck, error) { return actual, err } -func (e *HealthCheck) URL(cloud gce.GCECloud) string { +func (e *HealthCheck) URL(cloud gce.GCECloud, region string) string { + if region == "" { + return fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/global/healthChecks/%s", + cloud.Project(), + *e.Name) + } + return fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/regions/%s/healthChecks/%s", cloud.Project(), - cloud.Region(), + e.Region, *e.Name) } func (e *HealthCheck) find(cloud gce.GCECloud) (*HealthCheck, error) { - r, err := cloud.Compute().RegionHealthChecks().Get(cloud.Project(), cloud.Region(), *e.Name) + r, err := cloud.Compute().HealthChecks().Get(cloud.Project(), e.Region, *e.Name) if err != nil { if gce.IsNotFound(err) { return nil, nil @@ -72,9 +80,6 @@ func (e *HealthCheck) find(cloud gce.GCECloud) (*HealthCheck, error) { actual := &HealthCheck{} actual.Name = &r.Name - if r.TcpHealthCheck != nil { - actual.Port = r.TcpHealthCheck.Port - } return actual, nil } @@ -83,7 +88,7 @@ func (e *HealthCheck) Run(c *fi.CloudupContext) error { return fi.CloudupDefaultDeltaRunMethod(e, c) } -func (_ *HealthCheck) CheckChanges(a, e, changes *HealthCheck) error { +func (*HealthCheck) CheckChanges(a, e, changes *HealthCheck) error { if a != nil { if changes.Name != nil { return fi.CannotChangeField("Name") @@ -95,22 +100,21 @@ func (_ *HealthCheck) CheckChanges(a, e, changes *HealthCheck) error { return nil } -func (_ *HealthCheck) RenderGCE(t *gce.GCEAPITarget, a, e, changes *HealthCheck) error { +func (*HealthCheck) RenderGCE(t *gce.GCEAPITarget, a, e, changes *HealthCheck) error { cloud := t.Cloud hc := &compute.HealthCheck{ Name: *e.Name, - TcpHealthCheck: &compute.TCPHealthCheck{ - Port: e.Port, + HttpHealthCheck: &compute.HTTPHealthCheck{ + Port: fi.ValueOf(e.Port), + RequestPath: fi.ValueOf(e.RequestPath), }, - Type: "TCP", - - Region: cloud.Region(), + Type: "HTTP", } if a == nil { klog.V(2).Infof("Creating HealthCheck %q", hc.Name) - op, err := cloud.Compute().RegionHealthChecks().Insert(cloud.Project(), cloud.Region(), hc) + op, err := cloud.Compute().HealthChecks().Insert(cloud.Project(), e.Region, hc) if err != nil { return fmt.Errorf("error creating healthcheck: %v", err) } @@ -125,25 +129,35 @@ func (_ *HealthCheck) RenderGCE(t *gce.GCEAPITarget, a, e, changes *HealthCheck) return nil } -type terraformTCPBlock struct { - Port int64 `cty:"port"` +type terraformHTTPBlock struct { + Port int64 `cty:"port"` + RequestPath string `cty:"request_path"` } type terraformHealthCheck struct { - Name string `cty:"name"` - TCPHealthCheck terraformTCPBlock `cty:"tcp_health_check"` + Name string `cty:"name"` + Region string `cty:"region"` + HTTPHealthCheck terraformHTTPBlock `cty:"http_health_check"` } -func (_ *HealthCheck) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *HealthCheck) error { +func (*HealthCheck) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *HealthCheck) error { tf := &terraformHealthCheck{ Name: *e.Name, - TCPHealthCheck: terraformTCPBlock{ - Port: e.Port, + HTTPHealthCheck: terraformHTTPBlock{ + Port: fi.ValueOf(e.Port), + RequestPath: fi.ValueOf(e.RequestPath), }, } - return t.RenderResource("google_compute_health_check", *e.Name, tf) + if e.Region == "" { + return t.RenderResource("google_compute_health_check", *e.Name, tf) + } + tf.Region = e.Region + return t.RenderResource("google_compute_region_health_check", *e.Name, tf) } func (e *HealthCheck) TerraformAddress() *terraformWriter.Literal { + if e.Region == "" { + return terraformWriter.LiteralProperty("google_compute_region_health_check", *e.Name, "id") + } return terraformWriter.LiteralProperty("google_compute_health_check", *e.Name, "id") }