-
Notifications
You must be signed in to change notification settings - Fork 321
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
agentless: support updating health checks on consul clients during an…
… upgrade to agentless (#1690) We will need to handle the case when a user has many applications on the service mesh and needs to take time to re-inject all those applications. If that is the case, there may be situations where the application changes its health status, but because it still hasn’t been re-started (re-injected), its health status needs to be synced into Consul clients rather than servers the way we used to do before. To do this, we will add a new annotation to all pods pointing to the current consul-k8s version: consul.hashicorp.com/consul-k8s-version: 1.0.0. If this annotation is not there or if the version is less than the version that supports “agentless”, we will continue to sync health checks to Consul clients for that particular pod
- Loading branch information
Showing
14 changed files
with
713 additions
and
76 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
112 changes: 112 additions & 0 deletions
112
control-plane/connect-inject/controllers/endpoints/consul_client_health_checks.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,112 @@ | ||
| package endpoints | ||
|
|
||
| import ( | ||
| "fmt" | ||
|
|
||
| "github.com/hashicorp/consul-k8s/control-plane/connect-inject/constants" | ||
| "github.com/hashicorp/consul-k8s/control-plane/consul" | ||
| "github.com/hashicorp/consul-server-connection-manager/discovery" | ||
| "github.com/hashicorp/consul/api" | ||
| "github.com/hashicorp/go-version" | ||
| corev1 "k8s.io/api/core/v1" | ||
| ) | ||
|
|
||
| const minSupportedConsulDataplaneVersion = "v1.0.0-beta1" | ||
|
|
||
| // isConsulDataplaneSupported returns true if the consul-k8s version on the pod supports | ||
| // consul-dataplane architecture of Consul. | ||
| func isConsulDataplaneSupported(pod corev1.Pod) bool { | ||
| if anno, ok := pod.Annotations[constants.AnnotationConsulK8sVersion]; ok { | ||
| consulK8sVersion, err := version.NewVersion(anno) | ||
| if err != nil { | ||
| return false | ||
| } | ||
| consulDPSupportedVersion, err := version.NewVersion(minSupportedConsulDataplaneVersion) | ||
| if err != nil { | ||
| return false | ||
| } | ||
| if !consulK8sVersion.LessThan(consulDPSupportedVersion) { | ||
| return true | ||
| } | ||
| } | ||
| return false | ||
| } | ||
|
|
||
| func (r *Controller) consulClientCfgForNodeAgent(serverClient *api.Client, pod corev1.Pod, state discovery.State) (*api.Config, error) { | ||
| ccCfg := &api.Config{ | ||
| Scheme: r.ConsulClientConfig.APIClientConfig.Scheme, | ||
| } | ||
|
|
||
| consulClientHttpPort := 8500 | ||
| if ccCfg.Scheme == "https" { | ||
| consulClientHttpPort = 8501 | ||
| ccCfg.TLSConfig.CAFile = r.ConsulClientConfig.APIClientConfig.TLSConfig.CAFile | ||
| } | ||
| if r.consulClientHttpPort != 0 { | ||
| consulClientHttpPort = r.consulClientHttpPort | ||
| } | ||
| ccCfg.Address = fmt.Sprintf("%s:%d", pod.Status.HostIP, consulClientHttpPort) | ||
|
|
||
| ccCfg.Token = state.Token | ||
|
|
||
| // Check if auto-encrypt is enabled. If it is, we need to retrieve and set a different CA for the Consul client. | ||
| if r.EnableAutoEncrypt { | ||
| // Get Connect CA. | ||
| caRoots, _, err := serverClient.Agent().ConnectCARoots(nil) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| if caRoots == nil { | ||
| return nil, fmt.Errorf("ca root list is nil") | ||
| } | ||
| if caRoots.Roots == nil { | ||
| return nil, fmt.Errorf("ca roots is nil") | ||
| } | ||
| if len(caRoots.Roots) == 0 { | ||
| return nil, fmt.Errorf("the list of root CAs is empty") | ||
| } | ||
|
|
||
| for _, root := range caRoots.Roots { | ||
| if root.Active { | ||
| ccCfg.TLSConfig.CAFile = "" | ||
| ccCfg.TLSConfig.CAPem = []byte(root.RootCertPEM) | ||
| break | ||
| } | ||
| } | ||
| } | ||
| if r.EnableConsulNamespaces { | ||
| ccCfg.Namespace = r.consulNamespace(pod.Namespace) | ||
| } | ||
| return ccCfg, nil | ||
| } | ||
|
|
||
| func (r *Controller) updateHealthCheckOnConsulClient(consulClientCfg *api.Config, pod corev1.Pod, endpoints corev1.Endpoints, status string) error { | ||
| consulClient, err := consul.NewClient(consulClientCfg, r.ConsulClientConfig.APITimeout) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| filter := fmt.Sprintf(`Name == "Kubernetes Health Check" and ServiceID == %q`, serviceID(pod, endpoints)) | ||
| checks, err := consulClient.Agent().ChecksWithFilter(filter) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| if len(checks) > 1 { | ||
| return fmt.Errorf("more than one Kubernetes health check found") | ||
| } | ||
| if len(checks) == 0 { | ||
| r.Log.Info("detected no health checks to update", "name", endpoints.Name, "ns", endpoints.Namespace, "service-id", serviceID(pod, endpoints)) | ||
| return nil | ||
| } | ||
| for checkID := range checks { | ||
| output := "Kubernetes health checks passing" | ||
| if status == api.HealthCritical { | ||
| output = fmt.Sprintf(`Pod "%s/%s" is not ready`, pod.Namespace, pod.Name) | ||
| } | ||
| r.Log.Info("updating health check status", "name", endpoints.Name, "ns", endpoints.Namespace, "status", status) | ||
| err = consulClient.Agent().UpdateTTL(checkID, output, status) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
| return nil | ||
| } |
Oops, something went wrong.