Skip to content

Commit 8e7b5ae

Browse files
vishalbolluvishalbollu
authored
Allow ssl and ip address whitelisting fields to be updated on configure (#2305)
1 parent cb9b110 commit 8e7b5ae

File tree

6 files changed

+95
-27
lines changed

6 files changed

+95
-27
lines changed

‎cli/cmd/lib_cluster_config.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,10 @@ func confirmInstallClusterConfig(clusterConfig *clusterconfig.Config, awsClient
281281
func confirmConfigureClusterConfig(configureChanges clusterconfig.ConfigureChanges, oldCc, newCc clusterconfig.Config, disallowPrompt bool) {
282282
fmt.Printf("your %s cluster in region %s will be updated as follows:\n\n", newCc.ClusterName, newCc.Region)
283283

284+
for _, fieldToUpdate := range configureChanges.FieldsToUpdate {
285+
fmt.Printf("￮ %s will be updated\n", fieldToUpdate)
286+
}
287+
284288
for _, ngName := range configureChanges.NodeGroupsToScale {
285289
ngOld := oldCc.GetNodeGroupByName(ngName)
286290
ngScaled := newCc.GetNodeGroupByName(ngName)

‎manager/install.sh‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,8 @@ function cluster_configure() {
9090
add_nodegroups
9191
remove_nodegroups
9292

93+
update_networking
94+
9395
echo -n "￮ updating cluster configuration "
9496
setup_configmap
9597
echo "✓"
@@ -365,6 +367,51 @@ function setup_istio() {
365367
output_if_error istio-${ISTIO_VERSION}/bin/istioctl install --skip-confirmation --filename /workspace/istio.yaml
366368
}
367369

370+
function update_networking() {
371+
prev_ssl_certificate_arn=$(kubectl get svc ingressgateway-apis -n=istio-system -o json | jq -r '.metadata.annotations."service.beta.kubernetes.io/aws-load-balancer-ssl-cert"')
372+
373+
if [ "$prev_ssl_certificate_arn" = "null" ]; then
374+
prev_ssl_certificate_arn=""
375+
fi
376+
377+
new_ssl_certificate_arn=$(cat $CORTEX_CLUSTER_CONFIG_FILE | yq -r .ssl_certificate_arn)
378+
379+
if [ "$new_ssl_certificate_arn" = "null" ]; then
380+
new_ssl_certificate_arn=""
381+
fi
382+
383+
prev_api_whitelist_ip_address=$(kubectl get svc ingressgateway-apis -n=istio-system -o yaml | yq -r -c ".spec.loadBalancerSourceRanges")
384+
prev_operator_whitelist_ip_address=$(kubectl get svc ingressgateway-operator -n=istio-system -o yaml | yq -r -c ".spec.loadBalancerSourceRanges")
385+
386+
new_api_whitelist_ip_address=$(cat $CORTEX_CLUSTER_CONFIG_FILE | yq -r -c ".api_load_balancer_cidr_white_list")
387+
new_operator_whitelist_ip_address=$(cat $CORTEX_CLUSTER_CONFIG_FILE | yq -r -c ".operator_load_balancer_cidr_white_list")
388+
389+
if [ "$prev_ssl_certificate_arn" = "$new_ssl_certificate_arn" ] && [ "$prev_api_whitelist_ip_address" = "$new_api_whitelist_ip_address" ] && [ "$prev_operator_whitelist_ip_address" = "$new_operator_whitelist_ip_address" ] ; then
390+
return
391+
fi
392+
393+
echo -n "￮ updating networking configuration "
394+
395+
if [ "$new_ssl_certificate_arn" != "$prev_ssl_certificate_arn" ] ; then
396+
# there is a bug where changing the certificate annotation will not cause the HTTPS listener in the NLB to update
397+
# the current workaround is to delete the HTTPS listener and have it recreated with istioctl
398+
if [ "$prev_ssl_certificate_arn" != "" ] ; then
399+
kubectl patch svc ingressgateway-apis -n=istio-system --type=json -p="[{'op': 'remove', 'path': '/metadata/annotations/service.beta.kubernetes.io~1aws-load-balancer-ssl-cert'}]" >/dev/null
400+
fi
401+
https_index=$(kubectl get svc ingressgateway-apis -n=istio-system -o json | jq '.spec.ports | map(.name == "https") | index(true)')
402+
if [ "$https_index" != "null" ] ; then
403+
kubectl patch svc ingressgateway-apis -n=istio-system --type=json -p="[{'op': 'remove', 'path': '/spec/ports/$https_index'}]" >/dev/null
404+
fi
405+
fi
406+
407+
python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/istio.yaml.j2 > /workspace/istio.yaml
408+
output_if_error istio-${ISTIO_VERSION}/bin/istioctl install --skip-confirmation --filename /workspace/istio.yaml
409+
python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/apis.yaml.j2 > /workspace/apis.yaml
410+
kubectl apply -f /workspace/apis.yaml >/dev/null
411+
412+
echo "✓"
413+
}
414+
368415
function validate_cortex() {
369416
set +e
370417

‎manager/manifests/istio.yaml.j2‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -50,9 +50,7 @@ spec:
5050
service:
5151
type: LoadBalancer
5252
externalTrafficPolicy: Cluster # https://medium.com/pablo-perez/k8s-externaltrafficpolicy-local-or-cluster-40b259a19404, https://www.asykim.com/blog/deep-dive-into-kubernetes-external-traffic-policies
53-
{% if config.get('operator_load_balancer_cidr_white_list', [])|length > 0 %}
54-
loadBalancerSourceRanges: {{ config['operator_load_balancer_cidr_white_list'] }}
55-
{% endif %}
53+
loadBalancerSourceRanges: {{ config.get('operator_load_balancer_cidr_white_list', ['0.0.0.0/0']) }}
5654
selector:
5755
app: operator-istio-gateway
5856
istio: ingressgateway-operator
@@ -110,9 +108,7 @@ spec:
110108
{% endif %}
111109
service:
112110
type: LoadBalancer
113-
{% if config.get('api_load_balancer_cidr_white_list', [])|length > 0 %}
114-
loadBalancerSourceRanges: {{ config['api_load_balancer_cidr_white_list'] }}
115-
{% endif %}
111+
loadBalancerSourceRanges: {{ config.get('api_load_balancer_cidr_white_list', ['0.0.0.0/0']) }}
116112
externalTrafficPolicy: Cluster # https://medium.com/pablo-perez/k8s-externaltrafficpolicy-local-or-cluster-40b259a19404, https://www.asykim.com/blog/deep-dive-into-kubernetes-external-traffic-policies
117113
selector:
118114
app: apis-istio-gateway

‎pkg/types/clusterconfig/cluster_config.go‎

Lines changed: 39 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ import (
3939
"github.com/cortexlabs/cortex/pkg/lib/pointer"
4040
"github.com/cortexlabs/cortex/pkg/lib/sets/strset"
4141
"github.com/cortexlabs/cortex/pkg/lib/slices"
42+
libstr "github.com/cortexlabs/cortex/pkg/lib/strings"
4243
"github.com/cortexlabs/cortex/pkg/lib/structs"
4344
"github.com/cortexlabs/yaml"
4445
)
@@ -191,10 +192,11 @@ type ConfigureChanges struct {
191192
NodeGroupsToRemove []string
192193
NodeGroupsToScale []string
193194
EKSNodeGroupsToRemove []string // EKS node group names of (NodeGroupsToRemove ∩ Cortex-converted EKS node groups) ∪ (Cortex-converted EKS node groups - the new cluster config's nodegroups)
195+
FieldsToUpdate []string
194196
}
195197

196198
func (c *ConfigureChanges) HasChanges() bool {
197-
return len(c.NodeGroupsToAdd) != 0 || len(c.NodeGroupsToRemove) != 0 || len(c.NodeGroupsToScale) != 0 || len(c.EKSNodeGroupsToRemove) != 0
199+
return len(c.NodeGroupsToAdd)+len(c.NodeGroupsToRemove)+len(c.NodeGroupsToScale)+len(c.EKSNodeGroupsToRemove)+len(c.FieldsToUpdate) != 0
198200
}
199201

200202
// GetGhostEKSNodeGroups returns the set difference between EKSNodeGroupsToRemove and the EKS-converted NodeGroupsToRemove
@@ -1029,43 +1031,54 @@ func (cc *Config) validate(awsClient *aws.Client) error {
10291031
return nil
10301032
}
10311033

1032-
func (cc *Config) validateConfigDiff(oldConfig Config) error {
1033-
err := cc.validateTopLevelSectionDiff(oldConfig)
1034-
if err != nil {
1035-
return err
1036-
}
1037-
1038-
return cc.validateSharedNodeGroupsDiff(oldConfig)
1039-
}
1040-
1041-
func (cc *Config) validateTopLevelSectionDiff(oldConfig Config) error {
1034+
func (cc *Config) validateTopLevelSectionDiff(oldConfig Config) ([]string, error) {
1035+
var fieldsToUpdate []string
10421036
// validate actionable changes
10431037
newClusterConfigCopy, err := cc.DeepCopy()
10441038
if err != nil {
1045-
return err
1039+
return nil, err
10461040
}
10471041

10481042
oldClusterConfigCopy, err := oldConfig.DeepCopy()
10491043
if err != nil {
1050-
return err
1044+
return nil, err
1045+
}
1046+
1047+
if libstr.Obj(newClusterConfigCopy.SSLCertificateARN) != libstr.Obj(oldClusterConfigCopy.SSLCertificateARN) {
1048+
fieldsToUpdate = append(fieldsToUpdate, SSLCertificateARNKey)
10511049
}
10521050

1053-
newClusterConfigCopy.NodeGroups = []*NodeGroup{}
1054-
oldClusterConfigCopy.NodeGroups = []*NodeGroup{}
1051+
if libstr.Obj(newClusterConfigCopy.APILoadBalancerCIDRWhiteList) != libstr.Obj(oldClusterConfigCopy.APILoadBalancerCIDRWhiteList) {
1052+
fieldsToUpdate = append(fieldsToUpdate, APILoadBalancerCIDRWhiteListKey)
1053+
}
1054+
1055+
if libstr.Obj(newClusterConfigCopy.OperatorLoadBalancerCIDRWhiteList) != libstr.Obj(oldClusterConfigCopy.OperatorLoadBalancerCIDRWhiteList) {
1056+
fieldsToUpdate = append(fieldsToUpdate, OperatorLoadBalancerCIDRWhiteListKey)
1057+
}
1058+
1059+
clearUpdatableFields(&newClusterConfigCopy)
1060+
clearUpdatableFields(&oldClusterConfigCopy)
10551061

10561062
h1, err := newClusterConfigCopy.Hash()
10571063
if err != nil {
1058-
return err
1064+
return nil, err
10591065
}
10601066
h2, err := oldClusterConfigCopy.Hash()
10611067
if err != nil {
1062-
return err
1068+
return nil, err
10631069
}
10641070
if h1 != h2 {
1065-
return ErrorConfigCannotBeChangedOnConfigure()
1071+
return nil, ErrorConfigCannotBeChangedOnConfigure()
10661072
}
10671073

1068-
return nil
1074+
return fieldsToUpdate, nil
1075+
}
1076+
1077+
func clearUpdatableFields(clusterConfig *Config) {
1078+
clusterConfig.SSLCertificateARN = nil
1079+
clusterConfig.APILoadBalancerCIDRWhiteList = nil
1080+
clusterConfig.OperatorLoadBalancerCIDRWhiteList = nil
1081+
clusterConfig.NodeGroups = []*NodeGroup{}
10691082
}
10701083

10711084
func (cc *Config) validateSharedNodeGroupsDiff(oldConfig Config) error {
@@ -1139,7 +1152,12 @@ func (cc *Config) ValidateOnConfigure(awsClient *aws.Client, oldConfig Config, e
11391152
return ConfigureChanges{}, err
11401153
}
11411154

1142-
err = cc.validateConfigDiff(oldConfig)
1155+
fieldsToUpdate, err := cc.validateTopLevelSectionDiff(oldConfig)
1156+
if err != nil {
1157+
return ConfigureChanges{}, err
1158+
}
1159+
1160+
err = cc.validateSharedNodeGroupsDiff(oldConfig)
11431161
if err != nil {
11441162
return ConfigureChanges{}, err
11451163
}
@@ -1170,6 +1188,7 @@ func (cc *Config) ValidateOnConfigure(awsClient *aws.Client, oldConfig Config, e
11701188
NodeGroupsToRemove: GetNodeGroupNames(ngsToBeRemoved),
11711189
NodeGroupsToScale: GetNodeGroupNames(ngNamesToBeScaled),
11721190
EKSNodeGroupsToRemove: getStaleEksNodeGroups(cc.ClusterName, eksNodeGroupStacks, cc.NodeGroups, ngsToBeRemoved),
1191+
FieldsToUpdate: fieldsToUpdate,
11731192
}, nil
11741193
}
11751194

‎pkg/types/clusterconfig/config_key.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,8 @@ const (
5353
NATGatewayKey = "nat_gateway"
5454
APILoadBalancerSchemeKey = "api_load_balancer_scheme"
5555
OperatorLoadBalancerSchemeKey = "operator_load_balancer_scheme"
56+
APILoadBalancerCIDRWhiteListKey = "api_load_balancer_cidr_white_list"
57+
OperatorLoadBalancerCIDRWhiteListKey = "operator_load_balancer_cidr_white_list"
5658
VPCCIDRKey = "vpc_cidr"
5759
AccountIDKey = "account_id"
5860
TelemetryKey = "telemetry"

‎pkg/types/clusterconfig/errors.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -280,7 +280,7 @@ func ErrorNoNATGatewayWithSubnets() error {
280280
func ErrorConfigCannotBeChangedOnConfigure() error {
281281
return errors.WithStack(&errors.Error{
282282
Kind: ErrConfigCannotBeChangedOnConfigure,
283-
Message: fmt.Sprintf("in a running cluster, only the %s field can be modified", NodeGroupsKey),
283+
Message: fmt.Sprintf("in a running cluster, only %s can be modified", s.StrsAnd([]string{NodeGroupsKey, SSLCertificateARNKey, OperatorLoadBalancerCIDRWhiteListKey, APILoadBalancerCIDRWhiteListKey})),
284284
})
285285
}
286286

0 commit comments

Comments
 (0)