beta to stable

cluster/cluster.yaml

@@ -568,63 +568,95 @@ Resources:
               - Action:
                   - 'acm:*'
                   - 'apigateway:*'
+                  - 'application-autoscaling:*'
+                  - 'appsync:*'
+                  - 'athena:*'
                   - 'automation:*'
-                  - 'autoscaling:*'
                   - 'autoscaling-plans:*'
-                  - 'application-autoscaling:*'
+                  - 'autoscaling:*'
                   - 'aws-marketplace:View*'
-                  - 'aws-portal:View*'
+                  - 'aws-portal:ViewAccount'
+                  - 'aws-portal:ViewBilling'
+                  - 'aws-portal:ViewUsage'
+                  - 'backup-storage:*'
+                  - 'backup:*'
+                  - 'budgets:ViewBudget'
+                  - 'ce:*'
                   - 'cloudformation:*'
                   - 'cloudfront:*'
                   - 'cloudsearch:*'
                   - 'cloudtrail:DescribeTrails'
+                  - 'cloudtrail:GetEventSelectors'
                   - 'cloudtrail:GetTrailStatus'
                   - 'cloudtrail:LookupEvents'
                   - 'cloudtrail:StartLogging'
                   - 'cloudwatch:*'
                   - 'config:*'
+                  - 'cur:DescribeReportDefinitions'
                   - 'datapipeline:*'
                   - 'dax:*'
                   - 'devicefarm:*'
+                  - 'dlm:*'
+                  - 'ds:*'
                   - 'dynamodb:*'
-                  - 'ec2:*'
                   - 'ec2-reports:*'
+                  - 'ec2:*'
+                  - 'ecr:Get*'
+                  - 'ecr:BatchGetImage'
+                  - 'ecr:BatchCheckLayerAvailability'
+                  - 'ecr:Describe*'
+                  - 'ecr:List*'
                   - 'elasticache:*'
                   - 'elasticfilesystem:*'
                   - 'elasticloadbalancing:*'
                   - 'elasticmapreduce:*'
                   - 'elastictranscoder:*'
                   - 'es:*'
                   - 'events:*'
+                  - 'firehose:*'
                   - 'glacier:*'
                   - 'glue:*'
                   - 'health:*'
                   - 'iam:*'
+                  - 'kafka:*'
                   - 'kinesis:*'
+                  - 'kinesisanalytics:*'
                   - 'kms:*'
+                  - 'lambda:*'
+                  - 'lex:*'
                   - 'logs:*'
                   - 'machinelearning:*'
-                  - 'kafka:*'
+                  - 'mq:*'
+                  - 'pricing:Describe*'
+                  - 'pricing:Get*'
+                  - 'quicksight:*'
                   - 'rds:*'
                   - 'redshift:*'
                   - 'rekognition:*'
+                  - 'resource-groups:*'
                   - 'route53:*'
                   - 'route53domains:Get*'
                   - 'route53domains:List*'
                   - 's3:*'
+                  - 'sagemaker:*'
                   - 'sdb:*'
                   - 'secretsmanager:*'
+                  - 'serviceQuotas:Get*'
+                  - 'serviceQuotas:List*'
                   - 'ses:*'
                   - 'sns:*'
                   - 'sqs:*'
+                  - 'ssm:*'
                   - 'states:*'
+                  - 'storagegateway:*'
                   - 'sts:*'
                   - 'support:*'
                   - 'swf:*'
                   - 'tag:get*'
+                  - 'transfer:*'
                   - 'trustedadvisor:*'
-                  - 'firehose:*'
-                  - 'lambda:*'
+                  - 'waf-regional:*'
+                  - 'waf:*'
                 Effect: Allow
                 Resource: '*'
             Version: 2012-10-17

cluster/config-defaults.yaml

@@ -71,11 +71,7 @@ skipper_ingress_lightstep_min_period: "500ms"
 skipper_ingress_lightstep_max_period: "2500ms"
 # set to "log-events" to enable
 skipper_ingress_lightstep_log_events: ""
-{{if eq .Environment "production"}}
-lightstep_token: "aws:kms:AQICAHgrx06TPoR1aNmcPHJjFu5mmoICT5KJkx2fsTJpmXmbNAH+8Ml18b8ZkUO/0KAwtIZTAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMSf79AuT/RI5rvWWjAgEQgDuN7obV7JD4iBMnOJ4Th93DfM5j572dXjf+gWmHx4JKMTTJPX2w6hgfQXX3LjI49l0p479a6IXIlZJOSg=="
-{{else}}
-lightstep_token: "aws:kms:AQICAHgrx06TPoR1aNmcPHJjFu5mmoICT5KJkx2fsTJpmXmbNAHvvYXdV1r7NviF5S+Jyx5zAAAAfjB8BgkqhkiG9w0BBwagbzBtAgEAMGgGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMQBqSQk/2TuQOsHGOAgEQgDsrNbCwF4AxoQXZuxXUOPnuQhFCY02EhWcB4xqmjFy8DelZtiCldRtxRdLyDL4uXiEyV8vOFyhxgqso/A=="
-{{end}}
+lightstep_token: ""
 
 # tokeninfo
 skipper_ingress_tokeninfo_cpu: "1000m"
@@ -105,50 +101,10 @@ cadvisor_memory: "150Mi"
 node_exporter_cpu: "20m"
 node_exporter_memory: "75Mi"
 
-# Monitoring settings
-{{if eq .Environment "e2e"}}
-zmon_agent_replicas: "0"
-zmon_aws_agent_replicas: "0"
-zmon_redis_replicas: "0"
-zmon_scheduler_replicas: "0"
-zmon_scheduler_mem: "0Gi"
-zmon_worker_replicas: "0"
-zmon_worker_mem: "0Gi"
-zmon_worker_cpu: "0"
-zmon_worker_count: "0"
-{{else if eq .Environment "production"}}
-zmon_agent_replicas: "1"
-zmon_aws_agent_replicas: "1"
-zmon_redis_replicas: "1"
-zmon_scheduler_replicas: "1"
-zmon_scheduler_mem: "2.5Gi"
-zmon_worker_replicas: "6"
-zmon_worker_mem: "4Gi"
-zmon_worker_cpu: "750m"
-zmon_worker_count: "16"
-{{else}}
-zmon_agent_replicas: "1"
-zmon_aws_agent_replicas: "1"
-zmon_redis_replicas: "1"
-zmon_scheduler_replicas: "1"
-zmon_scheduler_mem: "2.5Gi"
-zmon_worker_replicas: "4"
-zmon_worker_mem: "4Gi"
-zmon_worker_cpu: "750m"
-zmon_worker_count: "16"
-{{end}}
-zmon_scalyr_region: "eu"
-zmon_worker_version: "v209-py2eol-61-gcd2c760-v251-py2eol"
-zmon_agent_version: "0.4-19-ga12da10-zv5"
+# Logging settings
 logging_s3_bucket: "zalando-logging-{{.InfrastructureAccount | getAWSAccountID}}-{{.Region}}"
 scalyr_team_token: ""
 
-zmon_redis_mem: "1Gi"
-zmon_agent_mem: "500Mi"
-zmon_agent_cpu: "200m"
-zmon_aws_agent_mem: "200Mi"
-zmon_aws_agent_cpu: "100m"
-
 prometheus_cpu: "1000m"
 prometheus_mem: "4Gi"
 prometheus_mem_min: "1Gi"
@@ -231,19 +187,13 @@ etcd_instance_count: "5"
 etcd_instance_count: "3"
 {{end}}
 
-{{if ne .Environment "e2e"}}
-etcd_scalyr_key: "aws:kms:AQECAHgNxQ3yghAnRNmN6GyaSh8l0hGHUap3hNqu00tmcgcOxgAAAIwwgYkGCSqGSIb3DQEHBqB8MHoCAQAwdQYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAwrD+79PBvwdNnIXGICARCASEVQTLFyRAWL1OWE1WSl19lm5wY6Cj38wWse8esMBlxUvYSDRZqqyKMJieZpDbAxaAnJO6FKEJdpOoyU8GsV8At/43DhtaHkdA=="
-{{else}}
 etcd_scalyr_key: ""
-{{end}}
-
 dynamodb_service_link_enabled: "false"
 
 cluster_dns: "coredns"
 coredns_log_svc_names: "true"
 
-kuberuntu_image_v1_14: {{ amiID "zalando-ubuntu-kubernetes-production-v1.14.8-master-77" "861068367966" }}
-kuberuntu_image_v1_15: {{ amiID "zalando-ubuntu-kubernetes-production-v1.15.6-master-81" "861068367966" }}
+kuberuntu_image_v1_15: {{ amiID "zalando-ubuntu-kubernetes-production-v1.15.9-master-89" "861068367966" }}
 
 # Feature toggle to allow gradual decommissioning of ingress-template-controller
 enable_ingress_template_controller: "false"
@@ -295,6 +245,8 @@ apiserver_proxy: "true"
 # when set to true, service account tokens can be used from outside the cluster
 # requires apiserver_proxy to be set to "true"
 allow_external_service_accounts: "false"
+# issue service account tokens with expiration time.
+rotate_service_account_tokens: "false"
 
 # use kube-aws-iam-controller for kube-system components
 kube_aws_iam_controller_kube_system_enable: "true"
@@ -308,3 +260,19 @@ custom_dns_zone_nameservers: "" # space seperated list of nameserver IP addresse
 
 # prefix prepended to ownership TXT records for external-dns
 external_dns_ownership_prefix: ""
+
+# special roles for test/pet clusters
+{{if eq .Cluster.Environment "e2e"}}
+collaborator_administrator_access: "true"
+{{else}}
+collaborator_administrator_access: "false"
+{{end}}
+
+# enable legacy serviceaccounts for smooth RBAC migration
+enable_operator_sa: "false"
+enable_default_sa: "false"
+enable_cdp_sa: "false"
+
+# virtual memory configuration
+vm_dirty_background_bytes: ""
+vm_dirty_bytes: ""

cluster/manifests/01-vertical-pod-autoscaler/admission-controller-deployment.yaml

@@ -19,7 +19,7 @@ spec:
         version: v0.6.1-internal.4
         component: vpa
       annotations:
-        config/hash: {{"secret.yaml" | manifestHash}}
+        config/hash: {{"02-secret.yaml" | manifestHash}}
     spec:
       priorityClassName: system-cluster-critical
       serviceAccountName: vpa-admission-controller

cluster/manifests/admission-control/daemonset.yaml

@@ -33,7 +33,7 @@ spec:
         effect: NoSchedule
       containers:
       - name: cluster-autoscaler
-        image: registry.opensource.zalan.do/teapot/admission-controller:master-45
+        image: registry.opensource.zalan.do/teapot/admission-controller:master-54
         command:
           - /registry-proxy
           - --address=127.0.0.1:8285

cluster/manifests/dashboard/deployment.yaml

@@ -47,3 +47,5 @@ spec:
           readOnlyRootFilesystem: true
           runAsNonRoot: true
           runAsUser: 1000
+      securityContext:
+        fsGroup: 1000

cluster/manifests/deletions.yaml

@@ -22,3 +22,12 @@ post_apply:
 - name: ingress-template-controller
   kind: ClusterRoleBinding
 {{ end }}
+- name: poweruser-new
+  kind: ClusterRoleBinding
+- name: readonly-new
+  kind: ClusterRoleBinding
+- name: zmon-external-new
+  kind: ClusterRoleBinding
+- name: collaborator
+  namespace: visibility
+  kind: RoleBinding

cluster/manifests/external-dns/deployment.yaml

@@ -5,7 +5,7 @@ metadata:
   namespace: kube-system
   labels:
     application: external-dns
-    version: v0.5.17
+    version: v0.5.18
 spec:
   strategy:
     type: Recreate
@@ -16,7 +16,7 @@ spec:
     metadata:
       labels:
         application: external-dns
-        version: v0.5.17
+        version: v0.5.18
 {{ if eq .ConfigItems.kube_aws_iam_controller_kube_system_enable "false"}}
       annotations:
         iam.amazonaws.com/role: "{{ .LocalID }}-app-external-dns"
@@ -30,7 +30,7 @@ spec:
       serviceAccountName: external-dns
       containers:
       - name: external-dns
-        image: pierone.stups.zalan.do/teapot/external-dns:v0.5.17
+        image: registry.opensource.zalan.do/teapot/external-dns:v0.5.18
         args:
         - --source=service
         - --source=ingress
@@ -65,6 +65,8 @@ spec:
           runAsUser: 65534
           capabilities:
             drop: ["ALL"]
+      securityContext:
+        fsGroup: 65534
 {{ if eq .ConfigItems.kube_aws_iam_controller_kube_system_enable "true"}}
       volumes:
       - name: aws-iam-credentials

cluster/manifests/heapster/deployment.yaml

@@ -24,6 +24,8 @@ spec:
             value: "1"
       priorityClassName: system-cluster-critical
       serviceAccountName: heapster
+      securityContext:
+        fsGroup: 65534
       containers:
         - image: registry.opensource.zalan.do/teapot/heapster:v1.5.4
           name: heapster

cluster/manifests/kube-proxy/configmap.yaml

@@ -24,6 +24,7 @@ data:
     enableProfiling: false
     featureGates:
       TaintBasedEvictions: true
+      BoundServiceAccountTokenVolume: {{ .Cluster.ConfigItems.rotate_service_account_tokens }}
     healthzBindAddress: 0.0.0.0:10256
     hostnameOverride: ""
     iptables:

cluster/manifests/kube-state-metrics/deployment.yaml

@@ -5,7 +5,7 @@ metadata:
   namespace: kube-system
   labels:
     application: kube-state-metrics
-    version: v1.8.0
+    version: v1.9.2
 spec:
   replicas: 1
   selector:
@@ -15,7 +15,7 @@ spec:
     metadata:
       labels:
         application: kube-state-metrics
-        version: v1.8.0
+        version: v1.9.2
     spec:
       dnsConfig:
         options:
@@ -25,7 +25,7 @@ spec:
       serviceAccountName: kube-state-metrics
       containers:
       - name: kube-state-metrics
-        image: registry.opensource.zalan.do/teapot/kube-state-metrics:v1.8.0
+        image: registry.opensource.zalan.do/teapot/kube-state-metrics:v1.9.2
         ports:
         - containerPort: 8080
           name: http-metrics
@@ -47,3 +47,5 @@ spec:
           runAsUser: 65534
           capabilities:
             drop: ["ALL"]
+      securityContext:
+        fsGroup: 65534

cluster/manifests/psp/pod_security_policy.yaml

@@ -43,3 +43,4 @@ spec:
   - persistentVolumeClaim
   - downwardAPI
   - configMap
+  - projected

cluster/manifests/roles/cdp-controller-rbac.yaml

@@ -94,3 +94,6 @@ subjects:
 - apiGroup: rbac.authorization.k8s.io
   kind: User
   name: zalando-iam:zalando:service:credprov-cdp-controller-cluster-token
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: zalando-iam:zalando:service:stups_cdp-controller

cluster/manifests/roles/collaborator-roles.yaml

@@ -5,11 +5,10 @@ metadata:
   namespace: visibility
 rules:
 - apiGroups:
-  - ""
+  - apps
+  - extensions
   resources:
   - daemonsets
-  resourceNames:
-  - logging-agent
   verbs:
   - create
   - update
@@ -19,12 +18,12 @@ rules:
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: collaborator
+  name: collaborator-binding
   namespace: visibility
 roleRef:
   apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: collaborator-visibility
+  kind: Role
+  name: collaborator
 subjects:
 - kind: Group
   name: CollaboratorPowerUser

cluster/manifests/roles/poweruser-binding.yaml

@@ -1,7 +1,7 @@
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: poweruser-new # TODO: migrate name
+  name: poweruser
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole